From a8684320acb4a0c87c3cb5b66f759b846938b0c2 Mon Sep 17 00:00:00 2001 From: orion Date: Fri, 23 Oct 2026 14:06:57 +0000 Subject: [PATCH 01/17] feat(helm): add optional SeaTunnel engine Compose the fork-maintained chart through an optional packaged dependency. Keep standalone and cluster topology independent from HugeGraph modes. Allow engine traffic through the release NetworkPolicy when enabled. Document imports and graph copies in Chinese with validation status. Protect chart archives and generated lock files during repository checks. --- .github/actions/helm-checks/action.yml | 12 +- LICENSE | 1 + NOTICE | 4 + helm/hugegraph/.helmignore | 4 +- helm/hugegraph/Chart.lock | 6 + helm/hugegraph/Chart.yaml | 11 +- helm/hugegraph/README.md | 8 + .../seatunnel-helm-3.0.1-hugegraph.1.tgz | Bin 0 -> 14468 bytes helm/hugegraph/docs/seatunnel.md | 205 ++++++++++++++++ helm/hugegraph/examples/seatunnel/Dockerfile | 24 ++ .../examples/seatunnel/hugegraph-copy.conf | 61 +++++ .../examples/seatunnel/hugegraph-sink.conf | 65 +++++ .../examples/seatunnel/hugegraph-source.conf | 43 ++++ .../examples/seatunnel/plugin_config | 20 ++ .../examples/values-seatunnel-cluster.yaml | 45 ++++ .../examples/values-seatunnel-standalone.yaml | 33 +++ helm/hugegraph/templates/NOTES.txt | 10 + helm/hugegraph/templates/_helpers.tpl | 13 + helm/hugegraph/templates/networkpolicy.yaml | 3 + .../templates/seatunnel-networkpolicy.yaml | 54 ++++ .../__snapshot__/networkpolicy_test.yaml.snap | 8 +- helm/hugegraph/tests/networkpolicy_test.yaml | 8 +- helm/hugegraph/values.schema.json | 21 ++ helm/hugegraph/values.yaml | 13 + helm/test_helm_seatunnel.py | 232 ++++++++++++++++++ pom.xml | 3 + 26 files changed, 893 insertions(+), 14 deletions(-) create mode 100644 helm/hugegraph/Chart.lock create mode 100644 helm/hugegraph/charts/seatunnel-helm-3.0.1-hugegraph.1.tgz create mode 100644 helm/hugegraph/docs/seatunnel.md create mode 100644 helm/hugegraph/examples/seatunnel/Dockerfile create mode 100644 helm/hugegraph/examples/seatunnel/hugegraph-copy.conf create mode 100644 helm/hugegraph/examples/seatunnel/hugegraph-sink.conf create mode 100644 helm/hugegraph/examples/seatunnel/hugegraph-source.conf create mode 100644 helm/hugegraph/examples/seatunnel/plugin_config create mode 100644 helm/hugegraph/examples/values-seatunnel-cluster.yaml create mode 100644 helm/hugegraph/examples/values-seatunnel-standalone.yaml create mode 100644 helm/hugegraph/templates/seatunnel-networkpolicy.yaml create mode 100644 helm/test_helm_seatunnel.py diff --git a/.github/actions/helm-checks/action.yml b/.github/actions/helm-checks/action.yml index 07922fd85d..f486f919ce 100644 --- a/.github/actions/helm-checks/action.yml +++ b/.github/actions/helm-checks/action.yml @@ -38,11 +38,17 @@ runs: helm lint helm/hugegraph -f helm/hugegraph/values-single.yaml helm lint helm/hugegraph -f helm/hugegraph/values-cluster.yaml helm lint helm/hugegraph -f helm/hugegraph/values-strict.yaml + helm lint helm/hugegraph -f helm/hugegraph/examples/values-seatunnel-standalone.yaml + helm lint helm/hugegraph -f helm/hugegraph/examples/values-seatunnel-cluster.yaml - name: packaged chart regression shell: 'bash -e {0}' run: python3 -m unittest discover -s helm -p 'test_helm_package.py' -v + - name: SeaTunnel integration regression + shell: 'bash -e {0}' + run: python3 -m unittest discover -s helm -p 'test_helm_seatunnel.py' -v + - name: helm unittest shell: 'bash -e {0}' run: | @@ -56,7 +62,8 @@ runs: - name: helm template shell: 'bash -e {0}' run: | - for preset in "" "-f helm/hugegraph/values-single.yaml" "-f helm/hugegraph/values-cluster.yaml" "-f helm/hugegraph/values-strict.yaml"; do + for preset in "" "-f helm/hugegraph/values-single.yaml" "-f helm/hugegraph/values-cluster.yaml" "-f helm/hugegraph/values-strict.yaml" \ + "-f helm/hugegraph/examples/values-seatunnel-standalone.yaml" "-f helm/hugegraph/examples/values-seatunnel-cluster.yaml"; do # shellcheck disable=SC2086 # $preset intentionally splits into flags helm template ci helm/hugegraph $preset > /dev/null done @@ -493,7 +500,8 @@ runs: curl -sSLo /tmp/kc.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz echo "95f14e87aa28c09d5941f11bd024c1d02fdc0303ccaa23f61cef67bc92619d73 /tmp/kc.tar.gz" | sha256sum -c - tar -xzf /tmp/kc.tar.gz -C /tmp - for preset in "" "-f helm/hugegraph/values-single.yaml" "-f helm/hugegraph/values-cluster.yaml" "-f helm/hugegraph/values-strict.yaml"; do + for preset in "" "-f helm/hugegraph/values-single.yaml" "-f helm/hugegraph/values-cluster.yaml" "-f helm/hugegraph/values-strict.yaml" \ + "-f helm/hugegraph/examples/values-seatunnel-standalone.yaml" "-f helm/hugegraph/examples/values-seatunnel-cluster.yaml"; do # shellcheck disable=SC2086 # $preset intentionally splits into flags helm template ci helm/hugegraph $preset | /tmp/kubeconform -strict -summary -kubernetes-version 1.23.0 done diff --git a/LICENSE b/LICENSE index 8445ec58dc..830eabaac8 100644 --- a/LICENSE +++ b/LICENSE @@ -218,3 +218,4 @@ hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/opencypher/Cyp hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/opencypher/CypherPlugin.java from https://github.com/opencypher/cypher-for-gremlin hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Cardinality.java from https://github.com/JanusGraph/janusgraph hugegraph-struct/src/main/java/org/apache/hugegraph/type/Namifiable.java from https://github.com/JanusGraph/janusgraph +helm/hugegraph/charts/seatunnel-helm-*.tgz contains the Kubernetes Helm chart from https://github.com/apache/seatunnel, maintained in https://github.com/hugegraph/seatunnel (Apache License 2.0). diff --git a/NOTICE b/NOTICE index 8e48b813b8..126decde3b 100644 --- a/NOTICE +++ b/NOTICE @@ -6,6 +6,10 @@ The Apache Software Foundation (http://www.apache.org/). The initial codebase was donated to the ASF by HugeGraph Authors, copyright 2017-2021. +The optional SeaTunnel Helm subchart in helm/hugegraph/charts/ contains code from: +Apache SeaTunnel +Copyright 2021-2024 The Apache Software Foundation + ----------------------------------------------------------------------- This product contains code form the Apache TinkerPop Project: diff --git a/helm/hugegraph/.helmignore b/helm/hugegraph/.helmignore index e37a0cdfb0..5551543a94 100644 --- a/helm/hugegraph/.helmignore +++ b/helm/hugegraph/.helmignore @@ -34,5 +34,5 @@ testdata/ # files in the release Secret; binary assets can exceed Kubernetes' 1 MiB limit. /images/ -# Never package a chart archive inside a chart. -*.tgz +# Ignore parent archives; dependencies under charts/ must be loaded by Helm. +/*.tgz diff --git a/helm/hugegraph/Chart.lock b/helm/hugegraph/Chart.lock new file mode 100644 index 0000000000..fed710c7e5 --- /dev/null +++ b/helm/hugegraph/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: seatunnel-helm + repository: file://../../../seatunnel/deploy/kubernetes/seatunnel + version: 3.0.1-hugegraph.1 +digest: sha256:0940d9b2f7fb25cb2b43e6464f838072bc38bf371821735b1782ea15feaf257e +generated: "2026-10-23T13:15:14.300959334Z" diff --git a/helm/hugegraph/Chart.yaml b/helm/hugegraph/Chart.yaml index 16397760b1..a300f6cdcd 100644 --- a/helm/hugegraph/Chart.yaml +++ b/helm/hugegraph/Chart.yaml @@ -17,9 +17,9 @@ apiVersion: v2 name: hugegraph-helm -description: Helm chart for Apache HugeGraph HStore cluster (PD + Store + Server) +description: Helm chart for Apache HugeGraph HStore cluster with optional SeaTunnel type: application -version: 0.2.0 +version: 0.3.0 # TODO: set appVersion to "1.8.0" when the release images are published. appVersion: "latest" kubeVersion: ">=1.23.0-0" @@ -32,6 +32,13 @@ home: https://hugegraph.apache.org/ icon: https://hugegraph.apache.org/img/logo.svg sources: - https://github.com/apache/hugegraph + - https://github.com/hugegraph/seatunnel/tree/dev/deploy/kubernetes/seatunnel +dependencies: + - name: seatunnel-helm + alias: seatunnel + version: 3.0.1-hugegraph.1 + repository: file://../../../seatunnel/deploy/kubernetes/seatunnel + condition: seatunnel.enabled maintainers: - name: HugeGraph Community url: https://hugegraph.apache.org/ diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 5f12ba6435..d7390d33ee 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -81,6 +81,14 @@ helm install hugegraph ./helm/hugegraph -n hugegraph --create-namespace \ Inactive credential values may remain in the values file when auth is disabled. +### 可选 SeaTunnel + +增加 `--set seatunnel.enabled=true`，与 HugeGraph 一起启动一个 SeaTunnel 混合节点； +增加 `--set seatunnel.mode=cluster`，切换为分离的 master/worker 集群。 +默认关闭，SeaTunnel 拓扑独立于 HugeGraph 的单机/集群预设和 `easy/strict` 访问模式。 +启动、凭据、直接导入 Graph 及 Graph A → Graph B 复制示例见 +[SeaTunnel 联合部署指南](docs/seatunnel.md)。 + ### Internal image registry Mirror the four component images (`hugegraph/pd`, `hugegraph/store`, diff --git a/helm/hugegraph/charts/seatunnel-helm-3.0.1-hugegraph.1.tgz b/helm/hugegraph/charts/seatunnel-helm-3.0.1-hugegraph.1.tgz new file mode 100644 index 0000000000000000000000000000000000000000..3a213a0d8bc3c4a1f8788d550e13bce7b8d1b727 GIT binary patch literal 14468 zcmZvjQ+Op!w1#6(xZ_N0W8#Tz+cqXnCbn(c=EQa;wrz8gon)W<7w24^x~RLV)m{Cq z@9AFaZK5bR0N8&6L<2@=B&p17A}Plq>&eY-%%;w4s={uit-{Trpsvm#r(tVtWN+%J zs^q{YX=ZB&cG(SdCm2ov%g#HgD|fJ+mR`%qom()y_w2jBhnETb)u?5RMDK@$H~tynx5xW*UVH5+*@(GzDesKD z?4q&N;d)a8ti%?HA2kYpcS3v>GHZpVabIV??REB0Xwj~bSCF6Zt6H-D+&N!dd~_cv z9Vz=7R0`1w-eo)-+#KA`-xc{p3Eucw4&r^wyjJXvcwZPi4TJr{sRzlc1D@_6q6-gC zf>m5Ap;1`U_QNMNERK3-Vd>y%r5nJhZTC@Az#a{U5@7L?q5RuW(kuc1)(B6!T3$wYS_J!b#SoQXT9YKlcPN+9G9%1foGJymJRBm2pIld7F|Y}|zWYNj zl{;B3Xtt_90k6e+Xw-A4zlM}0p|@idzx7eyHy}d0B3y8Qk51J5iO2+qGgwqR*0pba z9FBl^9!-XjwZJb~KFdZasU8me_JFG;2sY*S0gvV zAN6tmVHuUdKkEZLs~nMTvCIvFX4z+n0fIAw+DE(fLYDcn`Q`8xVS|*8Qj4X3*7p5v zgTouKVzUQW>S;R~oF)th0Gex*RJ*^>FZeaLmQ<<1Yh3&Os4x&fk`+oS`O(Mv5;aR9 zL9D`%>=B!f1x9n|0e<<5>?RW5vmoguyU|(ogQJ6k2Q6|Bk@z>_`~SE!2D=a3jr9nY zu2Q4L$Hz~NtWKX+lV_TmW~!Y^qZkP>+0{KI8vzqj7)>WdS=`PcxL8rUl6@-3%cfwGS0wq24zjjrQ|LQ)9g{EdgX4Lp8tC^Cqo_ zt+}VC`7c)^U+8Q-m?jJC6D)P?X~tUrrn0`N{exXt#3*M9e`W^~gjsV@=+9sX`qS{A z{mNtlFyJJF`((nY&*+0xhhPwSe`c}GCb1>IizfdGX9;+&CSr@F zMzs!QThk_Bkw-%U6G1ChjA!z1L`CAG0fq;1TRAW8(m)M~u(_`gG*Ael2h`tVtnGXj15 z+w;Rt1NsDoAOK>)M)H;$H2%f)_2F7cI$-{T=HCD%I!g3lyL*(zx`kkZ0N^M3UBgCg zK*$K`o(zBm;Phby-43W|giZ_ow)qp1ETBmS?XVn%D@F+%tT`eB^-xR2T!{y0r^zQFXG>4fthb-4E!8pg*XaTvSfuP;4j3pGdl z^cSh&W4Oj^6OTYp3?EgE%!9-!yjT^J7cF1ZPi+3uL&DM$#jeSSQg}I;T1FTi^K*MD zYg@=>OZaX)B#O!*!am~k7@82vW612AU0MFRT66sBIVJR7@=jrivys$nhN3 z7MQ=0q55dO7Q zqx?Srv?5EiyQbel3Uw~nZ-6~S^4ql3d{M+omgi>s#mAWn8ZZuV$U z+9iI=xZ*-legEVV2IkYXW&`>8uPGmL>!-hn5EP1)e^R6Y zDZX$%^ws_o@To~$(-Z~u6kLawKtV3ai!nEJ-2H(;3*tc8KU2v=e^ZOltR>KilzaVk zYq!OcMZ*!oYBV9SOA=`#J*3|u!~i2gO8*=xnAGS43ZDxj6#{D#D-%&V&<)j_NC}X$ zh7&!B`u;9Z+uo};z?7CIuAm(fC`kqR@qQlo{f0=*t4QsAIUK_<`&e=RTQvu|@laZ3 z)if>a`p7qIOW`Q~bew?HPMp)L!=hiGc%hsmuCUw}5E4Po+RyvZNXMU1xbzqBO?D-1 zYTO+G07*az_|Eo+_R+A(qC}Pi)9D#9_W;qR|E}U}9ix6TohA=mw1bPU-SsEp+_|25 zuKD}7Rda~y@6(YxQI>jAsiBQ21)cY3FCzawf2hi9mu4vdevK(%CJUBH^?>WK}S2b8eJS-y=$MYAUyikOQLThWl0Uk<2Fi2i5 zx}aY_W*k4=xj?+xpq3LqCVhHH6n4jO(Osj`TiJgGP)9WkD&o`QF`|I5 zF7QZ{dMD)`xNNgSvaxuw%L{79#m&Mb1u}mkNcn?VXcd)v1%m3K7?^FSVW)Kf0^h4U z(PsLzXI`l5A)DGa@r0yW4JYP`!&JE^>Hv{F!j5tnM*XRM6knv=Sa^wiszW$seadh6 z|M=_Oq)_8$?-+`i^r5(1K34Hp`N}x4z>$nFJdeD1qd<`egs|=;T1^zzm16O1)^6*zuy(kf*Hm@#u85NW-B;wJNCWDRl7L1JU`Y;W>alq{;|1^V29?l=qZ^$q2<5 zfU|01f#vZRB$W zEepk9`0NgibW!)1*SxX47&B*$eab|f@|JEY3GG9&x5D0MQ>ab;x9U()r1@_}sX;7> z&;v#b`^IS=anG0ixkW3;L|#w4G<)TavFaFz#AvW?Opl^x90a*|`PZ`cg?Y zEelW&(RgV@I*n0=enWOaTAOJoJW^=2YUbKU^yA9*&H3&QAl-9lC4tw1sIO6mK7n=U zFq4$*hH>mq($*BBB9a@1NYRsbwvKS&mo}O0W6|fc=0 zL~JJY9>i3o{afb#F&Hf07Dv4d?VjYZcHJFvpk%~NQhF7;1U)jr?UpnxJ73BG!C}26 zJ_2~J8@k7;96N4^5k|lP+F=bk*JpCGVVS7ZiBaC7nso8K9%3}0dtFK~Om;=47e2{f z*eb^3Feu>w2C!dwO3h`@rhtP8KQqw%4YD^Ke63H;YKI!0Ff1UCBMC6?tlN@AEZ9ur z!?Pzzrm}UW4Z*Qb0{Dwy0s50bz?);KMIwbx$&-)3MG7+uZ^@wEE<}r+c=ZiDgcN~# z^0G*`t7QjIpEnrXZ4v=G1?9rHU#d?^m_evNH!zc7WnfRC|aX)8I)HpFN^dbAaVCK z4ZhDXgOX-INC+mTpMC2;IMp==3p8YM%7p#7k`afJ%;vf=@Y``F8WzC8%7?n)QY!i5SQo)-0*E@C%c!$Az8_WvHe-`4$@*8H4SkoydQ0v!BWdC_Q zco;T-@X;A!S1RET!^ow8+r;4-K1EyrYj(5rn%&dha?mR)R+KFoApk9-ky5UDK_ZBZ z2-6d8zPx0a9vLRzSiJSU(}usqK&IPVKh^@0L24yENibP+JlTN4W^Da~X7q5^E-TP1 zR1q?&)UNE34YSj|y0qlCt(z|r)sS6R-mZFqAx|&nT+31PchFcxO!1lTpePf=DGMvBX4dz& z*5tArHG?L#n0rmm4iS6Yzo{%hNw!IutM}ivP7|pOch}3dkUU_%xT@z|)r%xHN84#z zn^8n3%d>=#PMolMZ5%*>w?Q_45<-c2cEnvgh`0e)IBF*}eTmj}FeFZq(@AHj%M%FQ zFtVauQo*b-=-~shd9Spz!>~D0YC_P-{)=;zV}N5S{89d9{BQ?9Dd!bcd3?A|7;(gl z)w=^nY*Xre`r%u6^ajMlVk*PN**G&XNSS!-eo>=v-K4GJFDJu+jFZLrV$f1UuHgMf zB~ZA2n8W!^Sc9nXAmd48#4uhM=3_}pq5Vp1G8#*!mC<6dgf|nV#fnWtMrsQo@bYS2 zOmXVkfHBb6yy#bhLYPZ_5gE~zXszYjn`0-t|I}IZrB^E)!U_sXge%ZHlChlccsTAQ z8)s+zoExk9t{QvJe~K(8n_^6-rHe!Hhz^@+UFA~8({*saqmr*)cP7??r~KrXE9`HD zz31|pr+LXx%5el)Dg1UUX@8>cAHUJ1XMX2|c3UXyy!UedE(^&&A#>xjfVgG{39#^| z{sqNpP>qDXg2+6-CZkWkbkIyLC@niMjk~qFST4iAQ}ic^a{Oe|KF(+l%6YKk2u(C8 z0Y}cZ4U~)98M5+4ST9%K;eu|{w;mWQsqn*q->yVW%cOgga9&rqGZ>eG^xPMJ5l&%X zoRs+O58^q{49c~-6FUGY;2mBLLx99^TjBrHOVGfqgO?RY!r?&)_|KaKNHiw>~pI%cx zSX?u6v^VR4N(4L~Pv_(%s)v;GYNO1PPX$5{49(&*@j_0W)Qw(YvjX_dMYPjTTTz43@KoK(b`BO z_RpM!eL#1?M*l_4Q_hEZDoqU&H48cW!j4lY+x{;&K|WQ6DB&MP2K$PDcZ7?S7dgNx ztC~KeWibZb0GAP7g8kHAWgR-)7!Pglh(c8kCpUYvyZ~UPfrHauR7txg*j#+0lt3s^ z{>cbGGZr#2$mUeXFNL6(k4%6*g>lumYG*YhZCDgy`f_0x!k1icH(_f?A{QKp7A>K& zqb6v*3*&&eGK`roQCf)-jVGXku?-g4uA4{U(Wf0xYEHbbq$ojRZHO%*fVeflX7+kO z&N#v5K4j3$W$pYuG?DURH42&@!S;0)Y?iW+(cCjlW+DR*Dv?32u$2^>5p(z!;XV;- z{J}ODNZZRnKD;7|0e%-eShk5ixa;CnO6Yu+Eu*2}`DdgG+PVbb7!f8tzN)hFOv79$ zI35jp>>+fO7ms|?@=>=Tbj4|1;*He2oV+8c6Ge*O2qbYUz$MNeh%X_D>FR_HomvsV z2ZInJgs)A$GHf*FSdc?=Gi1qM6l(0qk^_*sPO6p;wmv&al`HfaL&V~Bg5NVI!xL9+ z#G!R8mjF!vP%@`_NlI$-Ksj1=Bp`u_@od$AdBKsp`?&xQDGKC5z&aG8&}AP3=j=)L zR>dN`7HS))*0HQZoQ-r6Z`BQ}grfUU^W|_vdyp(irI#CO%Si{}LI}5^To03C=8_Sd z!}ncXS-Xkh=#D7_o9D7%I$rP+3j>6@kTVcUh7?sfKgto=Q)gNXRCzWBw z2D24eHfgj+6H_QQe3-xw2YCvPu?s@~K3mTB1*asPq{y6$)X$Oj5CHrYHSQqzNl!^T(TF48Ic|Ubp@i2MEl9>_ zhnZsj9!4++<5H49t;NAgVvVgF$P@lUxppQK084445PLEz$ZA^ij1F-QZeeQ`7zUQQ z2VZ|rOs~~hYe{BH_>&+Y_f(L^^Fqx3DZM&{p#Nj4oM-RI7@GatWye^7}Mgu!q)t(unc@Qt0S|hfW@@+ z7;jnYuEie(aw5UD1~ET7$W77QOwieSzd=Z@1fLgGxSQ%SlBZ2Nm!!DnR>qMdn5pR| zzp$*EEYY~3>}C7qxNJoC@#zs}`3nocw2KQNvMNeY|Ih&|mT%mUf+Z1A7^Ob!v0SG3 zK{}y}7`B4vD~%SzyKA(k9${{gbWK$;oT7=V!BE8)glm(&rj0Aq22BqEULbsKcQnkB z#2GR$7=;>F;4tJ6q`0%um)nKJMlUnX*2UL7LqT{Z<{vHiB>qV$oeYIi{-Xy-uW8Am^$sG z5ydyvzQ_5n*US(d}AgDEoUPXiP;{eNHo5}-x45NS%>uk@h4O|A9^L>@g7N- zHI_0a^!5SHaO_3q4a5}=cNEDIK{y-R_>54J$-;+0uJ}{hoS<+Fdyf+SqqVScRu7MT zC5+3kqk5^`O9#B6BJ1=oYI`_~WCKCYx5&ZL2uoCs))~ToWVV3v?`)#5{z6-q?@FzN z3*OKRB$#{FHj@!<_l;v`lpIiW(KXI}!Xmc<5g|CEx??T_fY>Yq=+)r;CRHaMMiaq>1DaasvMwT}_GoIoN5LT|oSMof#5 zz7=*IX8+J66;j&borW6|Ep;g|qK}XL=2=SRa_=gWU73Nc(ba&dxq!9o-2|FCqr=ooC*siek%ki{)&)9Oeb}5 zkTxcUrrKzXQcosLVALBHY*-Jks%}h7+&;~U03k;?Q5)g#BmHlNy@DO ztZ(3w7`vD*lv7hqwsy!=Gl{N!)%V{nTD1A2l4ph0hiT9g!MTdt6Er5WzqMPjwf=qz zXT57zNc}bZ>zei~qJb#ufbddW1>hJ9h$F_>OU@}-QJbHmMIp9e52F0RRodKI>S|YE z4EgI{Pt6pYuzB2tEd;`@c%?<~xNRv`JwAUyi9M~z!G)F<<9GbWd}T6D8F{%xC2Vr4IG-kw62q7Q*Hh%VG*Q7Q54dQHAOlc}9{!^X)s$0dz)6)hDc6HuP z+Vk)hkz9Vjf!NcZyiMl z8|*#X`F{TU(XwJPAsNtc94T1@%TeO8I;O^K)TuyiY;lrUK)x4IT4SppDY#;*HiAH< z-=;lKcto9)Dn**2OCE1DmW<@~DC-}orot4@Z5|P6h_@c?eQAvLDILBHC6%~idD1OO zmtj`Y@K#!@tYFZbu5d`LQD<+Fhk~AI#VJ#01s@GjpsO)6#b5ar6&xi!ei_ruGK!*uitW-+|B5w|Dy=t*#BJE7!&Tml) zDxC*I9y2EuU=$WEn#3b?H0Su#EU=2?pIFxf8+lze0hZ7L=mPEOhzi$BSS+-TW1scM zN(48Lj2SVqD=GBUGk<&+Iyt+X!OIKWYs%ONsP-%AtX;J(};n{T&R#rOu5$bw5IsO1`rqz>IK@DX#c^BF0?OjE- zh50JC5ku46>_7d|E*n(RhFQTL_^WNsVK)B1(8cN_r01!^q4(cX{aO$P9EOa}Hzmnd zO(?=;0tMmf1zyV6rh;d{Npm}p}3)7#JPF2ViFg*4YS z!hphzKE`C2DGpu71H`oiU*vwTeH|YNSg;#Gbkzwir!Sv!F{j)xo5mEJ6;#ek21~}7 zI2vfgNRSS)zM^u^^`eGjQ5Z2{{e6MI30?u@L8SOdja1|vcu-SeNS7ijIiQ zLq%^)xvBW3h2+7$O<{)zZ})Riu%OJkIsE2HGDieYQqDB{A~^EQF_uQoV$$Uphc&bu zho2`x`+ORIWn+C!l6MgcAZeA!^?;xHL)+dYFbaM5b(q^o<2OT4(vC%H>hV`Y(pMFO zK6x#W`hJ5Ct2N32JtfAJ5gx3*V+MhuJDN00j@h2QpTE%QAUyV1+KPOqZ|ns}u;2ot zG}$*t*!|ltm`LolNUQjLkrmC(Ry4{AHP=N8v3X>i46;1|@(bu{7HfZyK#`#M1*@zX z13EC*@ebpsi;H1~osGSEH0Clg_oghRu$f{Vs$-uCoM0ZBU;=H;L$P?&WTUh_P7+}i zLT7OnF0P4fsrm)AUL&c6h zNtE2gt~wrDmJ=?lcgCGt_6`A2=nnH^GmZA+`{ShvK6b^Y>*6GWoBi^UwudOsy-5S6 z=*aa+!m8ZSIRLlR``9++&2C`MbqUz6B8Q43to5#aFokA3E0`N++L-F?r_xB>?R*nk-w0t;| z!hu9Vm@qb(dO}!r=7^TejP00gHmbuHC6Jlkw%>7(m|r;5HL$#nKWl&?&8vyR`K&?X ze?d*}pok+DuKf>l-VTy^08@NR8tj*5JJ^bm7!vh@rITYS^~i93#jRT~?WE}eEY~A} zk`WU*USP5Na^Vz}nmKegX`qOw00jbCcahuEn!zi!*>WJ^I*NE@zNHXjW$-uCX^bQe z5yJ((_&M6ABgxr#O*VjhV`VR68UE2*NaUs{2t*RIo;6n<)VA z0os{^*hx@1&Egac-S^GCq#*5tQG&gCZn~4UlNF`0rOh1W_zyA80Ef>%Y;(j$Nghv~ z7|75E)MtcfBUyk3pM))dFV7hyrIDFb@6rlVxO>yMhRAzR{En<2|Ir&x$a_UNy1ec_ zB51cf&sUR*ioe89>rq2PUj3d4*^u)Glpe|Pszr(?u9P8q?!l@8{BJ0a7lO>7+ht6o z6Iz1CmI>LM@!)fWz^cYa+#wv>Lbb8wXS`yGD9(sStMe4xW#p}S!g4>u+J zGC++n-s?z7*<4Y)zXFhRg$f#wjW{uWZNfzCY9jfBSM(Ia{>(%eK5odWQDrkCot>gtr(V@D!AZ{QQ`ySsr& zZ5qsG%S{9JaafIpyh$upMA$KP#Po>wQVdq11pIYL<4a%H%D=9avYujv>vy6SudDct z)q^jXN3-SPJ+Gl*?^QtJKE3HvR#Zi@EMTun)p8rGLxMn++HLeGE`X5XWomUnJ9(2S}U{X8tAS z<7gKs?#+odm~G+-oAiQY7a7HP!F*tk%wHCfe~CJe-*ZP1?jL8?bI>q3XNGU*o7A3m zgsCk;{aH!{Ok4B{T`w?FK3$Sk=b^l&64Gx$vFRra^ctm4wek9e`bxbvVU9&$wju*H z6t&bZ{6?QA+^BPIguq8cF{BRc;v|+qCShR!+9^5&Nf?gx5&T*?tlXQ0WFtiC%SiVc zvg6iLaU}gc70u%$#;Br9wZJ7os}qNl$B9WTwFcK=Qy-yOqv$BLOJ3VxfLt0eRpUr|d_RmAwfzs& zEq4kvU8+cE@kGr?^-Q%hB_k(U4jT9ZEj3k~Gc-*_sE%Y*&+t(I`bS6I#eXyN>NCxYvnKld5h&I{OH^5-F zrJ_YaHcK0g<=Y&0Vc=ns%Rg5*aqv%=tt2)(JWo-2x#N!KG%%u99w9|LL6}TwcYiyU z+=MDaOOE7;?W#=@tiQ>*-+(Off&}UhI%N4Z-HTc-tUAdFkB;swBw-WCqoH#lqMbZg zo6%3BwA!=mWA#24cp>d%3md^G4Ul$)_C*pqpre=-UBcI|+cjh`U*OdMrmCBncYoBX zNN&(z&HJ&Gkh2y+AE_ET@5;GBUku8W^#`lSVS%CeleeX*|7=7<(_Q_u@7GC6 zo-1STtmj9|mb-7E($437_Z2=TuZBRo&soY9=yjW>yC2cUs3a-Y z6_AEHso=e2bBI-Ovw{z(@_G5cRoxD0`dGgW{qs2Jw_36J8NPXkLDi%2v0WTxSj3v& ztu)s=-O0(jvc1;+_V)Z2rPW~*>f{CS^_|l|_uBhj>*P2q)AxibyOPcA^DDlaeNQ{^ zIP6^Psf#$iaOXKj!`*&!>;~(i>z7uUPEXHcc03fBbYdnt6?}U(To4;29@FOMJ9b;G zk0Y?=g=fYfW%$0hvIGu5zI%mFbkp;Yq_dK@<9SiH<9#Pzpa{cZo|9&<`SEcgOuA?% zpYlz31r^#kG4#t6eLlD9ciFA$n-;u&K%JAPG>4D97@fNJ66dF-X~ewi zE~m5S-)T^7;9Pk;f_6Cn0=Z6@s$mt-+LZ(07kManDoxP9mUY(za0iw@z#A@ zKjSm``JAn7FzK#++^f4m<}U*x&yf1v(E zSRp~pR`2GG<4Bm5$*8}}Ce3~Ot?C-I!DQ$6c~-O2`v-WvvA?kMNjjUs-c}!_LDb~~d`02a_WcBX!+-0K2OP{RM(-#n@3VoMqqw?`0W!O~0QFMW z_j)KY_8Os`gU^@-_MNB#q)hOhNcNEzMd=oP`^%Xu+5|dmtJh=ba#omR zET*qKcolgW?AFr+V`J|P+V2srq}1>e;&~F z(J=2KjUkf+{LeHOQc3j0pa7njlWh*~t|up73H?&`wSS5g=s3#VeaG7mcudr+>@CKu z1+QN4Fp$^g@cHZHzUfQiv+eJ0CkF;Dxj@BeU>xJl+u>S{h2P5SZlvkibf@2zC$IxY zBG0?^VLVGq=HL4C#tL*OI?F;iVOm?y+tKr1cLxKn>t{rbp4a`xP%=7%5UQ)xd6|yC z_JN^remw@bJa&@yXv%4wCD}zG6q9FN|Xd84>mpAZ+yjlnzjLv1fL%t=@@ix zc)qHXK#R(4fF@r-jPUqpJRyzpmfL@i{D6Nz4%L(Yb!yqWKp@H7y$`>gKp-e|1@wLE z$N?UOqx`bTl@{C`$zJvR8z!YY0>rSla%xQfQQhwdogEV3T|0^k-5%N zGr!_cg>P12S#KIQGHQe;cVADO);C$zswD_8l6_-cE6omh|3IF?DSz6V3x1Q6eX(77 zI;O#d2S_v*G57DS*F4Jy>hRn&0uw5pRkt>OTa6CqXX>pepsz&P&ka zzu8ZsVYdI8w_hFH?n{s!{%3S|N%$RWbUjP+XdgkUyQ1W?M>)8O0ihg$p|speVscA*kxaT-^dDtwYe!Qo0!asaf9P z4`s_?h^;`OBeuS>Mp9rhJFySdo%ZJsScH-6Q=L-V_(N2E6JcC9@Dm_7d zg*=-eXY!d@5YZb*1mt%LLiqp{fCL^vAf`8vAj(x^d(yZhT#S$DkZgj+De!Ri#+9Ut zy&KgSN6cF$FW?ij#q;^b?eXb%J`~YV0QwL0`G2>1S%<{sf4M(90vTwcOtI{H1x;?X zG=zZ!IQTdc9zbOZvM4Jc!g01oP%rc+h#8D)0BkpXv-}AzP^sX($UjCw4t788Z$-mh zpLb3ad!W?sDQLwz%Bgi9hJ1 zIO!x=dZcJOX@r+X*2oyfx7u7rhVl{%2?ptTnqkO^9N$$b%%n#Rz8y#a#c}}l%4`g` zs3d~K7LeaVNM^fhUX5i6hc%|s6;CC#H-R{ip#mov&Ru}$g$B(9U;0I0b70y@2zt>5 zc0tPr9$;%_EzK{n@LRD;cvq!3rz(1{O@PB2`M-fnAuLXer+NEey`~Ec-?BkWXtsDA z7wCe!3TD5j0tebpSK#Xu^Mh{}iq}LJ@8-T(erzm-90q{*ld`6R1hLaRj|-2Ox*km` z4Sj$16c~%U2c2MgF;k+>P-i49bKGBgCM+0f%v&jp_;iT|UESlvcr>Dber2;KJaII1 zr>LKrT1)*Ew|y>4(~=}s_WX$-Ki?Gi|^5ok^K|0d4gGs>oOb*?lP!|V{R z(GLmcZkxT^B~m0)psD&XP~K1XC#+xRLfUJn`XRT6-8#1z^SD)>f8LIfwGfxHsB3F! ztwEq9myzBMK^{d^vMIST#d7N?1X2<1Dg?565PJ|`t{95wRl?~R=_)*o5`X(PP-Who z+Tn3Wj2fL8FY*BPB<2u)c;eYLNfU!3u}Xz60$NfE`Jam2q>yBCI-uMb4|1J~mBh+5-)15?fz&0gV( zXaWJ>xuYz02Fj0#CXNH@vA*r1Q#@-TfEP;8Y}o*`@(wm*q3*qu1{Z~J!bkE-SC8QN zk;#A~M#T!^g1w+gQ&`Bce2M_=H+xhFB= zBU$<Kn9tCFa$d1F*11@BXGD#EdsV5eugQit3}& zfrWtFR&yBK3y$@$MwvdFjez5!qfElgQw9I9oHHb`ku;O}Tg|Qo@^XB2^#R)7z8~Ui z6HBbH0-4Lli!f*;rljw>b9ule^j<7s6uZDAVsh}!OAW;>jO5mRvfLY=9^3D%$)X+= ziq}eX{#nk{i7g|~)U*)lwnZ(oTqR+%JvJOZ$_i+*Fca=C--tz{$!;nL|4aq@jQ4N? zh**6?J^swegXePGFyA9Xv)XF2j}tAi!t? zsyJnhN8s(!#Il1p$Yhnc`wD(CLPpEUSMqtfp4JX4Y26rfd7zDLpj+Xq_T%6PDNz09 zha_!_rR4RFDl$JfSzw7&(z>#!1`g>B8maWC{1%`=_QbU#Pt_yQG;__6YP?H%4NiOFP;EL|v0tw5?ovK)#!y=>nV9xCCVaYCo zZJ`TBy|Lh=P5q%780RD)0xhZwFI{q5!gq7XNJ}n%RVPlQ>8uULiH(q^QK11 /tmp/hugegraph-sink.conf' \ + < helm/hugegraph/examples/seatunnel/hugegraph-sink.conf +kubectl -n hugegraph exec deploy/hugegraph-seatunnel-master -- \ + /opt/seatunnel/bin/seatunnel.sh --master cluster --config /tmp/hugegraph-sink.conf +``` + +`--master cluster` 表示客户端提交到已启动的引擎，单机混合节点也使用这个参数。CLI 未开启 `--async`，等待作业结束。检查作业成功，并用 Gremlin 核对 Graph A 的顶点数量： + +```bash +curl -fsS -u "$HG_USERNAME:$HG_PASSWORD" -H 'Content-Type: application/json' \ + "$HG_URL/gremlin" -d '{"gremlin":"graph.traversal().V().hasLabel(\"seatunnel_demo_person\").count()","aliases":{"graph":"DEFAULT-graph_a"}}' +``` + +空图首次导入的 `result.data` 应为 `[10]`。[hugegraph-source.conf](../examples/seatunnel/hugegraph-source.conf) 可按相同方式提交，将 Graph A 的内容输出到 worker 日志；单机模式查看 master 日志。 + +## 场景二：Graph A 复制到 Graph B + +[hugegraph-copy.conf](../examples/seatunnel/hugegraph-copy.conf) 使用 HugeGraph 作为 source 和 sink，读取 `graph_a`，写入 `graph_b`。目标标签和属性不存在时由 sink 创建，两个图本身仍需预先存在。 + +```bash +kubectl -n hugegraph exec -i deploy/hugegraph-seatunnel-master -- \ + sh -c 'cat > /tmp/hugegraph-copy.conf' \ + < helm/hugegraph/examples/seatunnel/hugegraph-copy.conf +kubectl -n hugegraph exec deploy/hugegraph-seatunnel-master -- \ + /opt/seatunnel/bin/seatunnel.sh --master cluster --config /tmp/hugegraph-copy.conf +``` + +作业成功后，比较两图数量及按 `name` 排序后的逐条内容： + +```bash +curl -fsS -u "$HG_USERNAME:$HG_PASSWORD" -H 'Content-Type: application/json' \ + "$HG_URL/gremlin" -d '{"gremlin":"def scan = { x -> x.traversal().V().hasLabel(\"seatunnel_demo_person\").order().by(\"name\").project(\"name\",\"age\").by(\"name\").by(\"age\").toList() }; def aa=scan(a); def bb=scan(b); [count_a:aa.size(),count_b:bb.size(),same:aa==bb,rows:bb]","aliases":{"a":"DEFAULT-graph_a","b":"DEFAULT-graph_b"}}' +``` + +期望两边均为十条，`same=true`，目标数据包含 `person_01/21` 到 `person_10/30`。这里复制的是一个顶点标签，不是自动克隆全图 schema、多标签和全部边。复制边时需补充对应映射，并先写入端点顶点；其他 ID 策略按连接器规则配置。 + +更多配置见 [HugeGraph source](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/source/HugeGraph.md)、[sink](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/sink/HugeGraph.md) 和 [HugeGraph 连接器指南](https://hugegraph.apache.org/cn/docs/quickstart/toolchain/import/hugegraph-seatunnel-connector/)。Pod Ready、`helm --wait`、作业成功和图数据一致是不同层面的检查。 + +## 存储与网络 + +`seatunnel.engineConfig` 合并原生 `seatunnel.engine` 配置。默认 checkpoint 使用临时本地目录；跨 Pod 恢复需配置共享持久化存储及相应客户端依赖、凭据和网络，例如： + +```yaml +seatunnel: + engineConfig: + checkpoint: + storage: + type: hdfs + plugin-config: + storage.type: hdfs + fs.defaultFS: hdfs://namenode.example.com:8020 + namespace: /seatunnel/checkpoints +``` + +额外挂载使用 `seatunnel.extraVolumes`、`extraVolumeMounts`。完整文件覆盖使用 `seatunnel.configMap.data`；单机 JVM 文件是 `jvm_options`，集群是 `jvm_master_options`、`jvm_worker_options`。原始 `data["seatunnel.yaml"]` 会覆盖引擎配置合并结果。 + +使用外部 ConfigMap 时设置 `configMap.create=false` 和 `existingConfigMapName`，必须提供全部十个文件：`hazelcast.yaml`、`hazelcast-client.yaml`、`hazelcast-master.yaml`、`hazelcast-worker.yaml`、`jvm_options`、`jvm_client_options`、`jvm_master_options`、`jvm_worker_options`、`log4j2.properties`、`seatunnel.yaml`。生成配置变化会触发滚动更新，外部 ConfigMap 因为 `subPath` 挂载需要手动重启 Pod。 + +开启 NetworkPolicy 时自动允许 SeaTunnel 成员间 TCP 5801、HTTP 端口、DNS，以及访问本 release 的 HugeGraph Server API。外部数据源、checkpoint 服务和 REST 调用方通过 `networkPolicy.seatunnel.extraEgress/extraIngress` 补充原生规则；实际隔离依赖 CNI 支持。使用外部 SeaTunnel 时保持 `seatunnel.enabled=false`，按实际来源配置 `networkPolicy.server.extraIngress`。 + +本集成保留官方 Helm 的 Deployment 方式；[官方 Kubernetes 指南](https://seatunnel.apache.org/zh-CN/docs/3.0.0/getting-started/kubernetes/) 推荐生产环境使用 StatefulSet。副本数增加不等于 checkpoint 持久化，生产前需验证存储、调度和故障恢复。 + +## 维护与验证记录 + +SeaTunnel 模板只在 fork 的 `deploy/kubernetes/seatunnel` 维护。两个仓库并列放置时，在 HugeGraph 根目录执行： + +```bash +helm lint ../seatunnel/deploy/kubernetes/seatunnel +helm dependency update helm/hugegraph +helm lint helm/hugegraph --set seatunnel.enabled=true +``` + +版本变化时更新父 Chart 的依赖版本，移除旧归档，提交新归档及 `Chart.lock`。原生 `file://../../../seatunnel/deploy/kubernetes/seatunnel` 依赖负责打包；最终安装使用随 HugeGraph 发布的归档，不需要相邻源码仓库或联网下载 Chart。 + +2026-10-23 本地验证记录（台北时间）： + +| 检查 | 结果与范围 | +|---|---| +| Python Helm 回归 | 23 项通过 | +| Helm 单元测试 | 309 项通过 | +| HugeGraph 推送前检查 | `mvn editorconfig:format` 通过；`mvn clean compile -Dmaven.javadoc.skip=true` 全部 38 模块通过 | +| Kubernetes 真实 API | 安装及模式切换已执行；业务 Pod Ready 为 0，尚未证明集群内引擎可用 | +| 本机引擎启动 | 官方 SeaTunnel 3.0.0 二进制、Java 17；`/overview` 返回版本 3.0.0、一个 worker | +| 直接导入 graph_a | 固定十条 FakeSource 数据；作业 `1166014497995882497` 为 `FINISHED`，读取与提交均为 10 | +| graph_a → graph_b | 作业 `1166014497996079105` 为 `FINISHED`；使用文档中的 CLI 与环境变量方式再次复制，作业 `1166014645144649729` 读取 10、写入 10、失败 0 | +| 图数据核对 | 隔离的 HugeGraph 1.8.0/RocksDB；Gremlin 返回 `count_a=10`、`count_b=10`、`same=true`，逐条比对 `person_01` 至 `person_10` 的 name/age 完全一致 | + +以上业务记录来自本机二进制引擎，不等同于 Kubernetes 内完整业务验证；隔离 Kubernetes 中的引擎启动和同样两条业务路径正在补测。此次数据样例只有一个顶点标签，没有验证多标签、边、HStore 或生产故障恢复。 diff --git a/helm/hugegraph/examples/seatunnel/Dockerfile b/helm/hugegraph/examples/seatunnel/Dockerfile new file mode 100644 index 0000000000..85a46d5fc8 --- /dev/null +++ b/helm/hugegraph/examples/seatunnel/Dockerfile @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM apache/seatunnel:3.0.0 + +WORKDIR /opt/seatunnel +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl ca-certificates \ + && rm -rf /var/lib/apt/lists/* +COPY plugin_config config/plugin_config +RUN bash bin/install-plugin.sh 3.0.0 \ + && test -s connectors/connector-hugegraph-3.0.0.jar diff --git a/helm/hugegraph/examples/seatunnel/hugegraph-copy.conf b/helm/hugegraph/examples/seatunnel/hugegraph-copy.conf new file mode 100644 index 0000000000..a2fc221be8 --- /dev/null +++ b/helm/hugegraph/examples/seatunnel/hugegraph-copy.conf @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 将 graph_a 的单个顶点标签复制到 graph_b；两个图需预先创建。 +# PRIMARY_KEY 使用相同的 name 属性重建顶点 ID；该作业不会自动复制全图 schema 或边。 +env { + job.mode = "BATCH" + parallelism = 1 +} + +source { + HugeGraph { + host = "hugegraph-server" + port = 8080 + graph_space = "DEFAULT" + graph_name = "graph_a" + label = "seatunnel_demo_person" + label_type = "VERTEX" + username = ${HG_USERNAME} + password = ${HG_PASSWORD} + schema { + fields { + name = "string" + age = "int" + } + } + } +} + +sink { + HugeGraph { + host = "hugegraph-server" + port = 8080 + graph_space = "DEFAULT" + graph_name = "graph_b" + username = ${HG_USERNAME} + password = ${HG_PASSWORD} + schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" + mappings = [ + { + type = "VERTEX" + label = "seatunnel_demo_person" + idStrategy = "PRIMARY_KEY" + idFields = ["name"] + properties = ["name", "age"] + } + ] + } +} diff --git a/helm/hugegraph/examples/seatunnel/hugegraph-sink.conf b/helm/hugegraph/examples/seatunnel/hugegraph-sink.conf new file mode 100644 index 0000000000..c847ced697 --- /dev/null +++ b/helm/hugegraph/examples/seatunnel/hugegraph-sink.conf @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 先在 DEFAULT 图空间创建 graph_a。此示例创建演示标签并导入十条固定数据。 +env { + job.mode = "BATCH" + parallelism = 1 +} + +source { + FakeSource { + row.num = 10 + schema { + fields { + name = "string" + age = "int" + } + } + rows = [ + { kind = INSERT, fields = ["person_01", 21] }, + { kind = INSERT, fields = ["person_02", 22] }, + { kind = INSERT, fields = ["person_03", 23] }, + { kind = INSERT, fields = ["person_04", 24] }, + { kind = INSERT, fields = ["person_05", 25] }, + { kind = INSERT, fields = ["person_06", 26] }, + { kind = INSERT, fields = ["person_07", 27] }, + { kind = INSERT, fields = ["person_08", 28] }, + { kind = INSERT, fields = ["person_09", 29] }, + { kind = INSERT, fields = ["person_10", 30] } + ] + } +} + +sink { + HugeGraph { + host = "hugegraph-server" + port = 8080 + graph_space = "DEFAULT" + graph_name = "graph_a" + username = ${HG_USERNAME} + password = ${HG_PASSWORD} + schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" + mappings = [ + { + type = "VERTEX" + label = "seatunnel_demo_person" + idStrategy = "PRIMARY_KEY" + idFields = ["name"] + properties = ["name", "age"] + } + ] + } +} diff --git a/helm/hugegraph/examples/seatunnel/hugegraph-source.conf b/helm/hugegraph/examples/seatunnel/hugegraph-source.conf new file mode 100644 index 0000000000..5f60beae34 --- /dev/null +++ b/helm/hugegraph/examples/seatunnel/hugegraph-source.conf @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 读取 hugegraph-sink.conf 导入的标签；Console 输出到 worker 日志，单机模式为 master 日志。 +env { + job.mode = "BATCH" + parallelism = 1 +} + +source { + HugeGraph { + host = "hugegraph-server" + port = 8080 + graph_space = "DEFAULT" + graph_name = "graph_a" + label = "seatunnel_demo_person" + label_type = "VERTEX" + username = ${HG_USERNAME} + password = ${HG_PASSWORD} + schema { + fields { + name = "string" + age = "int" + } + } + } +} + +sink { + Console {} +} diff --git a/helm/hugegraph/examples/seatunnel/plugin_config b/helm/hugegraph/examples/seatunnel/plugin_config new file mode 100644 index 0000000000..bb4ccc593f --- /dev/null +++ b/helm/hugegraph/examples/seatunnel/plugin_config @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--seatunnel-connectors-- +connector-hugegraph +connector-fake +connector-console +--end-- diff --git a/helm/hugegraph/examples/values-seatunnel-cluster.yaml b/helm/hugegraph/examples/values-seatunnel-cluster.yaml new file mode 100644 index 0000000000..100cccf76f --- /dev/null +++ b/helm/hugegraph/examples/values-seatunnel-cluster.yaml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 使用官方的 master/worker 分离 Deployment，示例仅指定引擎规模。 +# 生产环境需另行配置持久化与故障恢复，见 ../docs/seatunnel.md。 +# 提交作业前确认镜像含 HugeGraph 连接器。 +seatunnel: + enabled: true + mode: cluster + master: + replicas: 2 + resources: + requests: + cpu: 500m + memory: 1536Mi + limits: + memory: 2Gi + worker: + replicas: 2 + resources: + requests: + cpu: 500m + memory: 1536Mi + limits: + memory: 2Gi + configMap: + data: + jvm_master_options: | + -Xms512m + -Xmx1g + jvm_worker_options: | + -Xms512m + -Xmx1g diff --git a/helm/hugegraph/examples/values-seatunnel-standalone.yaml b/helm/hugegraph/examples/values-seatunnel-standalone.yaml new file mode 100644 index 0000000000..c286e156e0 --- /dev/null +++ b/helm/hugegraph/examples/values-seatunnel-standalone.yaml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 启动一个持续接收作业的 SeaTunnel 混合节点，HugeGraph 拓扑独立选择。 +# 小型环境可同时使用 ../values-single.yaml。 +# 此预设只启动引擎；提交作业前确认镜像含 HugeGraph 连接器，见 ../docs/seatunnel.md。 +seatunnel: + enabled: true + mode: standalone + master: + resources: + requests: + cpu: 500m + memory: 1536Mi + limits: + memory: 2Gi + configMap: + data: + jvm_options: | + -Xms512m + -Xmx1g diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt index 10892ba8ef..edc98d8464 100644 --- a/helm/hugegraph/templates/NOTES.txt +++ b/helm/hugegraph/templates/NOTES.txt @@ -24,6 +24,9 @@ limitations under the License. {{- if $hubbleEnabled }} Hubble 1 replica (UI) {{- end }} +{{- if (.Values.seatunnel | default dict).enabled }} + SeaTunnel {{ .Values.seatunnel.mode }} mode +{{- end }} {{- $inheritedEntries := list }} {{- range $component := list "server" "hubble" }} {{- $cfg := get $.Values $component }} @@ -56,6 +59,13 @@ Store waits for a PD quorum before startup. If a Pod remains in Init, inspect its init-container logs and PD readiness; see helm/hugegraph/docs/operations.md#troubleshooting. Access: +{{- if (.Values.seatunnel | default dict).enabled }} + + SeaTunnel: + In-cluster API: http://{{ include "seatunnel.fullname" .Subcharts.seatunnel }}-master.{{ .Release.Namespace }}.svc:{{ .Values.seatunnel.service.port }} + Port-forward: kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "seatunnel.fullname" .Subcharts.seatunnel }}-master 8081:{{ .Values.seatunnel.service.port }} + Connector images, credentials and checkpoint storage: helm/hugegraph/docs/seatunnel.md +{{- end }} {{- range $component := list "server" "hubble" "pd" }} {{- if or (ne $component "hubble") $hubbleEnabled }} {{- $cfg := get $.Values $component }} diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index e31813e60e..127a442abc 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -1028,6 +1028,19 @@ resolver (CoreDNS in any namespace, NodeLocal DNSCache). {{- join "\n" $rules -}} {{- end }} +{{/* SeaTunnel is a separate subchart with its own stable application labels. */}} +{{- define "hugegraph.netpol.seatunnelLabels" -}} +app.kubernetes.io/name: seatunnel +app.kubernetes.io/instance: {{ .Release.Name | quote }} +app.kubernetes.io/part-of: seatunnel +{{- end }} + +{{- define "hugegraph.netpol.seatunnelPeer" -}} +- podSelector: + matchLabels: + {{- include "hugegraph.netpol.seatunnelLabels" . | nindent 6 }} +{{- end }} + {{- define "hugegraph.netpol.dns" -}} - ports: - protocol: UDP diff --git a/helm/hugegraph/templates/networkpolicy.yaml b/helm/hugegraph/templates/networkpolicy.yaml index e8638d6d74..cbe2f4ccac 100644 --- a/helm/hugegraph/templates/networkpolicy.yaml +++ b/helm/hugegraph/templates/networkpolicy.yaml @@ -135,6 +135,9 @@ spec: {{- if $hubbleEnabled }} {{- include "hugegraph.netpol.peer" (dict "root" . "component" "hubble") | nindent 8 }} {{- end }} + {{- if (.Values.seatunnel | default dict).enabled }} + {{- include "hugegraph.netpol.seatunnelPeer" . | nindent 8 }} + {{- end }} ports: {{- include "hugegraph.netpol.ports" (list $serverPort) | nindent 8 }} {{- with get (get $np "server" | default dict) "extraIngress" }} diff --git a/helm/hugegraph/templates/seatunnel-networkpolicy.yaml b/helm/hugegraph/templates/seatunnel-networkpolicy.yaml new file mode 100644 index 0000000000..89fef5b01b --- /dev/null +++ b/helm/hugegraph/templates/seatunnel-networkpolicy.yaml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{ include "hugegraph.resolveValues" . }} +{{- if and (.Values.seatunnel | default dict).enabled .Values.networkPolicy.enabled }} +{{- $rules := get .Values.networkPolicy "seatunnel" | default dict }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "hugegraph.fullname" . | trunc 53 | trimSuffix "-" }}-seatunnel + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: seatunnel +spec: + podSelector: + matchLabels: + {{- include "hugegraph.netpol.seatunnelLabels" . | nindent 6 }} + policyTypes: + - Ingress + - Egress + ingress: + - from: + {{- include "hugegraph.netpol.seatunnelPeer" . | nindent 8 }} + ports: + {{- include "hugegraph.netpol.ports" (list 5801 (.Values.seatunnel.service.port | default 8080)) | nindent 8 }} + {{- with get $rules "extraIngress" }} + {{- toYaml . | nindent 4 }} + {{- end }} + egress: + - to: + {{- include "hugegraph.netpol.seatunnelPeer" . | nindent 8 }} + ports: + {{- include "hugegraph.netpol.ports" (list 5801 (.Values.seatunnel.service.port | default 8080)) | nindent 8 }} + - to: + {{- include "hugegraph.netpol.peer" (dict "root" . "component" "server") | nindent 8 }} + ports: + {{- include "hugegraph.netpol.ports" (list .Values.server.port) | nindent 8 }} + {{- include "hugegraph.netpol.dns" . | nindent 4 }} + {{- with get $rules "extraEgress" }} + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/helm/hugegraph/tests/__snapshot__/networkpolicy_test.yaml.snap b/helm/hugegraph/tests/__snapshot__/networkpolicy_test.yaml.snap index a79353de84..a836601261 100644 --- a/helm/hugegraph/tests/__snapshot__/networkpolicy_test.yaml.snap +++ b/helm/hugegraph/tests/__snapshot__/networkpolicy_test.yaml.snap @@ -26,7 +26,7 @@ matches the reviewed cluster plus Hubble render: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: hugegraph app.kubernetes.io/version: latest - helm.sh/chart: hugegraph-helm-0.2.0 + helm.sh/chart: hugegraph-helm-0.3.0 name: RELEASE-NAME-hugegraph-pd spec: egress: @@ -97,7 +97,7 @@ matches the reviewed cluster plus Hubble render: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: hugegraph app.kubernetes.io/version: latest - helm.sh/chart: hugegraph-helm-0.2.0 + helm.sh/chart: hugegraph-helm-0.3.0 name: RELEASE-NAME-hugegraph-store spec: egress: @@ -174,7 +174,7 @@ matches the reviewed cluster plus Hubble render: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: hugegraph app.kubernetes.io/version: latest - helm.sh/chart: hugegraph-helm-0.2.0 + helm.sh/chart: hugegraph-helm-0.3.0 name: RELEASE-NAME-hugegraph-server spec: egress: @@ -238,7 +238,7 @@ matches the reviewed cluster plus Hubble render: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: hugegraph app.kubernetes.io/version: latest - helm.sh/chart: hugegraph-helm-0.2.0 + helm.sh/chart: hugegraph-helm-0.3.0 name: RELEASE-NAME-hugegraph-hubble spec: egress: diff --git a/helm/hugegraph/tests/networkpolicy_test.yaml b/helm/hugegraph/tests/networkpolicy_test.yaml index edc7f6e835..91124ceb86 100644 --- a/helm/hugegraph/tests/networkpolicy_test.yaml +++ b/helm/hugegraph/tests/networkpolicy_test.yaml @@ -832,7 +832,7 @@ tests: documentIndex: 0 - equal: path: metadata.labels["helm.sh/chart"] - value: hugegraph-helm-0.2.0 + value: hugegraph-helm-0.3.0 documentIndex: 0 - equal: path: metadata.labels["app.kubernetes.io/version"] @@ -840,7 +840,7 @@ tests: documentIndex: 1 - equal: path: metadata.labels["helm.sh/chart"] - value: hugegraph-helm-0.2.0 + value: hugegraph-helm-0.3.0 documentIndex: 1 - equal: path: metadata.labels["app.kubernetes.io/version"] @@ -848,7 +848,7 @@ tests: documentIndex: 2 - equal: path: metadata.labels["helm.sh/chart"] - value: hugegraph-helm-0.2.0 + value: hugegraph-helm-0.3.0 documentIndex: 2 - equal: path: metadata.labels["app.kubernetes.io/version"] @@ -856,7 +856,7 @@ tests: documentIndex: 3 - equal: path: metadata.labels["helm.sh/chart"] - value: hugegraph-helm-0.2.0 + value: hugegraph-helm-0.3.0 documentIndex: 3 - it: appends extra rules verbatim diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 987e650c16..97e961a9ec 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -58,6 +58,15 @@ "type": "boolean" } } + }, + "seatunnel": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "description": "Optional upstream-derived SeaTunnel subchart; see charts/seatunnel/values.yaml." } }, "definitions": { @@ -128,6 +137,18 @@ "$ref": "#/definitions/networkPolicyEgressRules" } } + }, + "seatunnel": { + "type": "object", + "additionalProperties": false, + "properties": { + "extraIngress": { + "$ref": "#/definitions/networkPolicyIngressRules" + }, + "extraEgress": { + "$ref": "#/definitions/networkPolicyEgressRules" + } + } } } }, diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index c81a9ff5c8..dcee3d8205 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -533,6 +533,13 @@ hubble: failureThreshold: 3 timeoutSeconds: 5 +# Optional SeaTunnel engine. Its topology is independent of HugeGraph's presets +# and easy/strict mode. See docs/seatunnel.md for the SeaTunnel chart's native +# configuration, connector images and credentials. +seatunnel: + enabled: false + mode: standalone + # Kubernetes NetworkPolicy: mode default plus explicit per-component rules. # easy does not isolate clients; strict isolates release traffic and DNS. Only # enforced when the network plugin implements NetworkPolicy (kind v0.25+, @@ -556,6 +563,12 @@ networkPolicy: # For Hubble's optional outside endpoints (es.urls, prometheus.url set # through extraEnv). PD, Store and Server reach only the release and DNS. extraEgress: [] + seatunnel: + # SeaTunnel peers, DNS and the HugeGraph Server API are allowed by default. + # Add submitters/monitoring here using native NetworkPolicy rules. + extraIngress: [] + # External source/sink systems and shared checkpoint storage need egress. + extraEgress: [] # No init Job; see README.md for the HStore initialization contract. # Install with: helm install ... --wait (no --wait-for-jobs) diff --git a/helm/test_helm_seatunnel.py b/helm/test_helm_seatunnel.py new file mode 100644 index 0000000000..dd73245081 --- /dev/null +++ b/helm/test_helm_seatunnel.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Check the optional engine's rendered topology, connectivity and release payload.""" + +from pathlib import Path +import shutil +import subprocess +import tarfile +import tempfile +import unittest + +import yaml + + +CHART = Path(__file__).resolve().parent / "hugegraph" + + +def matches(selector, labels): + return all(labels.get(key) == value for key, value in selector.items()) + + +def engine_resources(documents, kind=None): + return [doc for doc in documents if (kind is None or doc["kind"] == kind) + and doc["metadata"].get("labels", {}).get("app.kubernetes.io/part-of") == "seatunnel"] + + +def pod(workload): + return workload["spec"]["template"] + + +class SeaTunnelChartTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + if shutil.which("helm") is None: + raise RuntimeError("Helm is required for the SeaTunnel chart regression") + + def render(self, seatunnel=None, chart=CHART, namespace="default", **values): + if seatunnel is not None: + values["seatunnel"] = {"enabled": True, **seatunnel} + with tempfile.TemporaryDirectory(prefix="seatunnel-values-") as directory: + configuration = Path(directory) / "values.yaml" + configuration.write_text(yaml.safe_dump(values)) + result = subprocess.run(["helm", "template", "fixture", str(chart), + "--namespace", namespace, "-f", str(configuration)], + capture_output=True, text=True) + return result + + def documents(self, seatunnel=None, **values): + result = self.render(seatunnel, **values) + self.assertEqual(0, result.returncode, result.stderr) + return [doc for doc in yaml.safe_load_all(result.stdout) if doc] + + def test_disabled_has_no_engine_resources_or_child_validation(self): + documents = self.documents() + self.assertEqual([], engine_resources(documents)) + self.assertFalse(any("seatunnel" in doc["metadata"]["name"] for doc in documents)) + # Disabled dependencies must not validate inactive engine settings. + self.assertEqual(0, self.render({"enabled": False, "mode": "unused"}).returncode) + + def test_standalone_is_one_hybrid_with_usable_configuration(self): + documents = self.documents({}) + workloads = engine_resources(documents, "Deployment") + self.assertEqual(1, len(workloads)) + master = workloads[0] + self.assertEqual(1, master["spec"]["replicas"]) + self.assertEqual("Recreate", master["spec"]["strategy"]["type"]) + container = pod(master)["spec"]["containers"][0] + # SeaTunnel 3.0.0 selects the hybrid role when -r is omitted; its Java + # entrypoint rejects an explicit -r master_and_worker. + self.assertTrue(container["command"][-1].rstrip().endswith("/seatunnel-cluster.sh")) + self.assertNotIn(" -r ", container["command"][-1]) + self.assertFalse(pod(master)["spec"]["automountServiceAccountToken"]) + mounts = {mount["subPath"] for mount in container["volumeMounts"]} + self.assertTrue({"hazelcast.yaml", "jvm_options", "seatunnel.yaml"} <= mounts) + config = engine_resources(documents, "ConfigMap")[0]["data"] + engine = yaml.safe_load(config["seatunnel.yaml"])["seatunnel"]["engine"] + self.assertEqual(0, engine["backup-count"]) + self.assertTrue(engine["http"]["enable-http"]) + self.assertFalse(engine["http"]["enable-dynamic-port"]) + + def test_cluster_roles_and_mode_switch_preserve_master_identity(self): + standalone = self.documents({}) + cluster = self.documents({"mode": "cluster"}) + workloads = engine_resources(cluster, "Deployment") + self.assertEqual({"master", "worker"}, + {doc["metadata"]["labels"]["app.kubernetes.io/component"] for doc in workloads}) + for workload in workloads: + role = workload["metadata"]["labels"]["app.kubernetes.io/component"] + self.assertEqual(2, workload["spec"]["replicas"]) + self.assertTrue(pod(workload)["spec"]["containers"][0]["command"][-1].rstrip().endswith("-r " + role)) + first = engine_resources(standalone, "Deployment")[0] + master = next(doc for doc in workloads if doc["metadata"]["name"] == first["metadata"]["name"]) + self.assertEqual(first["spec"]["selector"], master["spec"]["selector"]) + services = lambda docs: {doc["metadata"]["name"]: doc["spec"]["selector"] + for doc in engine_resources(docs, "Service")} + self.assertEqual(services(standalone), services(cluster)) + self.assertNotEqual(pod(first)["metadata"]["annotations"]["checksum/config"], + pod(master)["metadata"]["annotations"]["checksum/config"]) + data = engine_resources(cluster, "ConfigMap")[0]["data"] + self.assertEqual(1, yaml.safe_load(data["seatunnel.yaml"])["seatunnel"]["engine"]["backup-count"]) + + def test_peer_service_selects_both_roles_and_only_its_release(self): + documents = self.documents({"mode": "cluster", "fullnameOverride": "transfer", + "clusterDomain": "corp.internal"}, namespace="graphs") + service = next(doc for doc in engine_resources(documents, "Service") + if doc["spec"].get("clusterIP") == "None") + self.assertEqual("transfer", service["metadata"]["name"]) + self.assertTrue(service["spec"]["publishNotReadyAddresses"]) + selector = service["spec"]["selector"] + workloads = [doc for doc in documents if doc["kind"] in ("Deployment", "StatefulSet")] + selected = [doc for doc in workloads if matches(selector, pod(doc)["metadata"]["labels"])] + self.assertEqual(2, len(selected)) + self.assertFalse(matches(selector, {**pod(selected[0])["metadata"]["labels"], + "app.kubernetes.io/instance": "another-release"})) + config = engine_resources(documents, "ConfigMap")[0]["data"] + for filename in ("hazelcast.yaml", "hazelcast-master.yaml", "hazelcast-worker.yaml"): + discovery = yaml.safe_load(config[filename])["hazelcast"]["network"]["join"]["kubernetes"] + self.assertEqual("transfer.graphs.svc.corp.internal", discovery["service-dns"]) + self.assertIn("transfer.graphs.svc.corp.internal:5801", config["hazelcast-client.yaml"]) + + def test_runtime_overrides_and_config_changes_reach_pods(self): + digest = "sha256:" + "a" * 64 + values = {"image": {"registry": "team/seatunnel", "digest": digest, "pullSecret": "registry-auth"}, + "env": [{"name": "GRAPH_PASSWORD", "valueFrom": { + "secretKeyRef": {"name": "graph-auth", "key": "password"}}}], + "service": {"port": 9080}, "engineConfig": {"checkpoint": {"interval": 10000}}} + documents = self.documents(values, **{"global": {"imageRegistry": "registry.example.com"}}) + master = engine_resources(documents, "Deployment")[0] + spec = pod(master)["spec"] + container = spec["containers"][0] + self.assertEqual("registry.example.com/team/seatunnel@" + digest, container["image"]) + self.assertEqual([{"name": "registry-auth"}], spec["imagePullSecrets"]) + self.assertEqual(values["env"], container["env"]) + data = engine_resources(documents, "ConfigMap")[0]["data"] + engine = yaml.safe_load(data["seatunnel.yaml"])["seatunnel"]["engine"] + self.assertEqual(9080, engine["http"]["port"]) + self.assertEqual(10000, engine["checkpoint"]["interval"]) + previous = engine_resources(self.documents({}), "Deployment")[0] + self.assertNotEqual(pod(previous)["metadata"]["annotations"]["checksum/config"], + pod(master)["metadata"]["annotations"]["checksum/config"]) + documents = self.documents({"image": {"registry": "private.example.com/engine", "tag": "tested"}}, + **{"global": {"imageRegistry": "registry.example.com"}}) + image = pod(engine_resources(documents, "Deployment")[0])["spec"]["containers"][0]["image"] + self.assertEqual("private.example.com/engine:tested", image) + + def test_invalid_enabled_values_fail_before_install(self): + for values in ({"mode": "invalid"}, {"master": {"replicas": 2}}, + {"mode": "cluster", "master": {"replicas": 0}}, + {"mode": "cluster", "worker": {"replicas": 0}}, + {"service": {"port": 5801}}, + {"configMap": {"create": False, "existingConfigMapName": ""}}): + with self.subTest(values=values): + self.assertNotEqual(0, self.render(values).returncode) + documents = self.documents({"configMap": {"create": False, "existingConfigMapName": "engine-config"}}) + self.assertEqual([], engine_resources(documents, "ConfigMap")) + spec = pod(engine_resources(documents, "Deployment")[0])["spec"] + self.assertEqual("engine-config", spec["volumes"][0]["configMap"]["name"]) + documents = self.documents({"mode": "cluster", "master": {"strategy": {"type": "Recreate"}}, + "worker": {"strategy": {"type": "Recreate"}}}) + for workload in engine_resources(documents, "Deployment"): + self.assertEqual({"type": "Recreate"}, workload["spec"]["strategy"]) + + def test_strict_policies_allow_peers_server_dns_and_native_extensions(self): + extra_in = {"from": [{"ipBlock": {"cidr": "10.10.0.0/16"}}], "ports": [{"port": 9080}]} + extra_out = {"to": [{"ipBlock": {"cidr": "10.20.0.0/16"}}], "ports": [{"port": 9099}]} + documents = self.documents({"mode": "cluster", "service": {"port": 9080}}, + mode="strict", server={"port": 8182}, + networkPolicy={"seatunnel": {"extraIngress": [extra_in], + "extraEgress": [extra_out]}}) + policies = [doc for doc in documents if doc["kind"] == "NetworkPolicy"] + engine = next(doc["spec"] for doc in policies if doc["metadata"]["name"].endswith("-seatunnel")) + server = next(doc["spec"] for doc in policies if doc["metadata"]["name"].endswith("-server")) + labels = pod(engine_resources(documents, "Deployment")[0])["metadata"]["labels"] + self.assertTrue(matches(engine["podSelector"]["matchLabels"], labels)) + self.assertIn(extra_in, engine["ingress"]) + self.assertIn(extra_out, engine["egress"]) + def allows(rules, direction, selector, port): + return any(peer.get("podSelector", {}).get("matchLabels") == selector + and any(entry["port"] == port for entry in rule.get("ports", [])) + for rule in rules for peer in rule.get(direction, [])) + engine_selector = engine["podSelector"]["matchLabels"] + self.assertEqual("fixture", engine_selector["app.kubernetes.io/instance"]) + self.assertTrue(allows(server["ingress"], "from", engine_selector, 8182)) + self.assertTrue(allows(engine["egress"], "to", server["podSelector"]["matchLabels"], 8182)) + for direction, peer_key in (("ingress", "from"), ("egress", "to")): + self.assertTrue(allows(engine[direction], peer_key, engine_selector, 5801)) + self.assertTrue(allows(engine[direction], peer_key, engine_selector, 9080)) + dns = [port for rule in engine["egress"] if not rule.get("to") for port in rule["ports"]] + self.assertTrue({("UDP", 53), ("TCP", 53)} <= {(p["protocol"], p["port"]) for p in dns}) + + def test_package_preserves_dependency_configuration_and_all_modes(self): + with tempfile.TemporaryDirectory(prefix="seatunnel-package-") as directory: + subprocess.run(["helm", "package", str(CHART), "--destination", directory], + check=True, capture_output=True, text=True) + package = next(Path(directory).glob("*.tgz")) + with tarfile.open(package, "r:gz") as archive: + parent = yaml.safe_load(archive.extractfile("hugegraph-helm/Chart.yaml")) + dependency = next(d for d in parent["dependencies"] if d.get("alias") == "seatunnel") + self.assertEqual("seatunnel.enabled", dependency["condition"]) + filename = f'{dependency["name"]}-{dependency["version"]}.tgz' + child = CHART / "charts" / filename + # HugeGraph ships one dependency archive; canonical templates + # belong to the SeaTunnel repository rather than a source copy. + self.assertEqual([child], list((CHART / "charts").iterdir())) + with tarfile.open(child, "r:gz") as source: + for member in source.getmembers(): + relative = member.name.split("/", 1)[-1] + if relative == "values.schema.json" or relative.startswith(("templates/", "conf/")): + name = "hugegraph-helm/charts/" + member.name + self.assertEqual(source.extractfile(member).read(), archive.extractfile(name).read(), name) + self.assertEqual([], engine_resources(self.documents(chart=package))) + for mode, count in (("standalone", 1), ("cluster", 2)): + documents = self.documents({"mode": mode}, chart=package) + self.assertEqual(count, len(engine_resources(documents, "Deployment"))) + + +if __name__ == "__main__": + unittest.main() diff --git a/pom.xml b/pom.xml index 927995611b..89b2cf6c6a 100644 --- a/pom.xml +++ b/pom.xml @@ -282,6 +282,7 @@ **/pd/grpc/**/*.java **/store/grpc/**/*.java **/target/** + helm/**/Chart.lock **/apache-hugegraph-*/lib/arthas/logback.xml dist/**/* @@ -439,6 +440,8 @@ **/*.txt + **/*.tgz + **/__pycache__/** **/.flattened-pom.xml **/apache-hugegraph-*/**/* From f60ad3e3bfc71d4404956fd2dbfb62c7bc002f13 Mon Sep 17 00:00:00 2001 From: orion Date: Fri, 23 Oct 2026 14:14:49 +0000 Subject: [PATCH 02/17] docs(helm): record SeaTunnel runtime checks --- helm/hugegraph/docs/seatunnel.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/helm/hugegraph/docs/seatunnel.md b/helm/hugegraph/docs/seatunnel.md index 90e45b5cc9..66aa687595 100644 --- a/helm/hugegraph/docs/seatunnel.md +++ b/helm/hugegraph/docs/seatunnel.md @@ -189,17 +189,24 @@ helm lint helm/hugegraph --set seatunnel.enabled=true 版本变化时更新父 Chart 的依赖版本，移除旧归档，提交新归档及 `Chart.lock`。原生 `file://../../../seatunnel/deploy/kubernetes/seatunnel` 依赖负责打包；最终安装使用随 HugeGraph 发布的归档，不需要相邻源码仓库或联网下载 Chart。 -2026-10-23 本地验证记录（台北时间）： +2026-10-23 本地验证记录（台北时间）：测试使用独立 kind 集群，直接安装 HugeGraph 仓库携带的 `seatunnel-helm-3.0.1-hugegraph.1.tgz`。验证镜像由经过官方 SHA-512 校验的 SeaTunnel 3.0.0 二进制、经过 Maven 校验的连接器和 Temurin 17 JRE 构建；HugeGraph 1.8.0/RocksDB 在集群外的隔离本机实例运行。 | 检查 | 结果与范围 | |---|---| | Python Helm 回归 | 23 项通过 | | Helm 单元测试 | 309 项通过 | | HugeGraph 推送前检查 | `mvn editorconfig:format` 通过；`mvn clean compile -Dmaven.javadoc.skip=true` 全部 38 模块通过 | -| Kubernetes 真实 API | 安装及模式切换已执行；业务 Pod Ready 为 0，尚未证明集群内引擎可用 | +| Kubernetes 单机启动 | 22:08，Pod `1/1 Running`、重启 0；REST 正常，实际注册一个 worker | +| Kubernetes 单机导入 | CLI 作业 `1166015307609800705`：读取 10、写入 10、失败 0；graph_a 为 10 条 | +| Kubernetes 单机 A → B | CLI 作业 `1166015324982607873`：读取 10、写入 10、失败 0；两图各 10 条，全部 name/age 属性一致 | +| Kubernetes 切到集群 | 22:10，1 master + 1 worker 均 Ready、重启 0；复制任务 `1166015886427947011` 读取 10、写入 10、失败 0；再次核对两图内容一致 | +| Kubernetes 切回单机 | 22:12，恢复一个 Ready 混合节点，REST 正常；两图数据仍各 10 条且内容一致 | +| 启动进程与凭据传递 | 两种模式的容器 PID 1 均为 Java；Secret keyRef → Pod 环境变量 → 未改写的 HOCON 凭据占位符 → CLI 提交通过 | | 本机引擎启动 | 官方 SeaTunnel 3.0.0 二进制、Java 17；`/overview` 返回版本 3.0.0、一个 worker | | 直接导入 graph_a | 固定十条 FakeSource 数据；作业 `1166014497995882497` 为 `FINISHED`，读取与提交均为 10 | | graph_a → graph_b | 作业 `1166014497996079105` 为 `FINISHED`；使用文档中的 CLI 与环境变量方式再次复制，作业 `1166014645144649729` 读取 10、写入 10、失败 0 | | 图数据核对 | 隔离的 HugeGraph 1.8.0/RocksDB；Gremlin 返回 `count_a=10`、`count_b=10`、`same=true`，逐条比对 `person_01` 至 `person_10` 的 name/age 完全一致 | -以上业务记录来自本机二进制引擎，不等同于 Kubernetes 内完整业务验证；隔离 Kubernetes 中的引擎启动和同样两条业务路径正在补测。此次数据样例只有一个顶点标签，没有验证多标签、边、HStore 或生产故障恢复。 +集群测试显式设置 `master.replicas=1`、`worker.replicas=1`、`engineConfig.backup-count=0`，用于验证分离角色及数据路径，不是 HA 测试。切换时观察到约 30 秒重新选举窗口，因此不能承诺无中断切换。全部检查完成后已清理本次临时集群及测试进程。 + +本次覆盖的是 Kubernetes 中的 SeaTunnel 访问外部 HugeGraph/RocksDB，没有验证完整 HugeGraph HStore Helm 集群、多标签、边、生产故障恢复或 CNI 对 NetworkPolicy 的实际执行。隔离 HugeGraph 关闭认证，因此仅确认了凭据注入与变量解析，没有验证认证授权。公开默认镜像的连接器清单仍需按实际镜像检查。 From 0e96f1017c3fb5159fd5e8eabb30a1eea3bd927d Mon Sep 17 00:00:00 2001 From: orion Date: Fri, 23 Oct 2026 14:33:12 +0000 Subject: [PATCH 03/17] fix(helm): exclude generated dependency lock from license headers --- .licenserc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.licenserc.yaml b/.licenserc.yaml index 8da741f65a..e2fd892442 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -59,6 +59,7 @@ header: # `header` section is configurations for source codes license header. - 'LICENSE' - 'NOTICE' - 'DISCLAIMER' + - 'helm/hugegraph/Chart.lock' # Generated by Helm dependency update. - '.serena/**' - '**/*.versionsBackup' - '**/*.versionsBackup' From 6a8a13094e855bb7a587080b4a8b8e15d1b82b73 Mon Sep 17 00:00:00 2001 From: orion Date: Fri, 23 Oct 2026 14:44:50 +0000 Subject: [PATCH 04/17] docs(helm): record authenticated HStore integration tests --- helm/hugegraph/docs/seatunnel.md | 58 ++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/helm/hugegraph/docs/seatunnel.md b/helm/hugegraph/docs/seatunnel.md index 66aa687595..6b4a760c67 100644 --- a/helm/hugegraph/docs/seatunnel.md +++ b/helm/hugegraph/docs/seatunnel.md @@ -9,7 +9,9 @@ SeaTunnel 默认关闭。给 HugeGraph Helm 命令增加 `--set seatunnel.enable | `seatunnel.mode: standalone` | 一个默认混合角色节点，同时承担 master 和 worker | | `seatunnel.mode: cluster` | 独立 master、worker Deployment，默认各两个副本 | -两套拓扑独立选择。SeaTunnel 源码由 [HugeGraph SeaTunnel fork](https://github.com/hugegraph/seatunnel/tree/dev/deploy/kubernetes/seatunnel) 维护，复用官方启动脚本、ConfigMap 挂载和 Headless Service 发现方式；HugeGraph 只引入固定版本的打包依赖，别名为 `seatunnel`，不维护第二套模板，也不需要 Operator 或 CRD。 +`values-single.yaml` 启动的是 1 PD + 1 Store + 1 Server 的 HStore 拓扑，数据由 Store 管理；这里的 single 指各组件一个副本，两套拓扑独立选择。 + +SeaTunnel 源码由 [HugeGraph SeaTunnel fork](https://github.com/hugegraph/seatunnel/tree/dev/deploy/kubernetes/seatunnel) 维护，复用官方启动脚本、ConfigMap 挂载和 Headless Service 发现方式；HugeGraph 只引入固定版本的打包依赖，别名为 `seatunnel`，不维护第二套模板，也不需要 Operator 或 CRD。 单机使用官方脚本的默认混合角色，不传 `-r`。SeaTunnel 3.0.0 的 Java 入口只接受显式的 `master` 或 `worker`，传入 `-r master_and_worker` 会启动失败；本次启动实测已发现并修正这一点。 @@ -18,7 +20,7 @@ SeaTunnel 默认关闭。给 HugeGraph Helm 命令增加 `--set seatunnel.enable 以下命令从 HugeGraph 仓库根目录执行，选择一种安装方式： ```bash -# 小型 HugeGraph + 单机 SeaTunnel +# HugeGraph 1 PD + 1 Store + 1 Server + 单机 SeaTunnel helm install hugegraph ./helm/hugegraph -n hugegraph --create-namespace \ -f helm/hugegraph/values-single.yaml \ -f helm/hugegraph/examples/values-seatunnel-standalone.yaml \ @@ -90,6 +92,8 @@ seatunnel: 这些变量同时进入 master 和 worker，作业文件通过 `${HG_USERNAME}`、`${HG_PASSWORD}` 读取凭据。Chart 不创建 HugeGraph 账号，也不复制管理员密码；账号需有目标图的读写及 schema 权限。更新 Secret 后需重启相关 Pod，刷新环境变量。 +使用默认自动生成的管理员 Secret 进行本地验证时，也可将 `HG_USERNAME` 设置为 `admin`，让 `HG_PASSWORD` 的 `secretKeyRef` 直接引用 `-admin` 的 `password` 键，无需另建或复制 Secret。本次 1+1+1 验证使用此方式；实际作业按需使用专用账号。 + `global.imageRegistry` 同样作用于 SeaTunnel；显式指定完整仓库地址优先。SeaTunnel 保留上游的 `image.registry`、`image.pullSecret` 键，不继承 HugeGraph 顶层的 `imagePullSecrets`。 ## 场景一：直接导入 Graph A @@ -189,24 +193,42 @@ helm lint helm/hugegraph --set seatunnel.enabled=true 版本变化时更新父 Chart 的依赖版本，移除旧归档，提交新归档及 `Chart.lock`。原生 `file://../../../seatunnel/deploy/kubernetes/seatunnel` 依赖负责打包；最终安装使用随 HugeGraph 发布的归档，不需要相邻源码仓库或联网下载 Chart。 -2026-10-23 本地验证记录（台北时间）：测试使用独立 kind 集群，直接安装 HugeGraph 仓库携带的 `seatunnel-helm-3.0.1-hugegraph.1.tgz`。验证镜像由经过官方 SHA-512 校验的 SeaTunnel 3.0.0 二进制、经过 Maven 校验的连接器和 Temurin 17 JRE 构建；HugeGraph 1.8.0/RocksDB 在集群外的隔离本机实例运行。 +### HStore 1+1+1 联合部署实测 + +2026-10-23（台北时间），在独立 kind / Kubernetes 1.34.0 中直接安装 HugeGraph 父 Chart，使用 `values-single.yaml`：**1 PD + 1 Store + 1 Server**，同时启用 SeaTunnel standalone。没有使用集群外的 HugeGraph 服务。测试 overlay 只调整本地镜像、资源、Service 类型、JVM 参数和 SeaTunnel 凭据引用，关闭 Hubble；保留 PD/Store 持久化及 Server/PD 默认认证。 + +PD、Store、Server 使用当前 1.8.0 源码 `f60ad3e3b` 的发行包，沿用仓库 Dockerfile 的运行阶段与官方入口。SeaTunnel 使用经官方 SHA-512 校验的 3.0.0 二进制、经 Maven 校验的 HugeGraph/FakeSource/Console 连接器和 Temurin 17 JRE。受测试环境的 Docker VFS/native snapshotter 磁盘限制，四个镜像合并为单层后导入节点，保留原始文件系统、ENV、CMD 和 ENTRYPOINT；没有为此修改项目代码。 + +| 检查 | 实际结果 | +|---|---| +| 父 Helm 安装 | 22:38，`helm install --wait` 成功；PD、Store、Server、SeaTunnel 四个 Pod 均 `1/1 Running`、重启 0 | +| 持久化 | PD 5Gi、Store 10Gi，两个 PVC 均 `Bound` | +| Helm 启动后检查 | `helm test hugegraph --logs` 成功；Server 1.8.0、Gremlin 3.8.1，默认图绑定及索引状态检查通过 | +| 实际存储后端 | Server 配置 `backend=hstore`，PD 中一个 Store 为 `Up`；graph_a、graph_b 的 API 均返回 `backend=hstore`，PD 图目录也包含两图 | +| 默认认证 | 匿名访问图列表返回 401，管理员认证返回 200；SeaTunnel 通过 `secretKeyRef` 直接引用 Helm 已生成的管理员密码 | +| 两个空图 | 导入前 `A=0、B=0` | +| 直接导入 A | 22:40，作业 `1166023243023253505` 为 `FINISHED`，读取 10、写入 10、失败 0；之后 `A=10、B=0` | +| A → B 复制 | 作业 `1166023260857434113` 为 `FINISHED`，读取 10、写入 10、失败 0；`A=10、B=10`，全部 name/age 逐条一致 | +| 图隔离 | 仅将 B 的 `person_01.age` 改为 999，A 仍为 21，确认两个 HStore 图的数据独立 | +| 重复复制 | 作业 `1166023273259991041` 为 `FINISHED`，读取 10、写入 10、失败 0；B 恢复为 21，两图仍各十条且内容一致 | +| 文档命令复核 | 使用仓库原始 HOCON 和本文 CLI 提交；本文 A/B 比较 Gremlin 原文执行成功，返回 `count_a=10、count_b=10、same=true` 及完整十行数据 | +| SeaTunnel 切至集群 | 22:41，1 master + 1 worker 均 Ready、PID 1 为 Java；复制作业 `1166023709937369090` 为 `FINISHED`，读取 10、写入 10、失败 0；B 单独改为 999 后再次复制恢复，两图各十条且逐属性一致 | +| SeaTunnel 切回单机 | 22:43，恢复一个 Ready 混合节点、PID 1 为 Java，worker 已移除；两图仍为 HStore，各十条且 `same=true`，认证仍返回 401/200 | +| 切换期间 HugeGraph 状态 | 原 PD、Store、Server Pod 均未重建、重启 0；两个 PVC 保持 Bound，三个认证 Secret 的值未变化 | + +Gremlin 将上述 Map 序列化为 `result.data=[{"count_a":10},{"count_b":10},{"same":true},{"rows":[...]}]`；自动校验时需合并这些对象，不能只读取第一项。 + +### 其他已完成检查 | 检查 | 结果与范围 | |---|---| | Python Helm 回归 | 23 项通过 | -| Helm 单元测试 | 309 项通过 | +| Helm 单元测试 | 309 项通过，36 suites、4 snapshots | | HugeGraph 推送前检查 | `mvn editorconfig:format` 通过；`mvn clean compile -Dmaven.javadoc.skip=true` 全部 38 模块通过 | -| Kubernetes 单机启动 | 22:08，Pod `1/1 Running`、重启 0；REST 正常，实际注册一个 worker | -| Kubernetes 单机导入 | CLI 作业 `1166015307609800705`：读取 10、写入 10、失败 0；graph_a 为 10 条 | -| Kubernetes 单机 A → B | CLI 作业 `1166015324982607873`：读取 10、写入 10、失败 0；两图各 10 条，全部 name/age 属性一致 | -| Kubernetes 切到集群 | 22:10，1 master + 1 worker 均 Ready、重启 0；复制任务 `1166015886427947011` 读取 10、写入 10、失败 0；再次核对两图内容一致 | -| Kubernetes 切回单机 | 22:12，恢复一个 Ready 混合节点，REST 正常；两图数据仍各 10 条且内容一致 | -| 启动进程与凭据传递 | 两种模式的容器 PID 1 均为 Java；Secret keyRef → Pod 环境变量 → 未改写的 HOCON 凭据占位符 → CLI 提交通过 | -| 本机引擎启动 | 官方 SeaTunnel 3.0.0 二进制、Java 17；`/overview` 返回版本 3.0.0、一个 worker | -| 直接导入 graph_a | 固定十条 FakeSource 数据；作业 `1166014497995882497` 为 `FINISHED`，读取与提交均为 10 | -| graph_a → graph_b | 作业 `1166014497996079105` 为 `FINISHED`；使用文档中的 CLI 与环境变量方式再次复制，作业 `1166014645144649729` 读取 10、写入 10、失败 0 | -| 图数据核对 | 隔离的 HugeGraph 1.8.0/RocksDB；Gremlin 返回 `count_a=10`、`count_b=10`、`same=true`，逐条比对 `person_01` 至 `person_10` 的 name/age 完全一致 | - -集群测试显式设置 `master.replicas=1`、`worker.replicas=1`、`engineConfig.backup-count=0`，用于验证分离角色及数据路径，不是 HA 测试。切换时观察到约 30 秒重新选举窗口，因此不能承诺无中断切换。全部检查完成后已清理本次临时集群及测试进程。 - -本次覆盖的是 Kubernetes 中的 SeaTunnel 访问外部 HugeGraph/RocksDB，没有验证完整 HugeGraph HStore Helm 集群、多标签、边、生产故障恢复或 CNI 对 NetworkPolicy 的实际执行。隔离 HugeGraph 关闭认证，因此仅确认了凭据注入与变量解析，没有验证认证授权。公开默认镜像的连接器清单仍需按实际镜像检查。 +| 远端许可检查 | 生成的 `Chart.lock` 按精确路径排除 license header 检查；提交 `0e96f1017` 的 Helm、license header 和 RAT 检查通过 | +| 前一轮 SeaTunnel 切换基线 | 22:08–22:12，使用同一打包依赖，SeaTunnel 单机 → 1 master + 1 worker → 单机均 Ready、重启 0；单机完成导入与复制，集群完成复制，每次读取 10、写入 10、失败 0，两图各十条且内容一致。此轮 HugeGraph 为集群外的 1.8.0/RocksDB，仅作补充记录 | +| 进程与凭据传递 | 两种 SeaTunnel 模式的 PID 1 均为 Java；Secret keyRef → Pod 环境变量 → 原始 HOCON 占位符 → CLI 提交通过 | + +两轮集群测试均使用 `master.replicas=1`、`worker.replicas=1`。本轮 HStore 沿用集群默认 `backup-count=1`，前一轮外部 RocksDB 测试显式设置 `backup-count=0`；这里只验证分离角色及数据路径。本轮两次 Helm 模式升级各约 30 秒完成；前一轮还观察到约 30 秒重新选举窗口，不能承诺无中断切换。 + +本次 HStore 验证覆盖开启默认认证的 1+1+1 Helm 部署及一个顶点标签的导入、复制和图隔离；未覆盖多标签、边、细粒度授权、生产 HA、持久化作业恢复或 CNI 对 NetworkPolicy 的实际执行。公开默认镜像的连接器清单仍需按实际镜像检查。 From 9fc87e201af87548bd7794438797f85995ecf9d3 Mon Sep 17 00:00:00 2001 From: orion Date: Fri, 23 Oct 2026 15:58:11 +0000 Subject: [PATCH 05/17] refactor(helm): reuse native SeaTunnel hybrid deployment --- .licenserc.yaml | 1 - LICENSE | 2 +- NOTICE | 2 +- helm/hugegraph/.helmignore | 4 +- helm/hugegraph/Chart.lock | 6 - helm/hugegraph/Chart.yaml | 8 +- helm/hugegraph/README.md | 4 +- .../seatunnel-helm-3.0.1-hugegraph.1.tgz | Bin 14468 -> 0 bytes helm/hugegraph/docs/seatunnel.md | 228 ++++++----------- helm/hugegraph/examples/seatunnel/Dockerfile | 24 -- .../examples/seatunnel/clone/company.conf | 54 ++++ .../examples/seatunnel/clone/copy-schema.py | 91 +++++++ .../examples/seatunnel/clone/knows.conf | 56 +++++ .../examples/seatunnel/clone/person.conf | 54 ++++ .../examples/seatunnel/clone/works_at.conf | 55 ++++ .../examples/seatunnel/hugegraph-copy.conf | 61 ----- .../examples/seatunnel/hugegraph-source.conf | 43 ---- .../examples/seatunnel/plugin_config | 20 -- .../examples/values-seatunnel-cluster.yaml | 30 +-- .../examples/values-seatunnel-standalone.yaml | 17 +- helm/hugegraph/templates/NOTES.txt | 5 +- helm/hugegraph/templates/_helpers.tpl | 22 +- .../templates/seatunnel-networkpolicy.yaml | 4 +- helm/hugegraph/templates/seatunnel.yaml | 186 ++++++++++++++ helm/hugegraph/values.schema.json | 15 +- helm/hugegraph/values.yaml | 23 +- helm/test_helm_seatunnel.py | 236 ++++++++---------- pom.xml | 3 - 28 files changed, 737 insertions(+), 517 deletions(-) delete mode 100644 helm/hugegraph/Chart.lock delete mode 100644 helm/hugegraph/charts/seatunnel-helm-3.0.1-hugegraph.1.tgz delete mode 100644 helm/hugegraph/examples/seatunnel/Dockerfile create mode 100644 helm/hugegraph/examples/seatunnel/clone/company.conf create mode 100644 helm/hugegraph/examples/seatunnel/clone/copy-schema.py create mode 100644 helm/hugegraph/examples/seatunnel/clone/knows.conf create mode 100644 helm/hugegraph/examples/seatunnel/clone/person.conf create mode 100644 helm/hugegraph/examples/seatunnel/clone/works_at.conf delete mode 100644 helm/hugegraph/examples/seatunnel/hugegraph-copy.conf delete mode 100644 helm/hugegraph/examples/seatunnel/hugegraph-source.conf delete mode 100644 helm/hugegraph/examples/seatunnel/plugin_config create mode 100644 helm/hugegraph/templates/seatunnel.yaml diff --git a/.licenserc.yaml b/.licenserc.yaml index e2fd892442..8da741f65a 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -59,7 +59,6 @@ header: # `header` section is configurations for source codes license header. - 'LICENSE' - 'NOTICE' - 'DISCLAIMER' - - 'helm/hugegraph/Chart.lock' # Generated by Helm dependency update. - '.serena/**' - '**/*.versionsBackup' - '**/*.versionsBackup' diff --git a/LICENSE b/LICENSE index 830eabaac8..105c58618a 100644 --- a/LICENSE +++ b/LICENSE @@ -218,4 +218,4 @@ hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/opencypher/Cyp hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/opencypher/CypherPlugin.java from https://github.com/opencypher/cypher-for-gremlin hugegraph-struct/src/main/java/org/apache/hugegraph/type/define/Cardinality.java from https://github.com/JanusGraph/janusgraph hugegraph-struct/src/main/java/org/apache/hugegraph/type/Namifiable.java from https://github.com/JanusGraph/janusgraph -helm/hugegraph/charts/seatunnel-helm-*.tgz contains the Kubernetes Helm chart from https://github.com/apache/seatunnel, maintained in https://github.com/hugegraph/seatunnel (Apache License 2.0). +helm/hugegraph/templates/seatunnel.yaml adapts the native hybrid Kubernetes deployment from Apache SeaTunnel 3.0.0 (Apache License 2.0). diff --git a/NOTICE b/NOTICE index 126decde3b..d770d1475e 100644 --- a/NOTICE +++ b/NOTICE @@ -6,7 +6,7 @@ The Apache Software Foundation (http://www.apache.org/). The initial codebase was donated to the ASF by HugeGraph Authors, copyright 2017-2021. -The optional SeaTunnel Helm subchart in helm/hugegraph/charts/ contains code from: +The optional native Kubernetes deployment in helm/hugegraph/templates/seatunnel.yaml adapts resources from: Apache SeaTunnel Copyright 2021-2024 The Apache Software Foundation diff --git a/helm/hugegraph/.helmignore b/helm/hugegraph/.helmignore index 5551543a94..e37a0cdfb0 100644 --- a/helm/hugegraph/.helmignore +++ b/helm/hugegraph/.helmignore @@ -34,5 +34,5 @@ testdata/ # files in the release Secret; binary assets can exceed Kubernetes' 1 MiB limit. /images/ -# Ignore parent archives; dependencies under charts/ must be loaded by Helm. -/*.tgz +# Never package a chart archive inside a chart. +*.tgz diff --git a/helm/hugegraph/Chart.lock b/helm/hugegraph/Chart.lock deleted file mode 100644 index fed710c7e5..0000000000 --- a/helm/hugegraph/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: seatunnel-helm - repository: file://../../../seatunnel/deploy/kubernetes/seatunnel - version: 3.0.1-hugegraph.1 -digest: sha256:0940d9b2f7fb25cb2b43e6464f838072bc38bf371821735b1782ea15feaf257e -generated: "2026-10-23T13:15:14.300959334Z" diff --git a/helm/hugegraph/Chart.yaml b/helm/hugegraph/Chart.yaml index a300f6cdcd..e38988e2a7 100644 --- a/helm/hugegraph/Chart.yaml +++ b/helm/hugegraph/Chart.yaml @@ -32,13 +32,7 @@ home: https://hugegraph.apache.org/ icon: https://hugegraph.apache.org/img/logo.svg sources: - https://github.com/apache/hugegraph - - https://github.com/hugegraph/seatunnel/tree/dev/deploy/kubernetes/seatunnel -dependencies: - - name: seatunnel-helm - alias: seatunnel - version: 3.0.1-hugegraph.1 - repository: file://../../../seatunnel/deploy/kubernetes/seatunnel - condition: seatunnel.enabled + - https://github.com/apache/seatunnel maintainers: - name: HugeGraph Community url: https://hugegraph.apache.org/ diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index d7390d33ee..a274868245 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -84,9 +84,9 @@ Inactive credential values may remain in the values file when auth is disabled. ### 可选 SeaTunnel 增加 `--set seatunnel.enabled=true`，与 HugeGraph 一起启动一个 SeaTunnel 混合节点； -增加 `--set seatunnel.mode=cluster`，切换为分离的 master/worker 集群。 +增加 `--set seatunnel.mode=cluster`，切换为多个 hybrid 节点（默认两个），复用同一套官方原生 Kubernetes 资源。 默认关闭，SeaTunnel 拓扑独立于 HugeGraph 的单机/集群预设和 `easy/strict` 访问模式。 -启动、凭据、直接导入 Graph 及 Graph A → Graph B 复制示例见 +引擎同时提供内置任务可视化前端。启动、凭据、直接导入 Graph 及 Graph A → Graph B 复制示例见 [SeaTunnel 联合部署指南](docs/seatunnel.md)。 ### Internal image registry diff --git a/helm/hugegraph/charts/seatunnel-helm-3.0.1-hugegraph.1.tgz b/helm/hugegraph/charts/seatunnel-helm-3.0.1-hugegraph.1.tgz deleted file mode 100644 index 3a213a0d8bc3c4a1f8788d550e13bce7b8d1b727..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14468 zcmZvjQ+Op!w1#6(xZ_N0W8#Tz+cqXnCbn(c=EQa;wrz8gon)W<7w24^x~RLV)m{Cq z@9AFaZK5bR0N8&6L<2@=B&p17A}Plq>&eY-%%;w4s={uit-{Trpsvm#r(tVtWN+%J zs^q{YX=ZB&cG(SdCm2ov%g#HgD|fJ+mR`%qom()y_w2jBhnETb)u?5RMDK@$H~tynx5xW*UVH5+*@(GzDesKD z?4q&N;d)a8ti%?HA2kYpcS3v>GHZpVabIV??REB0Xwj~bSCF6Zt6H-D+&N!dd~_cv z9Vz=7R0`1w-eo)-+#KA`-xc{p3Eucw4&r^wyjJXvcwZPi4TJr{sRzlc1D@_6q6-gC zf>m5Ap;1`U_QNMNERK3-Vd>y%r5nJhZTC@Az#a{U5@7L?q5RuW(kuc1)(B6!T3$wYS_J!b#SoQXT9YKlcPN+9G9%1foGJymJRBm2pIld7F|Y}|zWYNj zl{;B3Xtt_90k6e+Xw-A4zlM}0p|@idzx7eyHy}d0B3y8Qk51J5iO2+qGgwqR*0pba z9FBl^9!-XjwZJb~KFdZasU8me_JFG;2sY*S0gvV zAN6tmVHuUdKkEZLs~nMTvCIvFX4z+n0fIAw+DE(fLYDcn`Q`8xVS|*8Qj4X3*7p5v zgTouKVzUQW>S;R~oF)th0Gex*RJ*^>FZeaLmQ<<1Yh3&Os4x&fk`+oS`O(Mv5;aR9 zL9D`%>=B!f1x9n|0e<<5>?RW5vmoguyU|(ogQJ6k2Q6|Bk@z>_`~SE!2D=a3jr9nY zu2Q4L$Hz~NtWKX+lV_TmW~!Y^qZkP>+0{KI8vzqj7)>WdS=`PcxL8rUl6@-3%cfwGS0wq24zjjrQ|LQ)9g{EdgX4Lp8tC^Cqo_ zt+}VC`7c)^U+8Q-m?jJC6D)P?X~tUrrn0`N{exXt#3*M9e`W^~gjsV@=+9sX`qS{A z{mNtlFyJJF`((nY&*+0xhhPwSe`c}GCb1>IizfdGX9;+&CSr@F zMzs!QThk_Bkw-%U6G1ChjA!z1L`CAG0fq;1TRAW8(m)M~u(_`gG*Ael2h`tVtnGXj15 z+w;Rt1NsDoAOK>)M)H;$H2%f)_2F7cI$-{T=HCD%I!g3lyL*(zx`kkZ0N^M3UBgCg zK*$K`o(zBm;Phby-43W|giZ_ow)qp1ETBmS?XVn%D@F+%tT`eB^-xR2T!{y0r^zQFXG>4fthb-4E!8pg*XaTvSfuP;4j3pGdl z^cSh&W4Oj^6OTYp3?EgE%!9-!yjT^J7cF1ZPi+3uL&DM$#jeSSQg}I;T1FTi^K*MD zYg@=>OZaX)B#O!*!am~k7@82vW612AU0MFRT66sBIVJR7@=jrivys$nhN3 z7MQ=0q55dO7Q zqx?Srv?5EiyQbel3Uw~nZ-6~S^4ql3d{M+omgi>s#mAWn8ZZuV$U z+9iI=xZ*-legEVV2IkYXW&`>8uPGmL>!-hn5EP1)e^R6Y zDZX$%^ws_o@To~$(-Z~u6kLawKtV3ai!nEJ-2H(;3*tc8KU2v=e^ZOltR>KilzaVk zYq!OcMZ*!oYBV9SOA=`#J*3|u!~i2gO8*=xnAGS43ZDxj6#{D#D-%&V&<)j_NC}X$ zh7&!B`u;9Z+uo};z?7CIuAm(fC`kqR@qQlo{f0=*t4QsAIUK_<`&e=RTQvu|@laZ3 z)if>a`p7qIOW`Q~bew?HPMp)L!=hiGc%hsmuCUw}5E4Po+RyvZNXMU1xbzqBO?D-1 zYTO+G07*az_|Eo+_R+A(qC}Pi)9D#9_W;qR|E}U}9ix6TohA=mw1bPU-SsEp+_|25 zuKD}7Rda~y@6(YxQI>jAsiBQ21)cY3FCzawf2hi9mu4vdevK(%CJUBH^?>WK}S2b8eJS-y=$MYAUyikOQLThWl0Uk<2Fi2i5 zx}aY_W*k4=xj?+xpq3LqCVhHH6n4jO(Osj`TiJgGP)9WkD&o`QF`|I5 zF7QZ{dMD)`xNNgSvaxuw%L{79#m&Mb1u}mkNcn?VXcd)v1%m3K7?^FSVW)Kf0^h4U z(PsLzXI`l5A)DGa@r0yW4JYP`!&JE^>Hv{F!j5tnM*XRM6knv=Sa^wiszW$seadh6 z|M=_Oq)_8$?-+`i^r5(1K34Hp`N}x4z>$nFJdeD1qd<`egs|=;T1^zzm16O1)^6*zuy(kf*Hm@#u85NW-B;wJNCWDRl7L1JU`Y;W>alq{;|1^V29?l=qZ^$q2<5 zfU|01f#vZRB$W zEepk9`0NgibW!)1*SxX47&B*$eab|f@|JEY3GG9&x5D0MQ>ab;x9U()r1@_}sX;7> z&;v#b`^IS=anG0ixkW3;L|#w4G<)TavFaFz#AvW?Opl^x90a*|`PZ`cg?Y zEelW&(RgV@I*n0=enWOaTAOJoJW^=2YUbKU^yA9*&H3&QAl-9lC4tw1sIO6mK7n=U zFq4$*hH>mq($*BBB9a@1NYRsbwvKS&mo}O0W6|fc=0 zL~JJY9>i3o{afb#F&Hf07Dv4d?VjYZcHJFvpk%~NQhF7;1U)jr?UpnxJ73BG!C}26 zJ_2~J8@k7;96N4^5k|lP+F=bk*JpCGVVS7ZiBaC7nso8K9%3}0dtFK~Om;=47e2{f z*eb^3Feu>w2C!dwO3h`@rhtP8KQqw%4YD^Ke63H;YKI!0Ff1UCBMC6?tlN@AEZ9ur z!?Pzzrm}UW4Z*Qb0{Dwy0s50bz?);KMIwbx$&-)3MG7+uZ^@wEE<}r+c=ZiDgcN~# z^0G*`t7QjIpEnrXZ4v=G1?9rHU#d?^m_evNH!zc7WnfRC|aX)8I)HpFN^dbAaVCK z4ZhDXgOX-INC+mTpMC2;IMp==3p8YM%7p#7k`afJ%;vf=@Y``F8WzC8%7?n)QY!i5SQo)-0*E@C%c!$Az8_WvHe-`4$@*8H4SkoydQ0v!BWdC_Q zco;T-@X;A!S1RET!^ow8+r;4-K1EyrYj(5rn%&dha?mR)R+KFoApk9-ky5UDK_ZBZ z2-6d8zPx0a9vLRzSiJSU(}usqK&IPVKh^@0L24yENibP+JlTN4W^Da~X7q5^E-TP1 zR1q?&)UNE34YSj|y0qlCt(z|r)sS6R-mZFqAx|&nT+31PchFcxO!1lTpePf=DGMvBX4dz& z*5tArHG?L#n0rmm4iS6Yzo{%hNw!IutM}ivP7|pOch}3dkUU_%xT@z|)r%xHN84#z zn^8n3%d>=#PMolMZ5%*>w?Q_45<-c2cEnvgh`0e)IBF*}eTmj}FeFZq(@AHj%M%FQ zFtVauQo*b-=-~shd9Spz!>~D0YC_P-{)=;zV}N5S{89d9{BQ?9Dd!bcd3?A|7;(gl z)w=^nY*Xre`r%u6^ajMlVk*PN**G&XNSS!-eo>=v-K4GJFDJu+jFZLrV$f1UuHgMf zB~ZA2n8W!^Sc9nXAmd48#4uhM=3_}pq5Vp1G8#*!mC<6dgf|nV#fnWtMrsQo@bYS2 zOmXVkfHBb6yy#bhLYPZ_5gE~zXszYjn`0-t|I}IZrB^E)!U_sXge%ZHlChlccsTAQ z8)s+zoExk9t{QvJe~K(8n_^6-rHe!Hhz^@+UFA~8({*saqmr*)cP7??r~KrXE9`HD zz31|pr+LXx%5el)Dg1UUX@8>cAHUJ1XMX2|c3UXyy!UedE(^&&A#>xjfVgG{39#^| z{sqNpP>qDXg2+6-CZkWkbkIyLC@niMjk~qFST4iAQ}ic^a{Oe|KF(+l%6YKk2u(C8 z0Y}cZ4U~)98M5+4ST9%K;eu|{w;mWQsqn*q->yVW%cOgga9&rqGZ>eG^xPMJ5l&%X zoRs+O58^q{49c~-6FUGY;2mBLLx99^TjBrHOVGfqgO?RY!r?&)_|KaKNHiw>~pI%cx zSX?u6v^VR4N(4L~Pv_(%s)v;GYNO1PPX$5{49(&*@j_0W)Qw(YvjX_dMYPjTTTz43@KoK(b`BO z_RpM!eL#1?M*l_4Q_hEZDoqU&H48cW!j4lY+x{;&K|WQ6DB&MP2K$PDcZ7?S7dgNx ztC~KeWibZb0GAP7g8kHAWgR-)7!Pglh(c8kCpUYvyZ~UPfrHauR7txg*j#+0lt3s^ z{>cbGGZr#2$mUeXFNL6(k4%6*g>lumYG*YhZCDgy`f_0x!k1icH(_f?A{QKp7A>K& zqb6v*3*&&eGK`roQCf)-jVGXku?-g4uA4{U(Wf0xYEHbbq$ojRZHO%*fVeflX7+kO z&N#v5K4j3$W$pYuG?DURH42&@!S;0)Y?iW+(cCjlW+DR*Dv?32u$2^>5p(z!;XV;- z{J}ODNZZRnKD;7|0e%-eShk5ixa;CnO6Yu+Eu*2}`DdgG+PVbb7!f8tzN)hFOv79$ zI35jp>>+fO7ms|?@=>=Tbj4|1;*He2oV+8c6Ge*O2qbYUz$MNeh%X_D>FR_HomvsV z2ZInJgs)A$GHf*FSdc?=Gi1qM6l(0qk^_*sPO6p;wmv&al`HfaL&V~Bg5NVI!xL9+ z#G!R8mjF!vP%@`_NlI$-Ksj1=Bp`u_@od$AdBKsp`?&xQDGKC5z&aG8&}AP3=j=)L zR>dN`7HS))*0HQZoQ-r6Z`BQ}grfUU^W|_vdyp(irI#CO%Si{}LI}5^To03C=8_Sd z!}ncXS-Xkh=#D7_o9D7%I$rP+3j>6@kTVcUh7?sfKgto=Q)gNXRCzWBw z2D24eHfgj+6H_QQe3-xw2YCvPu?s@~K3mTB1*asPq{y6$)X$Oj5CHrYHSQqzNl!^T(TF48Ic|Ubp@i2MEl9>_ zhnZsj9!4++<5H49t;NAgVvVgF$P@lUxppQK084445PLEz$ZA^ij1F-QZeeQ`7zUQQ z2VZ|rOs~~hYe{BH_>&+Y_f(L^^Fqx3DZM&{p#Nj4oM-RI7@GatWye^7}Mgu!q)t(unc@Qt0S|hfW@@+ z7;jnYuEie(aw5UD1~ET7$W77QOwieSzd=Z@1fLgGxSQ%SlBZ2Nm!!DnR>qMdn5pR| zzp$*EEYY~3>}C7qxNJoC@#zs}`3nocw2KQNvMNeY|Ih&|mT%mUf+Z1A7^Ob!v0SG3 zK{}y}7`B4vD~%SzyKA(k9${{gbWK$;oT7=V!BE8)glm(&rj0Aq22BqEULbsKcQnkB z#2GR$7=;>F;4tJ6q`0%um)nKJMlUnX*2UL7LqT{Z<{vHiB>qV$oeYIi{-Xy-uW8Am^$sG z5ydyvzQ_5n*US(d}AgDEoUPXiP;{eNHo5}-x45NS%>uk@h4O|A9^L>@g7N- zHI_0a^!5SHaO_3q4a5}=cNEDIK{y-R_>54J$-;+0uJ}{hoS<+Fdyf+SqqVScRu7MT zC5+3kqk5^`O9#B6BJ1=oYI`_~WCKCYx5&ZL2uoCs))~ToWVV3v?`)#5{z6-q?@FzN z3*OKRB$#{FHj@!<_l;v`lpIiW(KXI}!Xmc<5g|CEx??T_fY>Yq=+)r;CRHaMMiaq>1DaasvMwT}_GoIoN5LT|oSMof#5 zz7=*IX8+J66;j&borW6|Ep;g|qK}XL=2=SRa_=gWU73Nc(ba&dxq!9o-2|FCqr=ooC*siek%ki{)&)9Oeb}5 zkTxcUrrKzXQcosLVALBHY*-Jks%}h7+&;~U03k;?Q5)g#BmHlNy@DO ztZ(3w7`vD*lv7hqwsy!=Gl{N!)%V{nTD1A2l4ph0hiT9g!MTdt6Er5WzqMPjwf=qz zXT57zNc}bZ>zei~qJb#ufbddW1>hJ9h$F_>OU@}-QJbHmMIp9e52F0RRodKI>S|YE z4EgI{Pt6pYuzB2tEd;`@c%?<~xNRv`JwAUyi9M~z!G)F<<9GbWd}T6D8F{%xC2Vr4IG-kw62q7Q*Hh%VG*Q7Q54dQHAOlc}9{!^X)s$0dz)6)hDc6HuP z+Vk)hkz9Vjf!NcZyiMl z8|*#X`F{TU(XwJPAsNtc94T1@%TeO8I;O^K)TuyiY;lrUK)x4IT4SppDY#;*HiAH< z-=;lKcto9)Dn**2OCE1DmW<@~DC-}orot4@Z5|P6h_@c?eQAvLDILBHC6%~idD1OO zmtj`Y@K#!@tYFZbu5d`LQD<+Fhk~AI#VJ#01s@GjpsO)6#b5ar6&xi!ei_ruGK!*uitW-+|B5w|Dy=t*#BJE7!&Tml) zDxC*I9y2EuU=$WEn#3b?H0Su#EU=2?pIFxf8+lze0hZ7L=mPEOhzi$BSS+-TW1scM zN(48Lj2SVqD=GBUGk<&+Iyt+X!OIKWYs%ONsP-%AtX;J(};n{T&R#rOu5$bw5IsO1`rqz>IK@DX#c^BF0?OjE- zh50JC5ku46>_7d|E*n(RhFQTL_^WNsVK)B1(8cN_r01!^q4(cX{aO$P9EOa}Hzmnd zO(?=;0tMmf1zyV6rh;d{Npm}p}3)7#JPF2ViFg*4YS z!hphzKE`C2DGpu71H`oiU*vwTeH|YNSg;#Gbkzwir!Sv!F{j)xo5mEJ6;#ek21~}7 zI2vfgNRSS)zM^u^^`eGjQ5Z2{{e6MI30?u@L8SOdja1|vcu-SeNS7ijIiQ zLq%^)xvBW3h2+7$O<{)zZ})Riu%OJkIsE2HGDieYQqDB{A~^EQF_uQoV$$Uphc&bu zho2`x`+ORIWn+C!l6MgcAZeA!^?;xHL)+dYFbaM5b(q^o<2OT4(vC%H>hV`Y(pMFO zK6x#W`hJ5Ct2N32JtfAJ5gx3*V+MhuJDN00j@h2QpTE%QAUyV1+KPOqZ|ns}u;2ot zG}$*t*!|ltm`LolNUQjLkrmC(Ry4{AHP=N8v3X>i46;1|@(bu{7HfZyK#`#M1*@zX z13EC*@ebpsi;H1~osGSEH0Clg_oghRu$f{Vs$-uCoM0ZBU;=H;L$P?&WTUh_P7+}i zLT7OnF0P4fsrm)AUL&c6h zNtE2gt~wrDmJ=?lcgCGt_6`A2=nnH^GmZA+`{ShvK6b^Y>*6GWoBi^UwudOsy-5S6 z=*aa+!m8ZSIRLlR``9++&2C`MbqUz6B8Q43to5#aFokA3E0`N++L-F?r_xB>?R*nk-w0t;| z!hu9Vm@qb(dO}!r=7^TejP00gHmbuHC6Jlkw%>7(m|r;5HL$#nKWl&?&8vyR`K&?X ze?d*}pok+DuKf>l-VTy^08@NR8tj*5JJ^bm7!vh@rITYS^~i93#jRT~?WE}eEY~A} zk`WU*USP5Na^Vz}nmKegX`qOw00jbCcahuEn!zi!*>WJ^I*NE@zNHXjW$-uCX^bQe z5yJ((_&M6ABgxr#O*VjhV`VR68UE2*NaUs{2t*RIo;6n<)VA z0os{^*hx@1&Egac-S^GCq#*5tQG&gCZn~4UlNF`0rOh1W_zyA80Ef>%Y;(j$Nghv~ z7|75E)MtcfBUyk3pM))dFV7hyrIDFb@6rlVxO>yMhRAzR{En<2|Ir&x$a_UNy1ec_ zB51cf&sUR*ioe89>rq2PUj3d4*^u)Glpe|Pszr(?u9P8q?!l@8{BJ0a7lO>7+ht6o z6Iz1CmI>LM@!)fWz^cYa+#wv>Lbb8wXS`yGD9(sStMe4xW#p}S!g4>u+J zGC++n-s?z7*<4Y)zXFhRg$f#wjW{uWZNfzCY9jfBSM(Ia{>(%eK5odWQDrkCot>gtr(V@D!AZ{QQ`ySsr& zZ5qsG%S{9JaafIpyh$upMA$KP#Po>wQVdq11pIYL<4a%H%D=9avYujv>vy6SudDct z)q^jXN3-SPJ+Gl*?^QtJKE3HvR#Zi@EMTun)p8rGLxMn++HLeGE`X5XWomUnJ9(2S}U{X8tAS z<7gKs?#+odm~G+-oAiQY7a7HP!F*tk%wHCfe~CJe-*ZP1?jL8?bI>q3XNGU*o7A3m zgsCk;{aH!{Ok4B{T`w?FK3$Sk=b^l&64Gx$vFRra^ctm4wek9e`bxbvVU9&$wju*H z6t&bZ{6?QA+^BPIguq8cF{BRc;v|+qCShR!+9^5&Nf?gx5&T*?tlXQ0WFtiC%SiVc zvg6iLaU}gc70u%$#;Br9wZJ7os}qNl$B9WTwFcK=Qy-yOqv$BLOJ3VxfLt0eRpUr|d_RmAwfzs& zEq4kvU8+cE@kGr?^-Q%hB_k(U4jT9ZEj3k~Gc-*_sE%Y*&+t(I`bS6I#eXyN>NCxYvnKld5h&I{OH^5-F zrJ_YaHcK0g<=Y&0Vc=ns%Rg5*aqv%=tt2)(JWo-2x#N!KG%%u99w9|LL6}TwcYiyU z+=MDaOOE7;?W#=@tiQ>*-+(Off&}UhI%N4Z-HTc-tUAdFkB;swBw-WCqoH#lqMbZg zo6%3BwA!=mWA#24cp>d%3md^G4Ul$)_C*pqpre=-UBcI|+cjh`U*OdMrmCBncYoBX zNN&(z&HJ&Gkh2y+AE_ET@5;GBUku8W^#`lSVS%CeleeX*|7=7<(_Q_u@7GC6 zo-1STtmj9|mb-7E($437_Z2=TuZBRo&soY9=yjW>yC2cUs3a-Y z6_AEHso=e2bBI-Ovw{z(@_G5cRoxD0`dGgW{qs2Jw_36J8NPXkLDi%2v0WTxSj3v& ztu)s=-O0(jvc1;+_V)Z2rPW~*>f{CS^_|l|_uBhj>*P2q)AxibyOPcA^DDlaeNQ{^ zIP6^Psf#$iaOXKj!`*&!>;~(i>z7uUPEXHcc03fBbYdnt6?}U(To4;29@FOMJ9b;G zk0Y?=g=fYfW%$0hvIGu5zI%mFbkp;Yq_dK@<9SiH<9#Pzpa{cZo|9&<`SEcgOuA?% zpYlz31r^#kG4#t6eLlD9ciFA$n-;u&K%JAPG>4D97@fNJ66dF-X~ewi zE~m5S-)T^7;9Pk;f_6Cn0=Z6@s$mt-+LZ(07kManDoxP9mUY(za0iw@z#A@ zKjSm``JAn7FzK#++^f4m<}U*x&yf1v(E zSRp~pR`2GG<4Bm5$*8}}Ce3~Ot?C-I!DQ$6c~-O2`v-WvvA?kMNjjUs-c}!_LDb~~d`02a_WcBX!+-0K2OP{RM(-#n@3VoMqqw?`0W!O~0QFMW z_j)KY_8Os`gU^@-_MNB#q)hOhNcNEzMd=oP`^%Xu+5|dmtJh=ba#omR zET*qKcolgW?AFr+V`J|P+V2srq}1>e;&~F z(J=2KjUkf+{LeHOQc3j0pa7njlWh*~t|up73H?&`wSS5g=s3#VeaG7mcudr+>@CKu z1+QN4Fp$^g@cHZHzUfQiv+eJ0CkF;Dxj@BeU>xJl+u>S{h2P5SZlvkibf@2zC$IxY zBG0?^VLVGq=HL4C#tL*OI?F;iVOm?y+tKr1cLxKn>t{rbp4a`xP%=7%5UQ)xd6|yC z_JN^remw@bJa&@yXv%4wCD}zG6q9FN|Xd84>mpAZ+yjlnzjLv1fL%t=@@ix zc)qHXK#R(4fF@r-jPUqpJRyzpmfL@i{D6Nz4%L(Yb!yqWKp@H7y$`>gKp-e|1@wLE z$N?UOqx`bTl@{C`$zJvR8z!YY0>rSla%xQfQQhwdogEV3T|0^k-5%N zGr!_cg>P12S#KIQGHQe;cVADO);C$zswD_8l6_-cE6omh|3IF?DSz6V3x1Q6eX(77 zI;O#d2S_v*G57DS*F4Jy>hRn&0uw5pRkt>OTa6CqXX>pepsz&P&ka zzu8ZsVYdI8w_hFH?n{s!{%3S|N%$RWbUjP+XdgkUyQ1W?M>)8O0ihg$p|speVscA*kxaT-^dDtwYe!Qo0!asaf9P z4`s_?h^;`OBeuS>Mp9rhJFySdo%ZJsScH-6Q=L-V_(N2E6JcC9@Dm_7d zg*=-eXY!d@5YZb*1mt%LLiqp{fCL^vAf`8vAj(x^d(yZhT#S$DkZgj+De!Ri#+9Ut zy&KgSN6cF$FW?ij#q;^b?eXb%J`~YV0QwL0`G2>1S%<{sf4M(90vTwcOtI{H1x;?X zG=zZ!IQTdc9zbOZvM4Jc!g01oP%rc+h#8D)0BkpXv-}AzP^sX($UjCw4t788Z$-mh zpLb3ad!W?sDQLwz%Bgi9hJ1 zIO!x=dZcJOX@r+X*2oyfx7u7rhVl{%2?ptTnqkO^9N$$b%%n#Rz8y#a#c}}l%4`g` zs3d~K7LeaVNM^fhUX5i6hc%|s6;CC#H-R{ip#mov&Ru}$g$B(9U;0I0b70y@2zt>5 zc0tPr9$;%_EzK{n@LRD;cvq!3rz(1{O@PB2`M-fnAuLXer+NEey`~Ec-?BkWXtsDA z7wCe!3TD5j0tebpSK#Xu^Mh{}iq}LJ@8-T(erzm-90q{*ld`6R1hLaRj|-2Ox*km` z4Sj$16c~%U2c2MgF;k+>P-i49bKGBgCM+0f%v&jp_;iT|UESlvcr>Dber2;KJaII1 zr>LKrT1)*Ew|y>4(~=}s_WX$-Ki?Gi|^5ok^K|0d4gGs>oOb*?lP!|V{R z(GLmcZkxT^B~m0)psD&XP~K1XC#+xRLfUJn`XRT6-8#1z^SD)>f8LIfwGfxHsB3F! ztwEq9myzBMK^{d^vMIST#d7N?1X2<1Dg?565PJ|`t{95wRl?~R=_)*o5`X(PP-Who z+Tn3Wj2fL8FY*BPB<2u)c;eYLNfU!3u}Xz60$NfE`Jam2q>yBCI-uMb4|1J~mBh+5-)15?fz&0gV( zXaWJ>xuYz02Fj0#CXNH@vA*r1Q#@-TfEP;8Y}o*`@(wm*q3*qu1{Z~J!bkE-SC8QN zk;#A~M#T!^g1w+gQ&`Bce2M_=H+xhFB= zBU$<Kn9tCFa$d1F*11@BXGD#EdsV5eugQit3}& zfrWtFR&yBK3y$@$MwvdFjez5!qfElgQw9I9oHHb`ku;O}Tg|Qo@^XB2^#R)7z8~Ui z6HBbH0-4Lli!f*;rljw>b9ule^j<7s6uZDAVsh}!OAW;>jO5mRvfLY=9^3D%$)X+= ziq}eX{#nk{i7g|~)U*)lwnZ(oTqR+%JvJOZ$_i+*Fca=C--tz{$!;nL|4aq@jQ4N? zh**6?J^swegXePGFyA9Xv)XF2j}tAi!t? zsyJnhN8s(!#Il1p$Yhnc`wD(CLPpEUSMqtfp4JX4Y26rfd7zDLpj+Xq_T%6PDNz09 zha_!_rR4RFDl$JfSzw7&(z>#!1`g>B8maWC{1%`=_QbU#Pt_yQG;__6YP?H%4NiOFP;EL|v0tw5?ovK)#!y=>nV9xCCVaYCo zZJ`TBy|Lh=P5q%780RD)0xhZwFI{q5!gq7XNJ}n%RVPlQ>8uULiH(q^QK11，查看集群、作业、DAG 和日志。作业仍通过 SeaTunnel CLI 或 REST API 提交；Helm 只启动引擎，不自动建图或导入数据。 -名称可由 `seatunnel.fullnameOverride` 修改；DNS 后缀由 `seatunnel.clusterDomain` 指定，默认为 `cluster.local`。两种 HugeGraph 访问模式都不会自动公开 SeaTunnel API。 +release 为 `hugegraph` 时，资源为 `hugegraph-seatunnel` StatefulSet、同名 HTTP Service、`hugegraph-seatunnel-headless` 发现 Service，以及 `hugegraph-seatunnel-configs` ConfigMap。名称遵循 HugeGraph 的 `fullnameOverride`；成员通过 Headless Service 的 DNS 发现，不访问 Kubernetes API，因此不需要发现用的 RBAC。 -切换前先停止作业或创建 savepoint，保留当前完整 values 和凭据，再升级： +切换前先停止全部作业，保留安装使用的完整 values，再升级： ```bash helm upgrade hugegraph ./helm/hugegraph -n hugegraph -f my-values.yaml \ --set seatunnel.enabled=true --set seatunnel.mode=cluster \ - --set seatunnel.master.replicas=2 --set seatunnel.worker.replicas=2 \ - --wait --timeout 15m + --set seatunnel.replicas=2 --wait --timeout 15m ``` -切回单机设置 `seatunnel.mode=standalone`、`seatunnel.master.replicas=1`。单机使用 Recreate 策略避免两个混合节点重叠；模式切换会重启 master，作业不会自动迁移或重新提交。设为 `seatunnel.enabled=false` 会移除本 release 管理的 SeaTunnel，不撤回已写入的图数据，也不删除外部 Secret 或存储。 +切回单机设置 `seatunnel.mode=standalone`。两种模式使用同一个 StatefulSet，但副本和引擎配置会变化；不承诺无中断切换，也不会自动恢复或重新提交作业。设置 `seatunnel.enabled=false` 移除本 release 的 SeaTunnel，已经写入 HugeGraph 的数据保留。 -## 连接器镜像与凭据 - -先确认所选镜像包含与引擎版本一致的 `connector-hugegraph` 及其依赖。已有合适镜像可以直接使用；Chart 不根据镜像标签推断连接器清单。如需固定连接器组合，可使用 [示例 Dockerfile](../examples/seatunnel/Dockerfile)： - -```bash -docker build -t registry.example.com/data/seatunnel-hugegraph:3.0.0 \ - helm/hugegraph/examples/seatunnel -docker push registry.example.com/data/seatunnel-hugegraph:3.0.0 -``` +## 镜像、凭据与原生配置 -它在构建时执行官方 `install-plugin.sh 3.0.0`，安装 HugeGraph、FakeSource、Console。实际数据源所需的其他连接器和驱动也应在构建时加入；Helm 不下载连接器或自动提交作业。 +优先复用官方镜像或已有镜像，需确认其包含版本匹配的 HugeGraph 连接器及依赖。镜像通过 `seatunnel.image.repository/tag/pullPolicy` 选择，继承 HugeGraph 的 `global.imageRegistry` 和 `imagePullSecrets`；Helm 不在启动时下载插件。 -准备 `hugegraph-seatunnel-credentials` Secret，包含作业账号的 `username`、`password` 两个键。将以下配置合并到安装使用的 values 文件，镜像已有连接器时保留自己的镜像引用即可： +凭据使用原生 Kubernetes Secret 引用。例如，本地验证可直接引用默认 Helm 管理员 Secret，无需复制密码： ```yaml seatunnel: - image: - registry: registry.example.com/data/seatunnel-hugegraph - tag: "3.0.0" - digest: "" # 可选，设置后优先于 tag - pullSecret: "" env: - name: HG_USERNAME - valueFrom: - secretKeyRef: - name: hugegraph-seatunnel-credentials - key: username + value: admin - name: HG_PASSWORD valueFrom: secretKeyRef: - name: hugegraph-seatunnel-credentials + name: hugegraph-admin key: password ``` -这些变量同时进入 master 和 worker，作业文件通过 `${HG_USERNAME}`、`${HG_PASSWORD}` 读取凭据。Chart 不创建 HugeGraph 账号，也不复制管理员密码；账号需有目标图的读写及 schema 权限。更新 Secret 后需重启相关 Pod，刷新环境变量。 +账号需有作业涉及的图、数据和 Schema 权限。实际作业可改用专用账号。所有 hybrid 节点接收同样的环境变量；更新 Secret 后重启相关 Pod，刷新环境变量。作业文件通过 `${HG_USERNAME}`、`${HG_PASSWORD}` 读取凭据。 -使用默认自动生成的管理员 Secret 进行本地验证时，也可将 `HG_USERNAME` 设置为 `admin`，让 `HG_PASSWORD` 的 `secretKeyRef` 直接引用 `-admin` 的 `password` 键，无需另建或复制 Secret。本次 1+1+1 验证使用此方式；实际作业按需使用专用账号。 +`seatunnel.javaOpts` 调整引擎 JVM，`resources` 调整容器资源。高级配置保留 SeaTunnel 原生文件格式，通过 `seatunnel.config` 按文件名完整覆盖，不逐项增加 Helm 开关，例如： + +```yaml +seatunnel: + javaOpts: "-Xms512m -Xmx1g" + config: + jvm_client_options: | + -Xms128m + -Xmx256m +``` -`global.imageRegistry` 同样作用于 SeaTunnel；显式指定完整仓库地址优先。SeaTunnel 保留上游的 `image.registry`、`image.pullSecret` 键，不继承 HugeGraph 顶层的 `imagePullSecrets`。 +默认生成 `hazelcast.yaml`、`hazelcast-client.yaml`、`seatunnel.yaml` 和 JVM 文件，其余配置沿用镜像。覆盖发现或引擎文件时保留 TCP 5801、HTTP 8080 和 `enable-http: true`，才能继续使用现有 Service、探针及前端。默认 checkpoint 使用临时本地目录；本集成不提供持久化作业恢复保证。 -## 场景一：直接导入 Graph A +strict 模式允许 SeaTunnel 成员通信、DNS，以及访问同 release 的 HugeGraph Server。远端图 A、其他数据源及调用方，通过 `networkPolicy.seatunnel.extraEgress/extraIngress` 补充原生规则；实际隔离依赖 CNI 支持。HugeGraph 的凭据、存储和访问方式见 [Chart README](../README.md)。 -示例链路为 `FakeSource → graph_a`，导入固定的十条数据：`person_01/21` 到 `person_10/30`。随后通过 `HugeGraph Source → HugeGraph Sink` 复制到 `graph_b`。示例仅使用 `seatunnel_demo_person` 顶点标签，主键为 `name`，属性为 `name`、`age`。 +## 简单导入：十条数据写入 Graph A -先在 `DEFAULT` 图空间准备两个空图 `graph_a`、`graph_b`。可通过 Hubble 创建，或使用有建图权限的账号调用 API。以下 curl 在本机执行，需先由凭据管理器向本机环境提供 `HG_USERNAME`、`HG_PASSWORD`；Pod 的环境变量不会自动传到本机。 +[hugegraph-sink.conf](../examples/seatunnel/hugegraph-sink.conf) 使用 `FakeSource → HugeGraph`，写入 `seatunnel_demo_person` 标签的十条固定数据。先创建 `DEFAULT/graph_a`；本机 curl 所用的 `HG_USERNAME`、`HG_PASSWORD` 需由凭据管理器提供，Pod 环境变量不会传到本机。 ```bash -# 在另一个终端保持端口转发 +# 在另一个终端保持转发 kubectl -n hugegraph port-forward svc/hugegraph-server 8082:8080 ``` ```bash export HG_URL=http://127.0.0.1:8082 -for demo_graph in graph_a graph_b; do - curl -fsS -u "$HG_USERNAME:$HG_PASSWORD" -H 'Content-Type: application/json' \ - -X POST "$HG_URL/graphspaces/DEFAULT/graphs/$demo_graph" -d '{}' -done -``` - -已有图不重复执行建图命令。按实际 release、Server 端口、图空间调整示例连接信息，再提交 [hugegraph-sink.conf](../examples/seatunnel/hugegraph-sink.conf)： +curl -fsS -u "$HG_USERNAME:$HG_PASSWORD" -H 'Content-Type: application/json' \ + -X POST "$HG_URL/graphspaces/DEFAULT/graphs/graph_a" -d '{}' -```bash -kubectl -n hugegraph exec -i deploy/hugegraph-seatunnel-master -- \ +kubectl -n hugegraph exec -i sts/hugegraph-seatunnel -- \ sh -c 'cat > /tmp/hugegraph-sink.conf' \ < helm/hugegraph/examples/seatunnel/hugegraph-sink.conf -kubectl -n hugegraph exec deploy/hugegraph-seatunnel-master -- \ +kubectl -n hugegraph exec sts/hugegraph-seatunnel -- \ /opt/seatunnel/bin/seatunnel.sh --master cluster --config /tmp/hugegraph-sink.conf -``` - -`--master cluster` 表示客户端提交到已启动的引擎，单机混合节点也使用这个参数。CLI 未开启 `--async`，等待作业结束。检查作业成功，并用 Gremlin 核对 Graph A 的顶点数量： -```bash curl -fsS -u "$HG_USERNAME:$HG_PASSWORD" -H 'Content-Type: application/json' \ "$HG_URL/gremlin" -d '{"gremlin":"graph.traversal().V().hasLabel(\"seatunnel_demo_person\").count()","aliases":{"graph":"DEFAULT-graph_a"}}' ``` -空图首次导入的 `result.data` 应为 `[10]`。[hugegraph-source.conf](../examples/seatunnel/hugegraph-source.conf) 可按相同方式提交，将 Graph A 的内容输出到 worker 日志；单机模式查看 master 日志。 +已有图不重复建图。首次空图导入后，作业应为 `FINISHED`，读取、写入各十条，Gremlin 的 `result.data` 为 `[10]`。`--master cluster` 表示提交给常驻引擎，单节点也使用该参数。 -## 场景二：Graph A 复制到 Graph B +## 新环境复制：Schema、顶点和边 -[hugegraph-copy.conf](../examples/seatunnel/hugegraph-copy.conf) 使用 HugeGraph 作为 source 和 sink，读取 `graph_a`，写入 `graph_b`。目标标签和属性不存在时由 sink 创建，两个图本身仍需预先存在。 - -```bash -kubectl -n hugegraph exec -i deploy/hugegraph-seatunnel-master -- \ - sh -c 'cat > /tmp/hugegraph-copy.conf' \ - < helm/hugegraph/examples/seatunnel/hugegraph-copy.conf -kubectl -n hugegraph exec deploy/hugegraph-seatunnel-master -- \ - /opt/seatunnel/bin/seatunnel.sh --master cluster --config /tmp/hugegraph-copy.conf -``` +这里使用已有图 A 和另一个环境中的空图 B，不接续上面的十条数据示例。复制顺序是 **Schema → 顶点 → 边**。SeaTunnel Sink 的自动建 Schema 只能根据映射和行类型创建所需结构，不能替代源图完整 Schema、索引及 ID 的复制。 -作业成功后，比较两图数量及按 `name` 排序后的逐条内容： +先通过 HugeGraph 原生 API 创建空的 `graph_b`，再执行 [copy-schema.py](../examples/seatunnel/clone/copy-schema.py)。脚本在目标图的 RESTORING 状态下保留源 Schema ID，复制属性键、顶点标签、边标签和索引；不传输图数据。源、目标应使用兼容的 HugeGraph 版本，账号需有相应管理权限，目标 Schema 必须为空。 ```bash -curl -fsS -u "$HG_USERNAME:$HG_PASSWORD" -H 'Content-Type: application/json' \ - "$HG_URL/gremlin" -d '{"gremlin":"def scan = { x -> x.traversal().V().hasLabel(\"seatunnel_demo_person\").order().by(\"name\").project(\"name\",\"age\").by(\"name\").by(\"age\").toList() }; def aa=scan(a); def bb=scan(b); [count_a:aa.size(),count_b:bb.size(),same:aa==bb,rows:bb]","aliases":{"a":"DEFAULT-graph_a","b":"DEFAULT-graph_b"}}' -``` - -期望两边均为十条，`same=true`，目标数据包含 `person_01/21` 到 `person_10/30`。这里复制的是一个顶点标签，不是自动克隆全图 schema、多标签和全部边。复制边时需补充对应映射，并先写入端点顶点；其他 ID 策略按连接器规则配置。 - -更多配置见 [HugeGraph source](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/source/HugeGraph.md)、[sink](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/sink/HugeGraph.md) 和 [HugeGraph 连接器指南](https://hugegraph.apache.org/cn/docs/quickstart/toolchain/import/hugegraph-seatunnel-connector/)。Pod Ready、`helm --wait`、作业成功和图数据一致是不同层面的检查。 - -## 存储与网络 - -`seatunnel.engineConfig` 合并原生 `seatunnel.engine` 配置。默认 checkpoint 使用临时本地目录；跨 Pod 恢复需配置共享持久化存储及相应客户端依赖、凭据和网络，例如： - -```yaml -seatunnel: - engineConfig: - checkpoint: - storage: - type: hdfs - plugin-config: - storage.type: hdfs - fs.defaultFS: hdfs://namenode.example.com:8020 - namespace: /seatunnel/checkpoints +# 两个 URL 均包含图空间及图名；对应账号密码由本机环境提供 +export HG_SOURCE_GRAPH_URL=https://source.example.com/graphspaces/DEFAULT/graphs/graph_a +export HG_TARGET_GRAPH_URL=https://target.example.com/graphspaces/DEFAULT/graphs/graph_b +python3 helm/hugegraph/examples/seatunnel/clone/copy-schema.py \ + --source "$HG_SOURCE_GRAPH_URL" --target "$HG_TARGET_GRAPH_URL" ``` -额外挂载使用 `seatunnel.extraVolumes`、`extraVolumeMounts`。完整文件覆盖使用 `seatunnel.configMap.data`；单机 JVM 文件是 `jvm_options`，集群是 `jvm_master_options`、`jvm_worker_options`。原始 `data["seatunnel.yaml"]` 会覆盖引擎配置合并结果。 - -使用外部 ConfigMap 时设置 `configMap.create=false` 和 `existingConfigMapName`，必须提供全部十个文件：`hazelcast.yaml`、`hazelcast-client.yaml`、`hazelcast-master.yaml`、`hazelcast-worker.yaml`、`jvm_options`、`jvm_client_options`、`jvm_master_options`、`jvm_worker_options`、`log4j2.properties`、`seatunnel.yaml`。生成配置变化会触发滚动更新，外部 ConfigMap 因为 `subPath` 挂载需要手动重启 Pod。 - -开启 NetworkPolicy 时自动允许 SeaTunnel 成员间 TCP 5801、HTTP 端口、DNS，以及访问本 release 的 HugeGraph Server API。外部数据源、checkpoint 服务和 REST 调用方通过 `networkPolicy.seatunnel.extraEgress/extraIngress` 补充原生规则；实际隔离依赖 CNI 支持。使用外部 SeaTunnel 时保持 `seatunnel.enabled=false`，按实际来源配置 `networkPolicy.server.extraIngress`。 - -本集成保留官方 Helm 的 Deployment 方式；[官方 Kubernetes 指南](https://seatunnel.apache.org/zh-CN/docs/3.0.0/getting-started/kubernetes/) 推荐生产环境使用 StatefulSet。副本数增加不等于 checkpoint 持久化，生产前需验证存储、调度和故障恢复。 +脚本读取 `HG_SOURCE_USERNAME/HG_SOURCE_PASSWORD` 和 `HG_TARGET_USERNAME/HG_TARGET_PASSWORD`。随后在 SeaTunnel Pod 中配置相同的两组凭据，以及 `HG_SOURCE_HOST/PORT/PROTOCOL`、`HG_TARGET_HOST/PORT/PROTOCOL`；密码继续用 Secret 注入。源、目标地址必须能从 SeaTunnel Pod 访问，不能填写仅在本机可达的端口转发地址。 -## 维护与验证记录 - -SeaTunnel 模板只在 fork 的 `deploy/kubernetes/seatunnel` 维护。两个仓库并列放置时，在 HugeGraph 根目录执行： +四个示例作业读取 `graph_a`、写入 `graph_b`，先复制两个顶点标签，再复制两类边： ```bash -helm lint ../seatunnel/deploy/kubernetes/seatunnel -helm dependency update helm/hugegraph -helm lint helm/hugegraph --set seatunnel.enabled=true +for clone_job in person company works_at knows; do + kubectl -n hugegraph exec -i sts/hugegraph-seatunnel -- \ + sh -c "cat > /tmp/$clone_job.conf" \ + < "helm/hugegraph/examples/seatunnel/clone/$clone_job.conf" + kubectl -n hugegraph exec sts/hugegraph-seatunnel -- \ + /opt/seatunnel/bin/seatunnel.sh --master cluster --config "/tmp/$clone_job.conf" \ + || break +done ``` -版本变化时更新父 Chart 的依赖版本，移除旧归档，提交新归档及 `Chart.lock`。原生 `file://../../../seatunnel/deploy/kubernetes/seatunnel` 依赖负责打包；最终安装使用随 HugeGraph 发布的归档，不需要相邻源码仓库或联网下载 Chart。 - -### HStore 1+1+1 联合部署实测 +作业使用 `ERROR_WHEN_SCHEMA_NOT_EXIST`，复用已经复制的 Schema；边依赖先前写入的端点顶点。这组映射覆盖 [person](../examples/seatunnel/clone/person.conf)、[company](../examples/seatunnel/clone/company.conf)、[works_at](../examples/seatunnel/clone/works_at.conf) 和 [knows](../examples/seatunnel/clone/knows.conf)。复制其他图时需按实际标签、属性和 ID 策略调整映射，不会自动枚举全图并提交迁移作业。 -2026-10-23（台北时间），在独立 kind / Kubernetes 1.34.0 中直接安装 HugeGraph 父 Chart，使用 `values-single.yaml`：**1 PD + 1 Store + 1 Server**，同时启用 SeaTunnel standalone。没有使用集群外的 HugeGraph 服务。测试 overlay 只调整本地镜像、资源、Service 类型、JVM 参数和 SeaTunnel 凭据引用，关闭 Hubble；保留 PD/Store 持久化及 Server/PD 默认认证。 +这组示例验证 `PRIMARY_KEY` 和 `CUSTOMIZE_NUMBER` 顶点 ID 策略。现有连接器对 `AUTOMATIC` 顶点会生成新 ID，不能用这组映射直接无损复制其边端点；迁移前先核对源图的 ID 策略。 -PD、Store、Server 使用当前 1.8.0 源码 `f60ad3e3b` 的发行包，沿用仓库 Dockerfile 的运行阶段与官方入口。SeaTunnel 使用经官方 SHA-512 校验的 3.0.0 二进制、经 Maven 校验的 HugeGraph/FakeSource/Console 连接器和 Temurin 17 JRE。受测试环境的 Docker VFS/native snapshotter 磁盘限制，四个镜像合并为单层后导入节点，保留原始文件系统、ENV、CMD 和 ENTRYPOINT；没有为此修改项目代码。 - -| 检查 | 实际结果 | -|---|---| -| 父 Helm 安装 | 22:38，`helm install --wait` 成功；PD、Store、Server、SeaTunnel 四个 Pod 均 `1/1 Running`、重启 0 | -| 持久化 | PD 5Gi、Store 10Gi，两个 PVC 均 `Bound` | -| Helm 启动后检查 | `helm test hugegraph --logs` 成功；Server 1.8.0、Gremlin 3.8.1，默认图绑定及索引状态检查通过 | -| 实际存储后端 | Server 配置 `backend=hstore`，PD 中一个 Store 为 `Up`；graph_a、graph_b 的 API 均返回 `backend=hstore`，PD 图目录也包含两图 | -| 默认认证 | 匿名访问图列表返回 401，管理员认证返回 200；SeaTunnel 通过 `secretKeyRef` 直接引用 Helm 已生成的管理员密码 | -| 两个空图 | 导入前 `A=0、B=0` | -| 直接导入 A | 22:40，作业 `1166023243023253505` 为 `FINISHED`，读取 10、写入 10、失败 0；之后 `A=10、B=0` | -| A → B 复制 | 作业 `1166023260857434113` 为 `FINISHED`，读取 10、写入 10、失败 0；`A=10、B=10`，全部 name/age 逐条一致 | -| 图隔离 | 仅将 B 的 `person_01.age` 改为 999，A 仍为 21，确认两个 HStore 图的数据独立 | -| 重复复制 | 作业 `1166023273259991041` 为 `FINISHED`，读取 10、写入 10、失败 0；B 恢复为 21，两图仍各十条且内容一致 | -| 文档命令复核 | 使用仓库原始 HOCON 和本文 CLI 提交；本文 A/B 比较 Gremlin 原文执行成功，返回 `count_a=10、count_b=10、same=true` 及完整十行数据 | -| SeaTunnel 切至集群 | 22:41，1 master + 1 worker 均 Ready、PID 1 为 Java；复制作业 `1166023709937369090` 为 `FINISHED`，读取 10、写入 10、失败 0；B 单独改为 999 后再次复制恢复，两图各十条且逐属性一致 | -| SeaTunnel 切回单机 | 22:43，恢复一个 Ready 混合节点、PID 1 为 Java，worker 已移除；两图仍为 HStore，各十条且 `same=true`，认证仍返回 401/200 | -| 切换期间 HugeGraph 状态 | 原 PD、Store、Server Pod 均未重建、重启 0；两个 PVC 保持 Bound，三个认证 Secret 的值未变化 | - -Gremlin 将上述 Map 序列化为 `result.data=[{"count_a":10},{"count_b":10},{"same":true},{"rows":[...]}]`；自动校验时需合并这些对象，不能只读取第一项。 - -### 其他已完成检查 - -| 检查 | 结果与范围 | -|---|---| -| Python Helm 回归 | 23 项通过 | -| Helm 单元测试 | 309 项通过，36 suites、4 snapshots | -| HugeGraph 推送前检查 | `mvn editorconfig:format` 通过；`mvn clean compile -Dmaven.javadoc.skip=true` 全部 38 模块通过 | -| 远端许可检查 | 生成的 `Chart.lock` 按精确路径排除 license header 检查；提交 `0e96f1017` 的 Helm、license header 和 RAT 检查通过 | -| 前一轮 SeaTunnel 切换基线 | 22:08–22:12，使用同一打包依赖，SeaTunnel 单机 → 1 master + 1 worker → 单机均 Ready、重启 0；单机完成导入与复制，集群完成复制，每次读取 10、写入 10、失败 0，两图各十条且内容一致。此轮 HugeGraph 为集群外的 1.8.0/RocksDB，仅作补充记录 | -| 进程与凭据传递 | 两种 SeaTunnel 模式的 PID 1 均为 Java；Secret keyRef → Pod 环境变量 → 原始 HOCON 占位符 → CLI 提交通过 | +验收应同时核对属性键类型、基数、标签、主键、边端点定义、索引和 Schema ID，以及全部顶点 ID/属性、边 ID/端点/属性；只比较数量或作业状态不够。相关选项见官方 [HugeGraph Source](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/source/HugeGraph.md) 和 [Sink](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/sink/HugeGraph.md)。 -两轮集群测试均使用 `master.replicas=1`、`worker.replicas=1`。本轮 HStore 沿用集群默认 `backup-count=1`，前一轮外部 RocksDB 测试显式设置 `backup-count=0`；这里只验证分离角色及数据路径。本轮两次 Helm 模式升级各约 30 秒完成；前一轮还观察到约 30 秒重新选举窗口，不能承诺无中断切换。 +## 本轮实测记录 -本次 HStore 验证覆盖开启默认认证的 1+1+1 Helm 部署及一个顶点标签的导入、复制和图隔离；未覆盖多标签、边、细粒度授权、生产 HA、持久化作业恢复或 CNI 对 NetworkPolicy 的实际执行。公开默认镜像的连接器清单仍需按实际镜像检查。 +待填写：使用上述原生 hybrid 实现，在 HugeGraph Helm 1 PD + 1 Store + 1 Server/HStore 环境验证单副本、多副本、内置前端，并完成新环境 A → B 的 Schema、1000 个点与 2400 条边复制。旧版分离 master/worker 的实测不作为本实现的验证结果。 diff --git a/helm/hugegraph/examples/seatunnel/Dockerfile b/helm/hugegraph/examples/seatunnel/Dockerfile deleted file mode 100644 index 85a46d5fc8..0000000000 --- a/helm/hugegraph/examples/seatunnel/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM apache/seatunnel:3.0.0 - -WORKDIR /opt/seatunnel -RUN apt-get update \ - && apt-get install -y --no-install-recommends curl ca-certificates \ - && rm -rf /var/lib/apt/lists/* -COPY plugin_config config/plugin_config -RUN bash bin/install-plugin.sh 3.0.0 \ - && test -s connectors/connector-hugegraph-3.0.0.jar diff --git a/helm/hugegraph/examples/seatunnel/clone/company.conf b/helm/hugegraph/examples/seatunnel/clone/company.conf new file mode 100644 index 0000000000..a2cbd1174c --- /dev/null +++ b/helm/hugegraph/examples/seatunnel/clone/company.conf @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Existing graph A -> empty graph B; copy exact Schema first through HugeGraph API. +env { job.mode = BATCH, parallelism = 1 } +source { + HugeGraph { + host = ${HG_SOURCE_HOST} + port = ${HG_SOURCE_PORT} + protocol = ${HG_SOURCE_PROTOCOL} + graph_space = DEFAULT + graph_name = graph_a + username = ${HG_SOURCE_USERNAME} + password = ${HG_SOURCE_PASSWORD} + label = "company" + label_type = VERTEX + page_size = 100 + time_zone = UTC + } +} +sink { + HugeGraph { + host = ${HG_TARGET_HOST} + port = ${HG_TARGET_PORT} + protocol = ${HG_TARGET_PROTOCOL} + graph_space = DEFAULT + graph_name = graph_b + username = ${HG_TARGET_USERNAME} + password = ${HG_TARGET_PASSWORD} + schema_save_mode = ERROR_WHEN_SCHEMA_NOT_EXIST + check_vertex = true + batch_size = 100 + mappings = [ + { + type = VERTEX + label = company + idStrategy = CUSTOMIZE_NUMBER + idFields = ["~id"] + } + ] + } +} diff --git a/helm/hugegraph/examples/seatunnel/clone/copy-schema.py b/helm/hugegraph/examples/seatunnel/clone/copy-schema.py new file mode 100644 index 0000000000..b4ec6e8547 --- /dev/null +++ b/helm/hugegraph/examples/seatunnel/clone/copy-schema.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Copy schema to an existing empty graph, preserving IDs for subsequent data copy.""" +import argparse +import base64 +import copy +import json +import os +import urllib.request + +KINDS = ('propertykeys', 'vertexlabels', 'edgelabels', 'indexlabels') + + +def client(base, role): + user = os.environ.get('HG_' + role + '_USERNAME', 'admin') + password = os.environ['HG_' + role + '_PASSWORD'] + token = base64.b64encode((user + ':' + password).encode()).decode() + + def request(method, suffix, body=None): + data = None if body is None else json.dumps(body).encode() + req = urllib.request.Request(base.rstrip('/') + suffix, data=data, + method=method, headers={ + 'Authorization': 'Basic ' + token, + 'Content-Type': 'application/json'}) + with urllib.request.urlopen(req, timeout=60) as response: + raw = response.read().decode() + return json.loads(raw) if raw else None + return request + + +def normalize(schema): + result = copy.deepcopy(schema) + for kind in KINDS: + for element in result[kind]: + element.pop('status', None) + element.get('user_data', {}).pop('~create_time', None) + for key in ('properties', 'nullable_keys', 'index_labels'): + if key in element: + element[key].sort() + result[kind].sort(key=lambda element: element['name']) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--source', required=True, help='Source graph URL, including graph name') + parser.add_argument('--target', required=True, help='Target graph URL, including graph name') + args = parser.parse_args() + source = client(args.source, 'SOURCE') + target = client(args.target, 'TARGET') + schema = source('GET', '/schema') + target_schema = target('GET', '/schema') + if any(target_schema[kind] for kind in KINDS): + raise SystemExit('Target schema must be empty; use a new graph.') + original_mode = target('GET', '/mode')['mode'] + if str(original_mode).upper() != 'NONE': + raise SystemExit('Target graph must be in NONE mode before copying schema.') + target('PUT', '/mode', 'RESTORING') + try: + for kind in KINDS: + for element in schema[kind]: + body = copy.deepcopy(element) + body.pop('status', None) + body.pop('index_labels', None) + if kind == 'indexlabels': + body['rebuild'] = False # Empty graph; subsequent data writes build indexes. + target('POST', '/schema/' + kind, body) + finally: + target('PUT', '/mode', original_mode) + if normalize(schema) != normalize(target('GET', '/schema')): + raise SystemExit('Schema comparison failed; inspect the new graph before copying data.') + print(json.dumps({'schema_equal': True, 'ids_preserved': True, + 'counts': {kind: len(schema[kind]) for kind in KINDS}})) + + +if __name__ == '__main__': + main() diff --git a/helm/hugegraph/examples/seatunnel/clone/knows.conf b/helm/hugegraph/examples/seatunnel/clone/knows.conf new file mode 100644 index 0000000000..8ba2259138 --- /dev/null +++ b/helm/hugegraph/examples/seatunnel/clone/knows.conf @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Existing graph A -> empty graph B; copy exact Schema first through HugeGraph API. +env { job.mode = BATCH, parallelism = 1 } +source { + HugeGraph { + host = ${HG_SOURCE_HOST} + port = ${HG_SOURCE_PORT} + protocol = ${HG_SOURCE_PROTOCOL} + graph_space = DEFAULT + graph_name = graph_a + username = ${HG_SOURCE_USERNAME} + password = ${HG_SOURCE_PASSWORD} + label = "knows" + label_type = EDGE + page_size = 100 + time_zone = UTC + } +} +sink { + HugeGraph { + host = ${HG_TARGET_HOST} + port = ${HG_TARGET_PORT} + protocol = ${HG_TARGET_PROTOCOL} + graph_space = DEFAULT + graph_name = graph_b + username = ${HG_TARGET_USERNAME} + password = ${HG_TARGET_PASSWORD} + schema_save_mode = ERROR_WHEN_SCHEMA_NOT_EXIST + check_vertex = true + batch_size = 100 + mappings = [ + { + type = EDGE + label = knows + frequency = MULTIPLE + sortKeys = [since] + sourceConfig { label = person, idFields = ["~source_id"] } + targetConfig { label = person, idFields = ["~target_id"] } + } + ] + } +} diff --git a/helm/hugegraph/examples/seatunnel/clone/person.conf b/helm/hugegraph/examples/seatunnel/clone/person.conf new file mode 100644 index 0000000000..91e37053b3 --- /dev/null +++ b/helm/hugegraph/examples/seatunnel/clone/person.conf @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Existing graph A -> empty graph B; copy exact Schema first through HugeGraph API. +env { job.mode = BATCH, parallelism = 1 } +source { + HugeGraph { + host = ${HG_SOURCE_HOST} + port = ${HG_SOURCE_PORT} + protocol = ${HG_SOURCE_PROTOCOL} + graph_space = DEFAULT + graph_name = graph_a + username = ${HG_SOURCE_USERNAME} + password = ${HG_SOURCE_PASSWORD} + label = "person" + label_type = VERTEX + page_size = 100 + time_zone = UTC + } +} +sink { + HugeGraph { + host = ${HG_TARGET_HOST} + port = ${HG_TARGET_PORT} + protocol = ${HG_TARGET_PROTOCOL} + graph_space = DEFAULT + graph_name = graph_b + username = ${HG_TARGET_USERNAME} + password = ${HG_TARGET_PASSWORD} + schema_save_mode = ERROR_WHEN_SCHEMA_NOT_EXIST + check_vertex = true + batch_size = 100 + mappings = [ + { + type = VERTEX + label = person + idStrategy = PRIMARY_KEY + idFields = [name] + } + ] + } +} diff --git a/helm/hugegraph/examples/seatunnel/clone/works_at.conf b/helm/hugegraph/examples/seatunnel/clone/works_at.conf new file mode 100644 index 0000000000..2575cf1be4 --- /dev/null +++ b/helm/hugegraph/examples/seatunnel/clone/works_at.conf @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Existing graph A -> empty graph B; copy exact Schema first through HugeGraph API. +env { job.mode = BATCH, parallelism = 1 } +source { + HugeGraph { + host = ${HG_SOURCE_HOST} + port = ${HG_SOURCE_PORT} + protocol = ${HG_SOURCE_PROTOCOL} + graph_space = DEFAULT + graph_name = graph_a + username = ${HG_SOURCE_USERNAME} + password = ${HG_SOURCE_PASSWORD} + label = "works_at" + label_type = EDGE + page_size = 100 + time_zone = UTC + } +} +sink { + HugeGraph { + host = ${HG_TARGET_HOST} + port = ${HG_TARGET_PORT} + protocol = ${HG_TARGET_PROTOCOL} + graph_space = DEFAULT + graph_name = graph_b + username = ${HG_TARGET_USERNAME} + password = ${HG_TARGET_PASSWORD} + schema_save_mode = ERROR_WHEN_SCHEMA_NOT_EXIST + check_vertex = true + batch_size = 100 + mappings = [ + { + type = EDGE + label = works_at + frequency = SINGLE + sourceConfig { label = person, idFields = ["~source_id"] } + targetConfig { label = company, idFields = ["~target_id"] } + } + ] + } +} diff --git a/helm/hugegraph/examples/seatunnel/hugegraph-copy.conf b/helm/hugegraph/examples/seatunnel/hugegraph-copy.conf deleted file mode 100644 index a2fc221be8..0000000000 --- a/helm/hugegraph/examples/seatunnel/hugegraph-copy.conf +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# 将 graph_a 的单个顶点标签复制到 graph_b；两个图需预先创建。 -# PRIMARY_KEY 使用相同的 name 属性重建顶点 ID；该作业不会自动复制全图 schema 或边。 -env { - job.mode = "BATCH" - parallelism = 1 -} - -source { - HugeGraph { - host = "hugegraph-server" - port = 8080 - graph_space = "DEFAULT" - graph_name = "graph_a" - label = "seatunnel_demo_person" - label_type = "VERTEX" - username = ${HG_USERNAME} - password = ${HG_PASSWORD} - schema { - fields { - name = "string" - age = "int" - } - } - } -} - -sink { - HugeGraph { - host = "hugegraph-server" - port = 8080 - graph_space = "DEFAULT" - graph_name = "graph_b" - username = ${HG_USERNAME} - password = ${HG_PASSWORD} - schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" - mappings = [ - { - type = "VERTEX" - label = "seatunnel_demo_person" - idStrategy = "PRIMARY_KEY" - idFields = ["name"] - properties = ["name", "age"] - } - ] - } -} diff --git a/helm/hugegraph/examples/seatunnel/hugegraph-source.conf b/helm/hugegraph/examples/seatunnel/hugegraph-source.conf deleted file mode 100644 index 5f60beae34..0000000000 --- a/helm/hugegraph/examples/seatunnel/hugegraph-source.conf +++ /dev/null @@ -1,43 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# 读取 hugegraph-sink.conf 导入的标签；Console 输出到 worker 日志，单机模式为 master 日志。 -env { - job.mode = "BATCH" - parallelism = 1 -} - -source { - HugeGraph { - host = "hugegraph-server" - port = 8080 - graph_space = "DEFAULT" - graph_name = "graph_a" - label = "seatunnel_demo_person" - label_type = "VERTEX" - username = ${HG_USERNAME} - password = ${HG_PASSWORD} - schema { - fields { - name = "string" - age = "int" - } - } - } -} - -sink { - Console {} -} diff --git a/helm/hugegraph/examples/seatunnel/plugin_config b/helm/hugegraph/examples/seatunnel/plugin_config deleted file mode 100644 index bb4ccc593f..0000000000 --- a/helm/hugegraph/examples/seatunnel/plugin_config +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---seatunnel-connectors-- -connector-hugegraph -connector-fake -connector-console ---end-- diff --git a/helm/hugegraph/examples/values-seatunnel-cluster.yaml b/helm/hugegraph/examples/values-seatunnel-cluster.yaml index 100cccf76f..9e6fe30cb0 100644 --- a/helm/hugegraph/examples/values-seatunnel-cluster.yaml +++ b/helm/hugegraph/examples/values-seatunnel-cluster.yaml @@ -13,33 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -# 使用官方的 master/worker 分离 Deployment，示例仅指定引擎规模。 -# 生产环境需另行配置持久化与故障恢复，见 ../docs/seatunnel.md。 -# 提交作业前确认镜像含 HugeGraph 连接器。 +# 复用官方原生 hybrid StatefulSet；HTTP8080 同时提供 REST 和任务可视化前端。 +# HugeGraph 拓扑独立选择，1 PD + 1 Store + 1 Server 可用 ../values-single.yaml。 seatunnel: enabled: true mode: cluster - master: - replicas: 2 - resources: - requests: - cpu: 500m - memory: 1536Mi - limits: - memory: 2Gi - worker: - replicas: 2 - resources: - requests: - cpu: 500m - memory: 1536Mi - limits: - memory: 2Gi - configMap: - data: - jvm_master_options: | - -Xms512m - -Xmx1g - jvm_worker_options: | - -Xms512m - -Xmx1g + replicas: 2 diff --git a/helm/hugegraph/examples/values-seatunnel-standalone.yaml b/helm/hugegraph/examples/values-seatunnel-standalone.yaml index c286e156e0..e75efd6a58 100644 --- a/helm/hugegraph/examples/values-seatunnel-standalone.yaml +++ b/helm/hugegraph/examples/values-seatunnel-standalone.yaml @@ -13,21 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# 启动一个持续接收作业的 SeaTunnel 混合节点，HugeGraph 拓扑独立选择。 -# 小型环境可同时使用 ../values-single.yaml。 -# 此预设只启动引擎；提交作业前确认镜像含 HugeGraph 连接器，见 ../docs/seatunnel.md。 +# 复用官方原生 hybrid StatefulSet；HTTP8080 同时提供 REST 和任务可视化前端。 +# HugeGraph 拓扑独立选择，1 PD + 1 Store + 1 Server 可用 ../values-single.yaml。 seatunnel: enabled: true mode: standalone - master: - resources: - requests: - cpu: 500m - memory: 1536Mi - limits: - memory: 2Gi - configMap: - data: - jvm_options: | - -Xms512m - -Xmx1g diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt index edc98d8464..2197451374 100644 --- a/helm/hugegraph/templates/NOTES.txt +++ b/helm/hugegraph/templates/NOTES.txt @@ -62,8 +62,9 @@ Access: {{- if (.Values.seatunnel | default dict).enabled }} SeaTunnel: - In-cluster API: http://{{ include "seatunnel.fullname" .Subcharts.seatunnel }}-master.{{ .Release.Namespace }}.svc:{{ .Values.seatunnel.service.port }} - Port-forward: kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "seatunnel.fullname" .Subcharts.seatunnel }}-master 8081:{{ .Values.seatunnel.service.port }} + In-cluster API: http://{{ include "hugegraph.seatunnel.name" . }}.{{ .Release.Namespace }}.svc:8080 + Port-forward: kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "hugegraph.seatunnel.name" . }} 8081:8080 + Web UI: http://localhost:8081/#/overview Connector images, credentials and checkpoint storage: helm/hugegraph/docs/seatunnel.md {{- end }} {{- range $component := list "server" "hubble" "pd" }} diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index 127a442abc..28bced02f1 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -48,7 +48,7 @@ {{- $registry := trimSuffix "/" (get (.Values.global | default dict) "imageRegistry" | default "") -}} {{- if contains "://" $registry -}}{{- fail "global.imageRegistry must be a registry host/path without http:// or https://" -}}{{- end -}} {{- if $registry -}} -{{- range $comp := list "pd" "store" "server" "hubble" -}} +{{- range $comp := list "pd" "store" "server" "hubble" "seatunnel" -}} {{- $image := get (get $.Values $comp) "image" | default dict -}} {{- if $image -}} {{- $_ := set $image "repository" (include "hugegraph.registryImage" (dict "reference" $image.repository "registry" $registry)) -}} @@ -733,6 +733,15 @@ The minimum Server replica count that a PDB must remain valid against. Cross-field validation that JSON Schema draft-07 cannot express. */}} {{- define "hugegraph.validateValues" -}} +{{- if (.Values.seatunnel | default dict).enabled -}} +{{- if not (has .Values.seatunnel.mode (list "standalone" "cluster")) -}} +{{- fail "seatunnel.mode must be standalone or cluster" -}} +{{- end -}} +{{- if and (eq .Values.seatunnel.mode "cluster") (lt (int .Values.seatunnel.replicas) 2) -}} +{{- fail "seatunnel.replicas must be at least 2 in cluster mode" -}} +{{- end -}} +{{- end -}} + {{- include "hugegraph.resolveValues" . -}} {{- $startupBudget := include "hugegraph.server.startupProbeBudget" . | int -}} {{- if lt $startupBudget (add (int .Values.server.storageWaitTimeoutSeconds) 120) -}} @@ -1028,11 +1037,14 @@ resolver (CoreDNS in any namespace, NodeLocal DNSCache). {{- join "\n" $rules -}} {{- end }} -{{/* SeaTunnel is a separate subchart with its own stable application labels. */}} +{{/* Native SeaTunnel resources belong to this HugeGraph release. */}} +{{- define "hugegraph.seatunnel.name" -}} +{{- printf "%s-seatunnel" (include "hugegraph.fullname" . | trunc 44 | trimSuffix "-") -}} +{{- end }} + {{- define "hugegraph.netpol.seatunnelLabels" -}} -app.kubernetes.io/name: seatunnel -app.kubernetes.io/instance: {{ .Release.Name | quote }} -app.kubernetes.io/part-of: seatunnel +{{ include "hugegraph.selectorLabels" . }} +app.kubernetes.io/component: seatunnel {{- end }} {{- define "hugegraph.netpol.seatunnelPeer" -}} diff --git a/helm/hugegraph/templates/seatunnel-networkpolicy.yaml b/helm/hugegraph/templates/seatunnel-networkpolicy.yaml index 89fef5b01b..b9746c95f5 100644 --- a/helm/hugegraph/templates/seatunnel-networkpolicy.yaml +++ b/helm/hugegraph/templates/seatunnel-networkpolicy.yaml @@ -34,7 +34,7 @@ spec: - from: {{- include "hugegraph.netpol.seatunnelPeer" . | nindent 8 }} ports: - {{- include "hugegraph.netpol.ports" (list 5801 (.Values.seatunnel.service.port | default 8080)) | nindent 8 }} + {{- include "hugegraph.netpol.ports" (list 5801 8080) | nindent 8 }} {{- with get $rules "extraIngress" }} {{- toYaml . | nindent 4 }} {{- end }} @@ -42,7 +42,7 @@ spec: - to: {{- include "hugegraph.netpol.seatunnelPeer" . | nindent 8 }} ports: - {{- include "hugegraph.netpol.ports" (list 5801 (.Values.seatunnel.service.port | default 8080)) | nindent 8 }} + {{- include "hugegraph.netpol.ports" (list 5801 8080) | nindent 8 }} - to: {{- include "hugegraph.netpol.peer" (dict "root" . "component" "server") | nindent 8 }} ports: diff --git a/helm/hugegraph/templates/seatunnel.yaml b/helm/hugegraph/templates/seatunnel.yaml new file mode 100644 index 0000000000..924bab90b2 --- /dev/null +++ b/helm/hugegraph/templates/seatunnel.yaml @@ -0,0 +1,186 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Based on Apache SeaTunnel v3.0.0 hybrid Kubernetes resources: +https://github.com/apache/seatunnel/blob/v3.0.0/docs/zh/getting-started/kubernetes/hybrid-cluster-mode.md +Use its DNS discovery alternative, so Kubernetes API RBAC is unnecessary. +*/}} + +{{- define "hugegraph.seatunnel.config" -}} +{{- $name := include "hugegraph.seatunnel.name" . -}} +{{- $dns := printf "%s-headless.%s.svc" $name .Release.Namespace -}} +{{- $data := dict -}} +{{- $hazelcast := dict "hazelcast" (dict + "cluster-name" $name + "network" (dict + "port" (dict "auto-increment" false "port" 5801) + "join" (dict "kubernetes" (dict "enabled" true "service-dns" $dns "service-dns-timeout" 10))) + "properties" (dict "hazelcast.logging.type" "log4j2")) -}} +{{- $_ := set $data "hazelcast.yaml" ($hazelcast | toYaml) -}} +{{- $client := dict "hazelcast-client" (dict + "cluster-name" $name + "properties" (dict "hazelcast.logging.type" "log4j2") + "connection-strategy" (dict "connection-retry" (dict "cluster-connect-timeout-millis" 7000)) + "network" (dict "cluster-members" (list (printf "%s:5801" $dns)))) -}} +{{- $_ := set $data "hazelcast-client.yaml" ($client | toYaml) -}} +{{- $engine := dict "seatunnel" (dict "engine" (dict + "backup-count" (ternary 0 1 (eq .Values.seatunnel.mode "standalone")) + "history-job-expire-minutes" 1440 + "classloader-cache-mode" true + "slot-service" (dict "dynamic-slot" true) + "checkpoint" (dict "interval" 180000 "timeout" 30000 "storage" (dict + "type" "hdfs" "max-retained" 3 "plugin-config" (dict + "namespace" "/tmp/seatunnel/checkpoint/" "storage.type" "hdfs" "fs.defaultFS" "file:///tmp/"))) + "http" (dict "enable-http" true "port" 8080 "enable-dynamic-port" false))) -}} +{{- $_ := set $data "seatunnel.yaml" ($engine | toYaml) -}} +{{/* The upstream script appends jvm_options after JAVA_OPTS; set heap in that file. */}} +{{- $_ := set $data "jvm_options" (.Values.seatunnel.javaOpts | replace " " "\n") -}} +{{- range $filename, $content := .Values.seatunnel.config -}} +{{- $_ := set $data $filename $content -}} +{{- end -}} +{{- toYaml $data -}} +{{- end -}} + +{{- if (.Values.seatunnel | default dict).enabled }} +{{- include "hugegraph.resolveValues" . }} +{{- include "hugegraph.validateValues" . }} +{{- $name := include "hugegraph.seatunnel.name" . }} +{{- $config := include "hugegraph.seatunnel.config" . | fromYaml }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-configs + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: seatunnel +data: + {{- toYaml $config | nindent 2 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }}-headless + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: seatunnel +spec: + clusterIP: None + publishNotReadyAddresses: true + ports: + - name: hazelcast + port: 5801 + targetPort: hazelcast + selector: + {{- include "hugegraph.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: seatunnel +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: seatunnel +spec: + type: ClusterIP + ports: + - name: http + port: 8080 + targetPort: http + selector: + {{- include "hugegraph.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: seatunnel +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $name }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: seatunnel +spec: + serviceName: {{ $name }}-headless + replicas: {{ ternary 1 (int .Values.seatunnel.replicas) (eq .Values.seatunnel.mode "standalone") }} + podManagementPolicy: Parallel + selector: + matchLabels: + {{- include "hugegraph.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: seatunnel + template: + metadata: + labels: + {{- include "hugegraph.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: seatunnel + annotations: + checksum/config: {{ $config | toYaml | sha256sum }} + spec: + automountServiceAccountToken: false + terminationGracePeriodSeconds: 120 + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: seatunnel + image: {{ include "hugegraph.image" (dict "image" .Values.seatunnel.image "appVersion" "3.0.0") | quote }} + imagePullPolicy: {{ .Values.seatunnel.image.pullPolicy }} + # The default upstream role is hybrid; an explicit -r is unnecessary. + command: ["/opt/seatunnel/bin/seatunnel-cluster.sh"] + env: + - name: SEATUNNEL_HOME + value: /opt/seatunnel + {{- with .Values.seatunnel.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8080 + - name: hazelcast + containerPort: 5801 + {{- with .Values.seatunnel.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + startupProbe: + tcpSocket: + port: http + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + tcpSocket: + port: http + periodSeconds: 10 + timeoutSeconds: 5 + livenessProbe: + tcpSocket: + port: hazelcast + periodSeconds: 30 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: ["/opt/seatunnel/bin/stop-seatunnel-cluster.sh"] + volumeMounts: + {{- range $filename, $_ := $config }} + - name: seatunnel-configs + mountPath: /opt/seatunnel/config/{{ $filename }} + subPath: {{ $filename }} + {{- end }} + volumes: + - name: seatunnel-configs + configMap: + name: {{ $name }}-configs +{{- end }} diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 97e961a9ec..99a5e03fe6 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -61,12 +61,23 @@ }, "seatunnel": { "type": "object", + "description": "Optional native SeaTunnel hybrid Kubernetes deployment.", "properties": { "enabled": { "type": "boolean" + }, + "mode": { + "type": "string", + "enum": [ + "standalone", + "cluster" + ] + }, + "replicas": { + "type": "integer", + "minimum": 1 } - }, - "description": "Optional upstream-derived SeaTunnel subchart; see charts/seatunnel/values.yaml." + } } }, "definitions": { diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index dcee3d8205..d6fc5f350c 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -533,12 +533,29 @@ hubble: failureThreshold: 3 timeoutSeconds: 5 -# Optional SeaTunnel engine. Its topology is independent of HugeGraph's presets -# and easy/strict mode. See docs/seatunnel.md for the SeaTunnel chart's native -# configuration, connector images and credentials. +# Optional engine using Apache SeaTunnel's native hybrid Kubernetes deployment. +# One hybrid StatefulSet serves both modes; HTTP includes the built-in Web UI. seatunnel: enabled: false mode: standalone + # Used in cluster mode; standalone always starts one hybrid node. + replicas: 2 + image: + repository: apache/seatunnel + tag: "3.0.0" + pullPolicy: IfNotPresent + javaOpts: "-Xms512m -Xmx1g" + resources: + requests: + cpu: 500m + memory: 1536Mi + limits: + memory: 2Gi + # Native Kubernetes env entries, including Secret keyRef credentials. + env: [] + # Full native configuration files, keyed by filename (e.g. seatunnel.yaml). + # Keep member port 5801 and HTTP 8080 when overriding discovery/engine files. + config: {} # Kubernetes NetworkPolicy: mode default plus explicit per-component rules. # easy does not isolate clients; strict isolates release traffic and DNS. Only diff --git a/helm/test_helm_seatunnel.py b/helm/test_helm_seatunnel.py index dd73245081..5858203546 100644 --- a/helm/test_helm_seatunnel.py +++ b/helm/test_helm_seatunnel.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Check the optional engine's rendered topology, connectivity and release payload.""" +"""Check native hybrid deployment, release isolation, connectivity and packaging.""" from pathlib import Path import shutil @@ -35,7 +35,7 @@ def matches(selector, labels): def engine_resources(documents, kind=None): return [doc for doc in documents if (kind is None or doc["kind"] == kind) - and doc["metadata"].get("labels", {}).get("app.kubernetes.io/part-of") == "seatunnel"] + and doc["metadata"].get("labels", {}).get("app.kubernetes.io/component") == "seatunnel"] def pod(workload): @@ -46,7 +46,7 @@ class SeaTunnelChartTest(unittest.TestCase): @classmethod def setUpClass(cls): if shutil.which("helm") is None: - raise RuntimeError("Helm is required for the SeaTunnel chart regression") + raise RuntimeError("Helm is required for the SeaTunnel integration regression") def render(self, seatunnel=None, chart=CHART, namespace="default", **values): if seatunnel is not None: @@ -59,173 +59,139 @@ def render(self, seatunnel=None, chart=CHART, namespace="default", **values): capture_output=True, text=True) return result - def documents(self, seatunnel=None, **values): - result = self.render(seatunnel, **values) + def documents(self, seatunnel=None, chart=CHART, namespace="default", **values): + result = self.render(seatunnel, chart=chart, namespace=namespace, **values) self.assertEqual(0, result.returncode, result.stderr) return [doc for doc in yaml.safe_load_all(result.stdout) if doc] - def test_disabled_has_no_engine_resources_or_child_validation(self): + def test_disabled_has_no_engine_resources(self): documents = self.documents() self.assertEqual([], engine_resources(documents)) self.assertFalse(any("seatunnel" in doc["metadata"]["name"] for doc in documents)) - # Disabled dependencies must not validate inactive engine settings. - self.assertEqual(0, self.render({"enabled": False, "mode": "unused"}).returncode) - - def test_standalone_is_one_hybrid_with_usable_configuration(self): - documents = self.documents({}) - workloads = engine_resources(documents, "Deployment") - self.assertEqual(1, len(workloads)) - master = workloads[0] - self.assertEqual(1, master["spec"]["replicas"]) - self.assertEqual("Recreate", master["spec"]["strategy"]["type"]) - container = pod(master)["spec"]["containers"][0] - # SeaTunnel 3.0.0 selects the hybrid role when -r is omitted; its Java - # entrypoint rejects an explicit -r master_and_worker. - self.assertTrue(container["command"][-1].rstrip().endswith("/seatunnel-cluster.sh")) - self.assertNotIn(" -r ", container["command"][-1]) - self.assertFalse(pod(master)["spec"]["automountServiceAccountToken"]) - mounts = {mount["subPath"] for mount in container["volumeMounts"]} - self.assertTrue({"hazelcast.yaml", "jvm_options", "seatunnel.yaml"} <= mounts) - config = engine_resources(documents, "ConfigMap")[0]["data"] - engine = yaml.safe_load(config["seatunnel.yaml"])["seatunnel"]["engine"] - self.assertEqual(0, engine["backup-count"]) - self.assertTrue(engine["http"]["enable-http"]) - self.assertFalse(engine["http"]["enable-dynamic-port"]) - - def test_cluster_roles_and_mode_switch_preserve_master_identity(self): - standalone = self.documents({}) - cluster = self.documents({"mode": "cluster"}) - workloads = engine_resources(cluster, "Deployment") - self.assertEqual({"master", "worker"}, - {doc["metadata"]["labels"]["app.kubernetes.io/component"] for doc in workloads}) - for workload in workloads: - role = workload["metadata"]["labels"]["app.kubernetes.io/component"] - self.assertEqual(2, workload["spec"]["replicas"]) - self.assertTrue(pod(workload)["spec"]["containers"][0]["command"][-1].rstrip().endswith("-r " + role)) - first = engine_resources(standalone, "Deployment")[0] - master = next(doc for doc in workloads if doc["metadata"]["name"] == first["metadata"]["name"]) - self.assertEqual(first["spec"]["selector"], master["spec"]["selector"]) + + def test_modes_share_one_native_hybrid_workload_and_services(self): + modes = [self.documents({}), self.documents({"mode": "cluster", "replicas": 3})] + workloads = [] + for documents, replicas, backups in zip(modes, (1, 3), (0, 1)): + workload, = engine_resources(documents, "StatefulSet") + workloads.append(workload) + self.assertEqual([], engine_resources(documents, "Deployment")) + self.assertEqual(replicas, workload["spec"]["replicas"]) + container, = pod(workload)["spec"]["containers"] + launch = " ".join(container.get("command", []) + container.get("args", [])) + self.assertIn("/opt/seatunnel/bin/seatunnel-cluster.sh", launch) + self.assertNotIn(" -r ", launch) + data = engine_resources(documents, "ConfigMap")[0]["data"] + engine = yaml.safe_load(data["seatunnel.yaml"])["seatunnel"]["engine"] + self.assertEqual(backups, engine["backup-count"]) + self.assertTrue(engine["http"]["enable-http"]) + self.assertEqual(8080, engine["http"]["port"]) + self.assertEqual(workloads[0]["metadata"]["name"], workloads[1]["metadata"]["name"]) + self.assertEqual(workloads[0]["spec"]["selector"], workloads[1]["spec"]["selector"]) services = lambda docs: {doc["metadata"]["name"]: doc["spec"]["selector"] for doc in engine_resources(docs, "Service")} - self.assertEqual(services(standalone), services(cluster)) - self.assertNotEqual(pod(first)["metadata"]["annotations"]["checksum/config"], - pod(master)["metadata"]["annotations"]["checksum/config"]) - data = engine_resources(cluster, "ConfigMap")[0]["data"] - self.assertEqual(1, yaml.safe_load(data["seatunnel.yaml"])["seatunnel"]["engine"]["backup-count"]) - - def test_peer_service_selects_both_roles_and_only_its_release(self): - documents = self.documents({"mode": "cluster", "fullnameOverride": "transfer", - "clusterDomain": "corp.internal"}, namespace="graphs") - service = next(doc for doc in engine_resources(documents, "Service") - if doc["spec"].get("clusterIP") == "None") - self.assertEqual("transfer", service["metadata"]["name"]) - self.assertTrue(service["spec"]["publishNotReadyAddresses"]) - selector = service["spec"]["selector"] - workloads = [doc for doc in documents if doc["kind"] in ("Deployment", "StatefulSet")] - selected = [doc for doc in workloads if matches(selector, pod(doc)["metadata"]["labels"])] - self.assertEqual(2, len(selected)) - self.assertFalse(matches(selector, {**pod(selected[0])["metadata"]["labels"], - "app.kubernetes.io/instance": "another-release"})) - config = engine_resources(documents, "ConfigMap")[0]["data"] - for filename in ("hazelcast.yaml", "hazelcast-master.yaml", "hazelcast-worker.yaml"): - discovery = yaml.safe_load(config[filename])["hazelcast"]["network"]["join"]["kubernetes"] - self.assertEqual("transfer.graphs.svc.corp.internal", discovery["service-dns"]) - self.assertIn("transfer.graphs.svc.corp.internal:5801", config["hazelcast-client.yaml"]) - - def test_runtime_overrides_and_config_changes_reach_pods(self): - digest = "sha256:" + "a" * 64 - values = {"image": {"registry": "team/seatunnel", "digest": digest, "pullSecret": "registry-auth"}, - "env": [{"name": "GRAPH_PASSWORD", "valueFrom": { - "secretKeyRef": {"name": "graph-auth", "key": "password"}}}], - "service": {"port": 9080}, "engineConfig": {"checkpoint": {"interval": 10000}}} - documents = self.documents(values, **{"global": {"imageRegistry": "registry.example.com"}}) - master = engine_resources(documents, "Deployment")[0] - spec = pod(master)["spec"] - container = spec["containers"][0] - self.assertEqual("registry.example.com/team/seatunnel@" + digest, container["image"]) + self.assertEqual(services(modes[0]), services(modes[1])) + + def test_dns_discovery_and_services_are_scoped_to_engine_and_release(self): + documents = self.documents({"mode": "cluster"}, namespace="graphs", fullnameOverride="transfer") + workload, = engine_resources(documents, "StatefulSet") + labels = pod(workload)["metadata"]["labels"] + services = engine_resources(documents, "Service") + headless, = [service for service in services if service["spec"].get("clusterIP") == "None"] + api, = [service for service in services if service is not headless] + self.assertEqual("transfer-seatunnel", workload["metadata"]["name"]) + self.assertEqual(headless["metadata"]["name"], workload["spec"]["serviceName"]) + self.assertTrue(headless["spec"]["publishNotReadyAddresses"]) + self.assertIn(8080, [port["port"] for port in api["spec"]["ports"]]) + for service in services: + selector = service["spec"]["selector"] + self.assertTrue(matches(selector, labels)) + self.assertFalse(matches(selector, {**labels, "app.kubernetes.io/instance": "other"})) + for other in documents: + if other["kind"] in ("Deployment", "StatefulSet") and other is not workload: + self.assertFalse(matches(selector, pod(other)["metadata"]["labels"])) + data = engine_resources(documents, "ConfigMap")[0]["data"] + discovery = yaml.safe_load(data["hazelcast.yaml"])["hazelcast"]["network"]["join"]["kubernetes"] + self.assertTrue(discovery["enabled"]) + self.assertTrue(discovery["service-dns"].startswith(headless["metadata"]["name"] + ".graphs.svc")) + self.assertNotIn("service-name", discovery) + client = yaml.safe_load(data["hazelcast-client.yaml"])["hazelcast-client"] + self.assertIn(discovery["service-dns"] + ":5801", client["network"]["cluster-members"]) + self.assertFalse(any(doc["kind"] in ("Role", "RoleBinding") for doc in engine_resources(documents))) + + def test_image_secret_environment_and_native_file_overrides_reach_engine(self): + custom = {"jvm_options": "-Xms128m\n-Xmx384m\n", "log4j2.properties": "status=warn\n"} + secret_env = {"name": "HG_PASSWORD", "valueFrom": {"secretKeyRef": {"name": "graph-auth", "key": "password"}}} + documents = self.documents({"image": {"repository": "team/seatunnel", "tag": "tested", "pullPolicy": "Never"}, + "env": [secret_env], "config": custom}, + **{"global": {"imageRegistry": "registry.example.com"}, + "imagePullSecrets": [{"name": "registry-auth"}]}) + workload, = engine_resources(documents, "StatefulSet") + spec = pod(workload)["spec"] + container, = spec["containers"] + self.assertEqual("registry.example.com/team/seatunnel:tested", container["image"]) + self.assertEqual("Never", container["imagePullPolicy"]) self.assertEqual([{"name": "registry-auth"}], spec["imagePullSecrets"]) - self.assertEqual(values["env"], container["env"]) + self.assertIn(secret_env, container["env"]) data = engine_resources(documents, "ConfigMap")[0]["data"] - engine = yaml.safe_load(data["seatunnel.yaml"])["seatunnel"]["engine"] - self.assertEqual(9080, engine["http"]["port"]) - self.assertEqual(10000, engine["checkpoint"]["interval"]) - previous = engine_resources(self.documents({}), "Deployment")[0] + mounts = {entry["subPath"] for entry in container["volumeMounts"]} + for filename, content in custom.items(): + self.assertEqual(content, data[filename]) + self.assertIn(filename, mounts) + previous, = engine_resources(self.documents({}), "StatefulSet") self.assertNotEqual(pod(previous)["metadata"]["annotations"]["checksum/config"], - pod(master)["metadata"]["annotations"]["checksum/config"]) - documents = self.documents({"image": {"registry": "private.example.com/engine", "tag": "tested"}}, - **{"global": {"imageRegistry": "registry.example.com"}}) - image = pod(engine_resources(documents, "Deployment")[0])["spec"]["containers"][0]["image"] - self.assertEqual("private.example.com/engine:tested", image) - - def test_invalid_enabled_values_fail_before_install(self): - for values in ({"mode": "invalid"}, {"master": {"replicas": 2}}, - {"mode": "cluster", "master": {"replicas": 0}}, - {"mode": "cluster", "worker": {"replicas": 0}}, - {"service": {"port": 5801}}, - {"configMap": {"create": False, "existingConfigMapName": ""}}): + pod(workload)["metadata"]["annotations"]["checksum/config"]) + + def test_invalid_enabled_mode_and_cluster_size_fail_before_install(self): + for values in ({"mode": "invalid"}, {"mode": "cluster", "replicas": 0}, + {"mode": "cluster", "replicas": 1}): with self.subTest(values=values): self.assertNotEqual(0, self.render(values).returncode) - documents = self.documents({"configMap": {"create": False, "existingConfigMapName": "engine-config"}}) - self.assertEqual([], engine_resources(documents, "ConfigMap")) - spec = pod(engine_resources(documents, "Deployment")[0])["spec"] - self.assertEqual("engine-config", spec["volumes"][0]["configMap"]["name"]) - documents = self.documents({"mode": "cluster", "master": {"strategy": {"type": "Recreate"}}, - "worker": {"strategy": {"type": "Recreate"}}}) - for workload in engine_resources(documents, "Deployment"): - self.assertEqual({"type": "Recreate"}, workload["spec"]["strategy"]) - - def test_strict_policies_allow_peers_server_dns_and_native_extensions(self): - extra_in = {"from": [{"ipBlock": {"cidr": "10.10.0.0/16"}}], "ports": [{"port": 9080}]} + + def test_strict_policies_allow_engine_server_dns_and_native_extensions(self): + extra_in = {"from": [{"ipBlock": {"cidr": "10.10.0.0/16"}}], "ports": [{"port": 8080}]} extra_out = {"to": [{"ipBlock": {"cidr": "10.20.0.0/16"}}], "ports": [{"port": 9099}]} - documents = self.documents({"mode": "cluster", "service": {"port": 9080}}, - mode="strict", server={"port": 8182}, - networkPolicy={"seatunnel": {"extraIngress": [extra_in], - "extraEgress": [extra_out]}}) + documents = self.documents({"mode": "cluster"}, mode="strict", server={"port": 8182}, + networkPolicy={"seatunnel": {"extraIngress": [extra_in], "extraEgress": [extra_out]}}) policies = [doc for doc in documents if doc["kind"] == "NetworkPolicy"] engine = next(doc["spec"] for doc in policies if doc["metadata"]["name"].endswith("-seatunnel")) server = next(doc["spec"] for doc in policies if doc["metadata"]["name"].endswith("-server")) - labels = pod(engine_resources(documents, "Deployment")[0])["metadata"]["labels"] - self.assertTrue(matches(engine["podSelector"]["matchLabels"], labels)) + workload, = engine_resources(documents, "StatefulSet") + self.assertTrue(matches(engine["podSelector"]["matchLabels"], pod(workload)["metadata"]["labels"])) self.assertIn(extra_in, engine["ingress"]) self.assertIn(extra_out, engine["egress"]) + def allows(rules, direction, selector, port): return any(peer.get("podSelector", {}).get("matchLabels") == selector and any(entry["port"] == port for entry in rule.get("ports", [])) for rule in rules for peer in rule.get(direction, [])) - engine_selector = engine["podSelector"]["matchLabels"] - self.assertEqual("fixture", engine_selector["app.kubernetes.io/instance"]) - self.assertTrue(allows(server["ingress"], "from", engine_selector, 8182)) + + selector = engine["podSelector"]["matchLabels"] + self.assertTrue(allows(server["ingress"], "from", selector, 8182)) self.assertTrue(allows(engine["egress"], "to", server["podSelector"]["matchLabels"], 8182)) for direction, peer_key in (("ingress", "from"), ("egress", "to")): - self.assertTrue(allows(engine[direction], peer_key, engine_selector, 5801)) - self.assertTrue(allows(engine[direction], peer_key, engine_selector, 9080)) + for port in (5801, 8080): + self.assertTrue(allows(engine[direction], peer_key, selector, port)) dns = [port for rule in engine["egress"] if not rule.get("to") for port in rule["ports"]] self.assertTrue({("UDP", 53), ("TCP", 53)} <= {(p["protocol"], p["port"]) for p in dns}) - def test_package_preserves_dependency_configuration_and_all_modes(self): + def test_packaged_chart_has_both_modes_without_seatunnel_dependency(self): with tempfile.TemporaryDirectory(prefix="seatunnel-package-") as directory: subprocess.run(["helm", "package", str(CHART), "--destination", directory], check=True, capture_output=True, text=True) package = next(Path(directory).glob("*.tgz")) with tarfile.open(package, "r:gz") as archive: - parent = yaml.safe_load(archive.extractfile("hugegraph-helm/Chart.yaml")) - dependency = next(d for d in parent["dependencies"] if d.get("alias") == "seatunnel") - self.assertEqual("seatunnel.enabled", dependency["condition"]) - filename = f'{dependency["name"]}-{dependency["version"]}.tgz' - child = CHART / "charts" / filename - # HugeGraph ships one dependency archive; canonical templates - # belong to the SeaTunnel repository rather than a source copy. - self.assertEqual([child], list((CHART / "charts").iterdir())) - with tarfile.open(child, "r:gz") as source: - for member in source.getmembers(): - relative = member.name.split("/", 1)[-1] - if relative == "values.schema.json" or relative.startswith(("templates/", "conf/")): - name = "hugegraph-helm/charts/" + member.name - self.assertEqual(source.extractfile(member).read(), archive.extractfile(name).read(), name) + metadata = yaml.safe_load(archive.extractfile("hugegraph-helm/Chart.yaml")) + self.assertFalse(any("seatunnel" in dependency["name"] + for dependency in metadata.get("dependencies", []))) + self.assertTrue(any(name.endswith("/templates/seatunnel.yaml") for name in archive.getnames())) self.assertEqual([], engine_resources(self.documents(chart=package))) - for mode, count in (("standalone", 1), ("cluster", 2)): + for mode, replicas in (("standalone", 1), ("cluster", 2)): documents = self.documents({"mode": mode}, chart=package) - self.assertEqual(count, len(engine_resources(documents, "Deployment"))) + workload, = engine_resources(documents, "StatefulSet") + self.assertEqual(replicas, workload["spec"]["replicas"]) + source, = engine_resources(self.documents({"mode": mode}), "StatefulSet") + self.assertEqual(source["spec"], workload["spec"]) if __name__ == "__main__": diff --git a/pom.xml b/pom.xml index 89b2cf6c6a..927995611b 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,6 @@ **/pd/grpc/**/*.java **/store/grpc/**/*.java **/target/** - helm/**/Chart.lock **/apache-hugegraph-*/lib/arthas/logback.xml dist/**/* @@ -440,8 +439,6 @@ **/*.txt - **/*.tgz - **/__pycache__/** **/.flattened-pom.xml **/apache-hugegraph-*/**/* From b816dae2dc36ba2440acf533bc6863a20c97ecc9 Mon Sep 17 00:00:00 2001 From: imbajin Date: Sat, 24 Oct 2026 00:10:00 +0800 Subject: [PATCH 06/17] test(helm): bound SeaTunnel Helm subprocesses --- helm/test_helm_seatunnel.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/helm/test_helm_seatunnel.py b/helm/test_helm_seatunnel.py index 5858203546..147d688a94 100644 --- a/helm/test_helm_seatunnel.py +++ b/helm/test_helm_seatunnel.py @@ -27,6 +27,8 @@ CHART = Path(__file__).resolve().parent / "hugegraph" +HELM_TEMPLATE_TIMEOUT_SECONDS = 120 +HELM_PACKAGE_TIMEOUT_SECONDS = 300 def matches(selector, labels): @@ -48,15 +50,23 @@ def setUpClass(cls): if shutil.which("helm") is None: raise RuntimeError("Helm is required for the SeaTunnel integration regression") + def run_helm(self, arguments, timeout, check=False): + command = ["helm", *arguments] + try: + return subprocess.run(command, check=check, capture_output=True, + text=True, timeout=timeout) + except subprocess.TimeoutExpired as error: + self.fail(f"{' '.join(command)} timed out after {timeout} seconds: {error}") + def render(self, seatunnel=None, chart=CHART, namespace="default", **values): if seatunnel is not None: values["seatunnel"] = {"enabled": True, **seatunnel} with tempfile.TemporaryDirectory(prefix="seatunnel-values-") as directory: configuration = Path(directory) / "values.yaml" configuration.write_text(yaml.safe_dump(values)) - result = subprocess.run(["helm", "template", "fixture", str(chart), - "--namespace", namespace, "-f", str(configuration)], - capture_output=True, text=True) + result = self.run_helm(["template", "fixture", str(chart), + "--namespace", namespace, "-f", str(configuration)], + HELM_TEMPLATE_TIMEOUT_SECONDS) return result def documents(self, seatunnel=None, chart=CHART, namespace="default", **values): @@ -177,8 +187,8 @@ def allows(rules, direction, selector, port): def test_packaged_chart_has_both_modes_without_seatunnel_dependency(self): with tempfile.TemporaryDirectory(prefix="seatunnel-package-") as directory: - subprocess.run(["helm", "package", str(CHART), "--destination", directory], - check=True, capture_output=True, text=True) + self.run_helm(["package", str(CHART), "--destination", directory], + HELM_PACKAGE_TIMEOUT_SECONDS, check=True) package = next(Path(directory).glob("*.tgz")) with tarfile.open(package, "r:gz") as archive: metadata = yaml.safe_load(archive.extractfile("hugegraph-helm/Chart.yaml")) From fc960bdef82e2c323285c4ecc7a858694edf3393 Mon Sep 17 00:00:00 2001 From: orion Date: Fri, 23 Oct 2026 16:18:51 +0000 Subject: [PATCH 07/17] fix(helm): bind SeaTunnel DNS discovery to member port --- helm/hugegraph/templates/seatunnel.yaml | 2 +- helm/test_helm_seatunnel.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/helm/hugegraph/templates/seatunnel.yaml b/helm/hugegraph/templates/seatunnel.yaml index 924bab90b2..73b1934028 100644 --- a/helm/hugegraph/templates/seatunnel.yaml +++ b/helm/hugegraph/templates/seatunnel.yaml @@ -27,7 +27,7 @@ Use its DNS discovery alternative, so Kubernetes API RBAC is unnecessary. "cluster-name" $name "network" (dict "port" (dict "auto-increment" false "port" 5801) - "join" (dict "kubernetes" (dict "enabled" true "service-dns" $dns "service-dns-timeout" 10))) + "join" (dict "kubernetes" (dict "enabled" true "service-dns" $dns "service-port" 5801 "service-dns-timeout" 10))) "properties" (dict "hazelcast.logging.type" "log4j2")) -}} {{- $_ := set $data "hazelcast.yaml" ($hazelcast | toYaml) -}} {{- $client := dict "hazelcast-client" (dict diff --git a/helm/test_helm_seatunnel.py b/helm/test_helm_seatunnel.py index 147d688a94..42823efe41 100644 --- a/helm/test_helm_seatunnel.py +++ b/helm/test_helm_seatunnel.py @@ -125,6 +125,12 @@ def test_dns_discovery_and_services_are_scoped_to_engine_and_release(self): self.assertTrue(discovery["enabled"]) self.assertTrue(discovery["service-dns"].startswith(headless["metadata"]["name"] + ".graphs.svc")) self.assertNotIn("service-name", discovery) + # DNS discovery otherwise dials Hazelcast's default5701 and creates + # isolated one-member clusters even while both Pods report Ready. + member_port = next(port["containerPort"] for port in + pod(workload)["spec"]["containers"][0]["ports"] + if port["name"] == "hazelcast") + self.assertEqual(member_port, discovery["service-port"]) client = yaml.safe_load(data["hazelcast-client.yaml"])["hazelcast-client"] self.assertIn(discovery["service-dns"] + ":5801", client["network"]["cluster-members"]) self.assertFalse(any(doc["kind"] in ("Role", "RoleBinding") for doc in engine_resources(documents))) From be55d187327a208e778b389ae9b86e7e85eafb28 Mon Sep 17 00:00:00 2001 From: orion Date: Fri, 23 Oct 2026 16:24:21 +0000 Subject: [PATCH 08/17] docs(helm): record native hybrid clone and UI validation --- helm/hugegraph/docs/seatunnel.md | 47 ++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/helm/hugegraph/docs/seatunnel.md b/helm/hugegraph/docs/seatunnel.md index 0c44b01a9f..6470b76bc4 100644 --- a/helm/hugegraph/docs/seatunnel.md +++ b/helm/hugegraph/docs/seatunnel.md @@ -22,13 +22,13 @@ HugeGraph 与 SeaTunnel 的拓扑独立选择；HugeGraph 的 `mode: easy/strict helm install hugegraph ./helm/hugegraph -n hugegraph --create-namespace \ -f helm/hugegraph/values-single.yaml \ -f helm/hugegraph/examples/values-seatunnel-standalone.yaml \ - --wait --timeout 15m + --wait --timeout 30m # 同样的 HugeGraph 拓扑，加两个 SeaTunnel hybrid 节点 helm install hugegraph ./helm/hugegraph -n hugegraph --create-namespace \ -f helm/hugegraph/values-single.yaml \ -f helm/hugegraph/examples/values-seatunnel-cluster.yaml \ - --wait --timeout 15m + --wait --timeout 30m ``` 默认使用 `apache/seatunnel:3.0.0`。两种模式都开启引擎自带的 HTTP 服务和任务可视化前端，无需另部署 Web 服务或数据库： @@ -46,14 +46,14 @@ release 为 `hugegraph` 时，资源为 `hugegraph-seatunnel` StatefulSet、同 ```bash helm upgrade hugegraph ./helm/hugegraph -n hugegraph -f my-values.yaml \ --set seatunnel.enabled=true --set seatunnel.mode=cluster \ - --set seatunnel.replicas=2 --wait --timeout 15m + --set seatunnel.replicas=2 --wait --timeout 30m ``` 切回单机设置 `seatunnel.mode=standalone`。两种模式使用同一个 StatefulSet，但副本和引擎配置会变化；不承诺无中断切换，也不会自动恢复或重新提交作业。设置 `seatunnel.enabled=false` 移除本 release 的 SeaTunnel，已经写入 HugeGraph 的数据保留。 ## 镜像、凭据与原生配置 -优先复用官方镜像或已有镜像，需确认其包含版本匹配的 HugeGraph 连接器及依赖。镜像通过 `seatunnel.image.repository/tag/pullPolicy` 选择，继承 HugeGraph 的 `global.imageRegistry` 和 `imagePullSecrets`；Helm 不在启动时下载插件。 +本轮已直接核实官方 `apache/seatunnel:3.0.0` 镜像自带 HugeGraph、FakeSource、Console 连接器和内置 UI，直接复用即可，无需维护派生镜像。更换版本或自定义镜像时再核对插件与依赖。镜像通过 `seatunnel.image.repository/tag/pullPolicy` 选择，继承 HugeGraph 的 `global.imageRegistry` 和 `imagePullSecrets`；Helm 不在启动时下载插件。官方镜像包含大量连接器，本次 amd64 镜像压缩后约 3.7GB；首次拉取较慢时可延长安装 timeout，或预拉取、镜像到内部仓库。 凭据使用原生 Kubernetes Secret 引用。例如，本地验证可直接引用默认 Helm 管理员 Secret，无需复制密码： @@ -141,7 +141,7 @@ for clone_job in person company works_at knows; do done ``` -作业使用 `ERROR_WHEN_SCHEMA_NOT_EXIST`，复用已经复制的 Schema；边依赖先前写入的端点顶点。这组映射覆盖 [person](../examples/seatunnel/clone/person.conf)、[company](../examples/seatunnel/clone/company.conf)、[works_at](../examples/seatunnel/clone/works_at.conf) 和 [knows](../examples/seatunnel/clone/knows.conf)。复制其他图时需按实际标签、属性和 ID 策略调整映射，不会自动枚举全图并提交迁移作业。 +作业使用 `ERROR_WHEN_SCHEMA_NOT_EXIST`，复用已经复制的 Schema；边依赖先前写入的端点顶点。这组映射覆盖 [person](../examples/seatunnel/clone/person.conf)、[company](../examples/seatunnel/clone/company.conf)、[works_at](../examples/seatunnel/clone/works_at.conf) 和 [knows](../examples/seatunnel/clone/knows.conf)。官方 Sink 要求显式提供 `mappings` 或 `schema_config`。复制其他图时需按实际标签、属性和 ID 策略调整原生映射，不会自动枚举全图并提交迁移作业。 这组示例验证 `PRIMARY_KEY` 和 `CUSTOMIZE_NUMBER` 顶点 ID 策略。现有连接器对 `AUTOMATIC` 顶点会生成新 ID，不能用这组映射直接无损复制其边端点；迁移前先核对源图的 ID 策略。 @@ -149,4 +149,39 @@ done ## 本轮实测记录 -待填写：使用上述原生 hybrid 实现，在 HugeGraph Helm 1 PD + 1 Store + 1 Server/HStore 环境验证单副本、多副本、内置前端，并完成新环境 A → B 的 Schema、1000 个点与 2400 条边复制。旧版分离 master/worker 的实测不作为本实现的验证结果。 +2026 年 10 月 23–24 日（Asia/Singapore）完成以下验证，均使用本页的一套原生 hybrid StatefulSet。源、目标是两个独立的 Helm 环境，各有 **1 PD + 1 Store + 1 Server，后端为 HStore**，分别使用自己的 PVC 和认证。源 `graph_a` 与目标 `graph_b` 不共享后端；另在目标环境的 `graph_a` 测试十条数据直接导入。 + +SeaTunnel 使用原始 `apache/seatunnel:3.0.0` 镜像。实际校验了 OCI 清单、配置及全部九层的大小和 SHA-256，并在容器中确认 HugeGraph、FakeSource、Console 连接器及内置 UI 均存在；无需自建镜像。镜像索引 digest 为 `sha256:914fcf37b2d5acdf9f95f7dbbd7d4c62d150fcbc5ff3e5f59957d65840fea8dd`，实际运行 amd64 清单 digest 为 `sha256:dd8f357dfed2192a68184aeac41507e5529eb06c208b12aae4bf6466ed0a0bb0`。 + +测试环境资源有限，kind 节点使用 FUSE overlay 承载官方镜像原始层；PD、Store、Server 则复用此前从源码构建的缓存运行镜像。以上均为测试基础设施设置，不是 Chart 的部署依赖。HugeGraph 测试运行产物的源码版本为 `f60ad3e3bfc71d4404956fd2dbfb62c7bc002f13`；本轮同时对最新 `master` 基础上的改动完成了编译检查。 + +| 验证项 | 实际结果 | +| --- | --- | +| 直接导入 | 原始 `hugegraph-sink.conf` 未修改，作业 `1166045557131575297` 为 `FINISHED`，读 10、写 10、失败 0；目标图查询及逐条内容核对均为 10 条 | +| Schema 复制 | 空目标先复制 10 个属性键、2 个顶点标签、2 个边标签、3 个索引；保留原 Schema ID，逐项核对类型、基数、ID 策略、nullable、索引和自定义元数据 | +| 单机复制 | 00:08:49 完成，1 个 hybrid 节点；4 个作业全部 `FINISHED`，1000 个顶点、2400 条边的 ID、全部属性和边端点与源图相同 | +| 集群复制 | 00:18:51 完成，2 个 hybrid 节点；人为扰动目标一个属性后重新运行 4 个作业，目标恢复到与源图完全一致；每个 Pod 在作业前后均报告 Workers=2 | +| 切回单机 | 00:21:16 完成只读复查，Workers=1，运行中作业为 0；全部 Schema、1000 个顶点、2400 条边及另图的 10 条导入数据保持一致 | +| 升级影响 | 单机 → 集群 → 单机使用同一个 StatefulSet；两环境原有 6 个 HugeGraph Pod UID、4 个 PVC、6 个认证 Secret 均未变化，Pod 全部 Ready 且重启为 0；匿名访问 401、有效账号 200 | +| 内置前端 | 真实 Chromium（Asia/Singapore）验证两种模式的 Overview、作业列表和 DAG：Workers 分别为 1、2；选中作业的 DAG 均为 2 节点、1 条连接，显示读写各 800；浏览器 JS、关键 API、静态资源错误均为 0 | + +复制作业的逐项结果如下；两种模式每个作业的失败数均为 0。 + +| 标签 | 类型 | 读取 / 写入 | 单机作业 ID | 集群作业 ID | +| --- | --- | --- | --- | --- | +| person | PRIMARY_KEY 顶点 | 800 / 800 | 1166045485111181313 | 1166048035369648130 | +| company | CUSTOMIZE_NUMBER 顶点 | 200 / 200 | 1166045504388202497 | 1166048053988163586 | +| works_at | 边 | 800 / 800 | 1166045519504474113 | 1166048070853459969 | +| knows | MULTIPLE 边 | 1600 / 1600 | 1166045539670687745 | 1166048087110582273 | + +验收使用全量数据比较，单机、集群和切回单机后的 Schema、顶点、边三组规范化哈希分别完全一致。Schema 比较仅忽略元素顶层的运行状态及系统 `user_data.~create_time`，保留业务自定义的 `status/create_time` 元数据。另验证了 SET 内容、LIST 顺序和重复元素、可空属性、自定义元数据、同端点且 sort key 为 2020/2021 的两条多重边；三个索引均为 `CREATED`，范围查询 `age >= 80` 返回 88 条，二级索引查询 `company_123` 返回 1 条，源、目标查询结果相同。 + +集群测试曾发现两个 Pod 虽然 Ready，却各自形成单节点集群。原因是 Hazelcast DNS 发现未指定端口时使用 5701，而引擎监听 5801；模板已明确配置 `service-port: 5801`。修复后逐 Pod 确认 Workers=2、成员数为 2，持续观察一分钟，再重新执行上述集群作业和前端验证。仅 Pod Ready 不作为集群组建成功的依据。 + +本轮还通过了 22 项 Python Helm 回归、309 项 Helm 单元测试（36 个 suite、4 个快照）、两种模式的 strict Helm lint，以及目标环境的 `helm test --logs`。`mvn editorconfig:format` 和 `mvn clean compile -Dmaven.javadoc.skip=true` 的 38 个模块均通过。 + +本次范围与已知限制： + +- 已覆盖 `PRIMARY_KEY`、`CUSTOMIZE_NUMBER`，暂不覆盖 `AUTOMATIC` 顶点的无损复制；后者需要解决新旧顶点 ID 与边端点的对应关系。 +- 官方 3.0.0 UI 的 Duration 显示存在上游问题：无时区时间字符串在新加坡浏览器中会产生 8 小时偏差，且已完成作业的显示时长仍随当前时间增长。以 CLI/REST 的完成状态及开始、结束时间核对实际结果；本次未修改上游 UI 或硬编码时区。 +- 测试覆盖停止作业后的模式升级和数据保留，未验证运行中作业不中断切换、生产 HA、持久化恢复或 CNI 的实际网络隔离。默认临时 checkpoint 目录不能据此视为持久化恢复方案。 From 9f9699e531eb78792725024372fbbfe8fc15b87b Mon Sep 17 00:00:00 2001 From: orion Date: Fri, 23 Oct 2026 17:03:09 +0000 Subject: [PATCH 09/17] feat(helm): secure SeaTunnel integration Follow easy/strict defaults for the HTTP Service and keep discovery internal. Mount native HTTP Basic credentials in an independent kept Secret. Bind local HugeGraph credentials and mount ready source/sink examples. --- helm/hugegraph/README.md | 15 +- helm/hugegraph/docs/seatunnel.md | 132 +++++++++++-- .../examples/seatunnel/hugegraph-sink.conf | 12 +- .../examples/seatunnel/hugegraph-source.conf | 40 ++++ helm/hugegraph/templates/NOTES.txt | 50 ++++- helm/hugegraph/templates/_helpers.tpl | 14 +- .../templates/seatunnel-auth-secret.yaml | 74 +++++++ helm/hugegraph/templates/seatunnel.yaml | 61 +++++- helm/hugegraph/values.schema.json | 17 ++ helm/hugegraph/values.yaml | 16 +- helm/test_helm_seatunnel.py | 182 +++++++++++++++++- 11 files changed, 566 insertions(+), 47 deletions(-) create mode 100644 helm/hugegraph/examples/seatunnel/hugegraph-source.conf create mode 100644 helm/hugegraph/templates/seatunnel-auth-secret.yaml diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index a274868245..406bae362a 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -83,11 +83,16 @@ Inactive credential values may remain in the values file when auth is disabled. ### 可选 SeaTunnel -增加 `--set seatunnel.enabled=true`，与 HugeGraph 一起启动一个 SeaTunnel 混合节点； -增加 `--set seatunnel.mode=cluster`，切换为多个 hybrid 节点（默认两个），复用同一套官方原生 Kubernetes 资源。 -默认关闭，SeaTunnel 拓扑独立于 HugeGraph 的单机/集群预设和 `easy/strict` 访问模式。 -引擎同时提供内置任务可视化前端。启动、凭据、直接导入 Graph 及 Graph A → Graph B 复制示例见 -[SeaTunnel 联合部署指南](docs/seatunnel.md)。 +增加 `--set seatunnel.enabled=true`，与 HugeGraph 一起启动一个常驻 hybrid 节点； +增加 `--set seatunnel.mode=cluster`，使用同一套官方原生 Kubernetes 资源扩为多个节点（默认两个）。 +SeaTunnel 默认关闭，拓扑独立于 HugeGraph 的单机/集群预设。两种模式均提供引擎内置的任务可视化前端； +HTTP 入口继承 `easy` 的 NodePort 或 `strict` 的 ClusterIP/NetworkPolicy，可用 `seatunnel.service` 显式覆盖并指定 NodePort。 + +Chart 自动为同 release 的演示作业注入 Server 地址、图空间和管理员 Secret 引用， +并挂载 Source/Sink 配置；先创建目标图，即可手动提交，无需复制密码或上传作业文件。 +Helm 不自动建图或导入数据。前端和 REST 提交接口使用独立管理员的原生 HTTP Basic 认证， +默认密码自动生成在 SeaTunnel Secret 中并随升级保留；它与作业连接 HugeGraph 的凭据分别配置。 +启动、访问、演示作业与跨环境 Schema／顶点／边复制见 [SeaTunnel 联合部署指南](docs/seatunnel.md)。 ### Internal image registry diff --git a/helm/hugegraph/docs/seatunnel.md b/helm/hugegraph/docs/seatunnel.md index 6470b76bc4..5628a61ed2 100644 --- a/helm/hugegraph/docs/seatunnel.md +++ b/helm/hugegraph/docs/seatunnel.md @@ -11,7 +11,7 @@ SeaTunnel 默认关闭。启用后，一个常驻 hybrid 节点同时承担调 | `seatunnel.mode: standalone` | 一个常驻 hybrid 节点 | | `seatunnel.mode: cluster`、`replicas: 2` | 两个 hybrid 节点，副本数至少为 2 | -HugeGraph 与 SeaTunnel 的拓扑独立选择；HugeGraph 的 `mode: easy/strict` 仍只控制其访问入口和 NetworkPolicy 默认值。 +HugeGraph 与 SeaTunnel 的拓扑独立选择；顶层 `mode: easy/strict` 控制同一 release 的 HTTP 入口和 NetworkPolicy 默认值。SeaTunnel 单机和集群均提供相同的内置前端与 API。 ## 启动、访问前端与切换 @@ -31,13 +31,44 @@ helm install hugegraph ./helm/hugegraph -n hugegraph --create-namespace \ --wait --timeout 30m ``` -默认使用 `apache/seatunnel:3.0.0`。两种模式都开启引擎自带的 HTTP 服务和任务可视化前端，无需另部署 Web 服务或数据库： +默认使用 `apache/seatunnel:3.0.0`。两种模式都开启引擎自带的 HTTP 服务和任务可视化前端，无需另部署 Web 服务或数据库。 + +SeaTunnel HTTP Service 沿用 HugeGraph 的访问模式；显式设置 `seatunnel.service.type` 时优先使用指定值： + +| 配置 | HTTP Service 默认值 | 访问方式 | +|---|---|---| +| `mode: easy` | `NodePort` | 可达节点 IP 加分配的 NodePort | +| `mode: strict` | `ClusterIP`，开启 NetworkPolicy | 集群内访问，或由运维人员临时端口转发 | +| 显式 `seatunnel.service.type` | 使用指定的 `ClusterIP`、`NodePort` 或 `LoadBalancer` | 按实际 Service 地址访问 | + +例如指定 NodePort；不设置 `nodePort` 时由 Kubernetes 分配： + +```yaml +seatunnel: + enabled: true + service: + type: NodePort + nodePort: 30081 + annotations: {} +``` + +```bash +kubectl -n hugegraph get svc hugegraph-seatunnel \ + -o jsonpath='{.spec.ports[?(@.name=="http")].nodePort}{"\n"}' +kubectl get nodes -o wide +``` + +浏览器入口为 `http://<可达节点IP>:/#/overview`，API 使用同一地址的相应 REST 路径。使用 LoadBalancer 时，通过 `kubectl -n hugegraph get svc hugegraph-seatunnel -o wide` 查看分配的地址，再访问 `http://<负载均衡地址>:8080/#/overview`。kind、minikube 等本地环境需按其网络方式提供可达入口。 + +所有 Service 类型均可使用端口转发： ```bash kubectl -n hugegraph port-forward svc/hugegraph-seatunnel 8081:8080 ``` -打开 ，查看集群、作业、DAG 和日志。作业仍通过 SeaTunnel CLI 或 REST API 提交；Helm 只启动引擎，不自动建图或导入数据。 +对应前端地址为 ，使用下节的独立 SeaTunnel 账号完成浏览器 Basic 登录。前端用于查看集群、作业、DAG 和日志；作业通过 SeaTunnel CLI 或受保护的 REST API 提交。Helm 只启动引擎并准备示例配置，不自动建图或执行作业。 + +外部入口只包含 HTTP 8080。Hazelcast 5801 仍仅通过内部 Headless Service 供成员发现及 CLI 通信，不随 NodePort 或 LoadBalancer 一起公开。strict 模式下，额外的应用、监控或入口代理需通过 `networkPolicy.seatunnel.extraIngress` 配置原生规则；本 Chart 不额外管理 SeaTunnel Ingress。 release 为 `hugegraph` 时，资源为 `hugegraph-seatunnel` StatefulSet、同名 HTTP Service、`hugegraph-seatunnel-headless` 发现 Service，以及 `hugegraph-seatunnel-configs` ConfigMap。名称遵循 HugeGraph 的 `fullnameOverride`；成员通过 Headless Service 的 DNS 发现，不访问 Kubernetes API，因此不需要发现用的 RBAC。 @@ -49,27 +80,64 @@ helm upgrade hugegraph ./helm/hugegraph -n hugegraph -f my-values.yaml \ --set seatunnel.replicas=2 --wait --timeout 30m ``` -切回单机设置 `seatunnel.mode=standalone`。两种模式使用同一个 StatefulSet，但副本和引擎配置会变化；不承诺无中断切换，也不会自动恢复或重新提交作业。设置 `seatunnel.enabled=false` 移除本 release 的 SeaTunnel，已经写入 HugeGraph 的数据保留。 +切回单机设置 `seatunnel.mode=standalone`。两种模式使用同一个 StatefulSet，但副本和引擎配置会变化；不承诺无中断切换，也不会自动恢复或重新提交作业。设置 `seatunnel.enabled=false` 移除本 release 的 SeaTunnel 工作负载和 Service，已经写入 HugeGraph 的数据保留。认证 Secret 带有保留策略，禁用或卸载时仍会保留，便于再次启用时复用凭据。 + +## SeaTunnel 入口认证 + +前端、静态资源和 REST 提交接口默认使用引擎原生 HTTP Basic 认证，不新增认证代理或独立登录服务。SeaTunnel 使用独立入口管理员：`seatunnel.auth.username` 默认为 `admin`；密码未显式设置时，Chart 自动生成强密码并保存在 `-auth` Secret，升级时复用已有密码。也可通过受控 values 文件显式设置 `seatunnel.auth.username/password`，用于自定义账号或轮换。自定义值不能有首尾空白或 `${...}` 占位符；内部空格和引号仍可使用。只做离线模板或 GitOps 渲染时，应显式固定 `seatunnel.auth.password`；默认密码的升级复用依赖在线 Helm lookup。 + +默认 release 的 Secret 为 `hugegraph-seatunnel-auth`。保持前述端口转发，在本机读取账号和密码后调用 REST： + +```bash +ST_USERNAME="$(kubectl -n hugegraph get secret hugegraph-seatunnel-auth \ + -o jsonpath='{.data.username}' | base64 --decode)" +ST_PASSWORD="$(kubectl -n hugegraph get secret hugegraph-seatunnel-auth \ + -o jsonpath='{.data.password}' | base64 --decode)" + +curl -fsS --user "$ST_USERNAME:$ST_PASSWORD" http://127.0.0.1:8081/overview +``` + +浏览器打开 `/#/overview` 时，使用 Secret 中的同一组账号密码完成 HTTP Basic 登录；访问其他 NodePort 或 LoadBalancer 地址也使用这组入口凭据。外部提交作业使用同一受保护 HTTP 入口，并按 [官方 REST API](https://github.com/apache/seatunnel/blob/v3.0.0/docs/zh/engines/zeta/rest-api-and-web-ui.md) 的格式携带认证信息。 + +入口 Secret 包含 `username`、`password` 及最终的 `seatunnel.yaml`；包含认证信息的引擎文件直接从 Secret 挂载，不进入 ConfigMap、Java 参数或日志。轮换时在受控 values 中指定新密码并执行 `helm upgrade`，同步更新 Secret 内容并滚动重启；只修改 Secret 的 `data.password` 不会同步修改其中的引擎文件。`HG_USERNAME/HG_PASSWORD` 是作业连接 HugeGraph 的另一组凭据，不会替代入口认证。即使 `server.auth.enabled=false`，SeaTunnel 入口仍使用独立认证。 + +HTTP Basic 不覆盖 Hazelcast 5801 的内部成员和 CLI 协议。下文在 Pod 内执行的 `seatunnel.sh --master cluster` 属于可信 Kubernetes/RBAC 管理范围；外部客户端应使用受保护的 REST 入口，不能把 5801 对外公开。本配置提供供可信管理人员使用的单一管理员入口，不提供按用户或作业划分的细粒度权限，也不复用 HugeGraph 的多用户权限、RBAC 或 SSO。需要对外服务时可接入已有 TLS 入口；本 Chart 不新增 Ingress 管理。 ## 镜像、凭据与原生配置 本轮已直接核实官方 `apache/seatunnel:3.0.0` 镜像自带 HugeGraph、FakeSource、Console 连接器和内置 UI，直接复用即可，无需维护派生镜像。更换版本或自定义镜像时再核对插件与依赖。镜像通过 `seatunnel.image.repository/tag/pullPolicy` 选择，继承 HugeGraph 的 `global.imageRegistry` 和 `imagePullSecrets`；Helm 不在启动时下载插件。官方镜像包含大量连接器，本次 amd64 镜像压缩后约 3.7GB；首次拉取较慢时可延长安装 timeout，或预拉取、镜像到内部仓库。 -凭据使用原生 Kubernetes Secret 引用。例如，本地验证可直接引用默认 Helm 管理员 Secret，无需复制密码： +启用 SeaTunnel 后，Chart 为同 release 的 HugeGraph 演示作业准备以下环境变量，无需手写 Server 地址或复制管理员密码： + +| 变量 | 默认来源 | +|---|---| +| `HG_HOST` | 当前 release 的 HugeGraph Server Service DNS | +| `HG_PORT` | `server.port` | +| `HG_PROTOCOL` | `http`，访问集群内部 Server | +| `HG_USERNAME` | `admin` | +| `HG_PASSWORD` | 当前 Server 管理员 Secret 的实际名称与键；认证关闭时为空密码 | +| `HG_GRAPH_SPACE` | `server.graphSpace` | +| `HG_GRAPH_NAME` | `graph_a` | + +`HG_PASSWORD` 使用原生 `secretKeyRef`，兼容 `server.auth.admin.existingSecret` 及其自定义键，不把密码写入演示文件。Chart 不因此创建额外的 HugeGraph 账号；更新密码 Secret 后，按 Server 账号同步流程处理，再重启相关 SeaTunnel Pod 刷新环境变量。 + +`seatunnel.env` 中同名条目显式覆盖自动值，最终每个变量只有一个有效条目。例如更换演示图，或使用已有的专用作业账号： ```yaml seatunnel: env: + - name: HG_GRAPH_NAME + value: import_demo - name: HG_USERNAME - value: admin + value: seatunnel-job - name: HG_PASSWORD valueFrom: secretKeyRef: - name: hugegraph-admin + name: hugegraph-job-credentials key: password ``` -账号需有作业涉及的图、数据和 Schema 权限。实际作业可改用专用账号。所有 hybrid 节点接收同样的环境变量；更新 Secret 后重启相关 Pod，刷新环境变量。作业文件通过 `${HG_USERNAME}`、`${HG_PASSWORD}` 读取凭据。 +专用账号需预先存在，并具备所用图的数据和 Schema 权限。这组 `HG_*` 默认值只服务于本 release 的简单 Source/Sink 示例；跨环境复制仍由用户提供下文的 `HG_SOURCE_*`、`HG_TARGET_*`，不会自动发现或配置远端图。SeaTunnel 前端/API 自身的认证也不能由这组连接器变量替代。 `seatunnel.javaOpts` 调整引擎 JVM，`resources` 调整容器资源。高级配置保留 SeaTunnel 原生文件格式，通过 `seatunnel.config` 按文件名完整覆盖，不逐项增加 Helm 开关，例如： @@ -82,16 +150,23 @@ seatunnel: -Xmx256m ``` -默认生成 `hazelcast.yaml`、`hazelcast-client.yaml`、`seatunnel.yaml` 和 JVM 文件，其余配置沿用镜像。覆盖发现或引擎文件时保留 TCP 5801、HTTP 8080 和 `enable-http: true`，才能继续使用现有 Service、探针及前端。默认 checkpoint 使用临时本地目录；本集成不提供持久化作业恢复保证。 +默认在 ConfigMap 中准备 `hazelcast.yaml`、`hazelcast-client.yaml`、JVM 和演示作业文件；包含入口认证的 `seatunnel.yaml` 从独立 Secret 挂载，其余配置沿用镜像。`seatunnel.config` 可覆盖引擎文件，但最终 HTTP Basic 开关和凭据始终由 `seatunnel.auth` 管理。覆盖发现或引擎文件时保留 TCP 5801、HTTP 8080 和 `enable-http: true`，才能继续使用现有 Service、探针及前端。默认 checkpoint 使用临时本地目录；本集成不提供持久化作业恢复保证。 strict 模式允许 SeaTunnel 成员通信、DNS，以及访问同 release 的 HugeGraph Server。远端图 A、其他数据源及调用方，通过 `networkPolicy.seatunnel.extraEgress/extraIngress` 补充原生规则；实际隔离依赖 CNI 支持。HugeGraph 的凭据、存储和访问方式见 [Chart README](../README.md)。 -## 简单导入：十条数据写入 Graph A +## 同 release 演示：直接导入并读取 -[hugegraph-sink.conf](../examples/seatunnel/hugegraph-sink.conf) 使用 `FakeSource → HugeGraph`，写入 `seatunnel_demo_person` 标签的十条固定数据。先创建 `DEFAULT/graph_a`；本机 curl 所用的 `HG_USERNAME`、`HG_PASSWORD` 需由凭据管理器提供，Pod 环境变量不会传到本机。 +Chart 启动时把 [hugegraph-sink.conf](../examples/seatunnel/hugegraph-sink.conf) 和 [hugegraph-source.conf](../examples/seatunnel/hugegraph-source.conf) 放入现有的 SeaTunnel ConfigMap，分别挂载为： + +- `/opt/seatunnel/config/hugegraph-sink.conf`：`FakeSource → HugeGraph`，向 `seatunnel_demo_person` 标签写入十条固定数据。 +- `/opt/seatunnel/config/hugegraph-source.conf`：读取同一图中的演示标签，输出到 Console。 + +两个文件均使用前述自动 `HG_*` 环境变量；无需上传临时文件或手动补全当前 Server 地址、端口、图空间及密码。需要调整作业时，可在 `seatunnel.config` 中按同名文件完整覆盖。 + +先通过 Hubble 或 HugeGraph API 创建 `graph_a`。如果改写了 `HG_GRAPH_SPACE/HG_GRAPH_NAME`，则创建对应图。以下命令按默认 release、命名空间及图空间演示建图；本机的 `HG_USERNAME/HG_PASSWORD` 由凭据管理器提供，Pod 环境变量不会传到本机： ```bash -# 在另一个终端保持转发 +# 在另一个终端保持 Server 端口转发 kubectl -n hugegraph port-forward svc/hugegraph-server 8082:8080 ``` @@ -99,18 +174,31 @@ kubectl -n hugegraph port-forward svc/hugegraph-server 8082:8080 export HG_URL=http://127.0.0.1:8082 curl -fsS -u "$HG_USERNAME:$HG_PASSWORD" -H 'Content-Type: application/json' \ -X POST "$HG_URL/graphspaces/DEFAULT/graphs/graph_a" -d '{}' +``` + +已有图不重复建图。随后直接提交已挂载的 Sink，再提交 Source： + +```bash +kubectl -n hugegraph exec sts/hugegraph-seatunnel -- \ + /opt/seatunnel/bin/seatunnel.sh --master cluster \ + --config /opt/seatunnel/config/hugegraph-sink.conf -kubectl -n hugegraph exec -i sts/hugegraph-seatunnel -- \ - sh -c 'cat > /tmp/hugegraph-sink.conf' \ - < helm/hugegraph/examples/seatunnel/hugegraph-sink.conf kubectl -n hugegraph exec sts/hugegraph-seatunnel -- \ - /opt/seatunnel/bin/seatunnel.sh --master cluster --config /tmp/hugegraph-sink.conf + /opt/seatunnel/bin/seatunnel.sh --master cluster \ + --config /opt/seatunnel/config/hugegraph-source.conf +``` + +`--master cluster` 表示提交给常驻引擎，单节点也使用该参数。首次空图导入后，Sink 作业应为 `FINISHED`，读取、写入各十条；Source 应读取十条并输出到执行节点的日志。可先在前端查看作业状态和 DAG，再检查各 hybrid Pod 日志及图中实际数据： + +```bash +kubectl -n hugegraph logs -l app.kubernetes.io/instance=hugegraph,app.kubernetes.io/component=seatunnel \ + --all-containers --prefix --tail=200 curl -fsS -u "$HG_USERNAME:$HG_PASSWORD" -H 'Content-Type: application/json' \ "$HG_URL/gremlin" -d '{"gremlin":"graph.traversal().V().hasLabel(\"seatunnel_demo_person\").count()","aliases":{"graph":"DEFAULT-graph_a"}}' ``` -已有图不重复建图。首次空图导入后，作业应为 `FINISHED`，读取、写入各十条，Gremlin 的 `result.data` 为 `[10]`。`--master cluster` 表示提交给常驻引擎，单节点也使用该参数。 +数量查询的 `result.data` 应为 `[10]`。集群模式下作业可能由任一 hybrid 节点执行，因此 Console 输出不限定在 `hugegraph-seatunnel-0`。这两个演示作业也不会在 Helm 安装或升级时自动执行。 ## 新环境复制：Schema、顶点和边 @@ -147,7 +235,7 @@ done 验收应同时核对属性键类型、基数、标签、主键、边端点定义、索引和 Schema ID，以及全部顶点 ID/属性、边 ID/端点/属性；只比较数量或作业状态不够。相关选项见官方 [HugeGraph Source](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/source/HugeGraph.md) 和 [Sink](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/sink/HugeGraph.md)。 -## 本轮实测记录 +## 已完成实测记录：原生 hybrid 基线 2026 年 10 月 23–24 日（Asia/Singapore）完成以下验证，均使用本页的一套原生 hybrid StatefulSet。源、目标是两个独立的 Helm 环境，各有 **1 PD + 1 Store + 1 Server，后端为 HStore**，分别使用自己的 PVC 和认证。源 `graph_a` 与目标 `graph_b` 不共享后端；另在目标环境的 `graph_a` 测试十条数据直接导入。 @@ -178,10 +266,16 @@ SeaTunnel 使用原始 `apache/seatunnel:3.0.0` 镜像。实际校验了 OCI 清 集群测试曾发现两个 Pod 虽然 Ready，却各自形成单节点集群。原因是 Hazelcast DNS 发现未指定端口时使用 5701，而引擎监听 5801；模板已明确配置 `service-port: 5801`。修复后逐 Pod 确认 Workers=2、成员数为 2，持续观察一分钟，再重新执行上述集群作业和前端验证。仅 Pod Ready 不作为集群组建成功的依据。 -本轮还通过了 22 项 Python Helm 回归、309 项 Helm 单元测试（36 个 suite、4 个快照）、两种模式的 strict Helm lint，以及目标环境的 `helm test --logs`。`mvn editorconfig:format` 和 `mvn clean compile -Dmaven.javadoc.skip=true` 的 38 个模块均通过。 +上述基线还通过了 22 项 Python Helm 回归、309 项 Helm 单元测试（36 个 suite、4 个快照）、两种模式的 strict Helm lint，以及目标环境的 `helm test --logs`。`mvn editorconfig:format` 和 `mvn clean compile -Dmaven.javadoc.skip=true` 的 38 个模块均通过。 本次范围与已知限制： - 已覆盖 `PRIMARY_KEY`、`CUSTOMIZE_NUMBER`，暂不覆盖 `AUTOMATIC` 顶点的无损复制；后者需要解决新旧顶点 ID 与边端点的对应关系。 - 官方 3.0.0 UI 的 Duration 显示存在上游问题：无时区时间字符串在新加坡浏览器中会产生 8 小时偏差，且已完成作业的显示时长仍随当前时间增长。以 CLI/REST 的完成状态及开始、结束时间核对实际结果；本次未修改上游 UI 或硬编码时区。 - 测试覆盖停止作业后的模式升级和数据保留，未验证运行中作业不中断切换、生产 HA、持久化恢复或 CNI 的实际网络隔离。默认临时 checkpoint 目录不能据此视为持久化恢复方案。 + +## 本轮增量验证记录 + +本轮新增 SeaTunnel HTTP 入口继承 easy/strict、独立原生 HTTP Basic 认证、同 release 的 `HG_*` 默认连接信息，以及启动时挂载 Source/Sink 演示文件。上一节的原生 hybrid 基线记录验证了引擎、跨环境 Schema／点／边复制及内置 UI，不作为这些新增行为的运行验证结论。 + +本轮增量实测结果将在完成后补充。 diff --git a/helm/hugegraph/examples/seatunnel/hugegraph-sink.conf b/helm/hugegraph/examples/seatunnel/hugegraph-sink.conf index c847ced697..fbfb685b1f 100644 --- a/helm/hugegraph/examples/seatunnel/hugegraph-sink.conf +++ b/helm/hugegraph/examples/seatunnel/hugegraph-sink.conf @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# 先在 DEFAULT 图空间创建 graph_a。此示例创建演示标签并导入十条固定数据。 +# 先创建 HG_GRAPH_SPACE/HG_GRAPH_NAME 指定的图，默认 DEFAULT/graph_a。 +# Helm 自动配置同 release 的 HugeGraph 地址与 Secret 凭据。 env { job.mode = "BATCH" parallelism = 1 @@ -45,10 +46,11 @@ source { sink { HugeGraph { - host = "hugegraph-server" - port = 8080 - graph_space = "DEFAULT" - graph_name = "graph_a" + host = ${HG_HOST} + port = ${HG_PORT} + protocol = ${HG_PROTOCOL} + graph_space = ${HG_GRAPH_SPACE} + graph_name = ${HG_GRAPH_NAME} username = ${HG_USERNAME} password = ${HG_PASSWORD} schema_save_mode = "CREATE_SCHEMA_WHEN_NOT_EXIST" diff --git a/helm/hugegraph/examples/seatunnel/hugegraph-source.conf b/helm/hugegraph/examples/seatunnel/hugegraph-source.conf new file mode 100644 index 0000000000..622e5effa6 --- /dev/null +++ b/helm/hugegraph/examples/seatunnel/hugegraph-source.conf @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 先运行 hugegraph-sink.conf，再读取同一图中的十条演示顶点。 +env { + job.mode = BATCH + parallelism = 1 +} + +source { + HugeGraph { + host = ${HG_HOST} + port = ${HG_PORT} + protocol = ${HG_PROTOCOL} + graph_space = ${HG_GRAPH_SPACE} + graph_name = ${HG_GRAPH_NAME} + username = ${HG_USERNAME} + password = ${HG_PASSWORD} + label = seatunnel_demo_person + label_type = VERTEX + page_size = 100 + time_zone = UTC + } +} + +sink { + Console {} +} diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt index 2197451374..0db01562bc 100644 --- a/helm/hugegraph/templates/NOTES.txt +++ b/helm/hugegraph/templates/NOTES.txt @@ -62,10 +62,52 @@ Access: {{- if (.Values.seatunnel | default dict).enabled }} SeaTunnel: - In-cluster API: http://{{ include "hugegraph.seatunnel.name" . }}.{{ .Release.Namespace }}.svc:8080 - Port-forward: kubectl port-forward -n {{ .Release.Namespace }} svc/{{ include "hugegraph.seatunnel.name" . }} 8081:8080 - Web UI: http://localhost:8081/#/overview - Connector images, credentials and checkpoint storage: helm/hugegraph/docs/seatunnel.md +{{- $stName := include "hugegraph.seatunnel.name" . }} +{{- $stService := .Values.seatunnel.service | default dict }} + 集群内 API: http://{{ $stName }}.{{ .Release.Namespace }}.svc:8080 +{{- if eq $stService.type "NodePort" }} + HTTP NodePort: + kubectl get svc -n {{ .Release.Namespace }} {{ $stName }} -o jsonpath='{.spec.ports[?(@.name=="http")].nodePort}{"\n"}' + kubectl get nodes -o wide + 前端: http://<可达节点IP>:{{ $stService.nodePort | default "" }}/#/overview + API: http://<可达节点IP>:{{ $stService.nodePort | default "" }} +{{- else if eq $stService.type "LoadBalancer" }} + 负载均衡地址: + kubectl get svc -n {{ .Release.Namespace }} {{ $stName }} -o wide + kubectl get svc -n {{ .Release.Namespace }} {{ $stName }} -o jsonpath='{.status.loadBalancer.ingress}{"\n"}' + 地址分配后访问前端 http://<负载均衡地址>:8080/#/overview，API 使用同一 HTTP 地址。 +{{- else }} + HTTP Service 为 ClusterIP；集群外访问可使用下方端口转发。 +{{- end }} + 所有 Service 类型均可端口转发: + kubectl port-forward -n {{ .Release.Namespace }} svc/{{ $stName }} 8081:8080 + 本机前端: http://127.0.0.1:8081/#/overview + 本机 API: http://127.0.0.1:8081 + Hazelcast 5801 仅用于集群内部发现及 CLI 通信，不通过以上外部入口公开。 + 前端、静态资源和 REST 提交接口使用独立 SeaTunnel 账号的 HTTP Basic 认证。 + 以下命令读取凭据到本机变量，不在安装输出中显示密码: + ST_USERNAME="$(kubectl get secret -n {{ .Release.Namespace }} {{ $stName }}-auth -o jsonpath='{.data.username}' | base64 --decode)" + ST_PASSWORD="$(kubectl get secret -n {{ .Release.Namespace }} {{ $stName }}-auth -o jsonpath='{.data.password}' | base64 --decode)" + 浏览器 Basic 登录使用此 Secret 中的账号密码；保持端口转发后可验证 REST: + curl --user "${ST_USERNAME}:${ST_PASSWORD}" http://127.0.0.1:8081/overview + 该入口账号独立于 HugeGraph 连接器账号；关闭 HugeGraph 认证也不会关闭 SeaTunnel 入口认证。 + 内部 CLI/5801 不经 HTTP Basic，仅在可信 Kubernetes/RBAC 范围使用，外部提交走受保护的 REST。 + 默认凭据随在线 Helm 升级保留；轮换用受控 values 的 seatunnel.auth.password，再 helm upgrade。 + GitOps/template-only 渲染请显式固定密码；认证 Secret 在禁用或卸载后仍保留。 + 账号、轮换及作业说明: helm/hugegraph/docs/seatunnel.md +{{- if and .Values.networkPolicy.enabled (has $stService.type (list "NodePort" "LoadBalancer")) }} +{{- $stRules := get .Values.networkPolicy "seatunnel" | default dict }} +{{- if empty (get $stRules "extraIngress") }} + NetworkPolicy 已开启；外部 HTTP 调用方还需通过 networkPolicy.seatunnel.extraIngress 允许。 +{{- end }} +{{- end }} + + 同 release 演示作业: + 已注入 Server 地址、端口、图空间及凭据引用，示例配置已挂载，无需上传临时文件。 + 先创建目标图（默认 {{ .Values.server.graphSpace }}/graph_a；HG_* 显式覆盖时以覆盖值为准），再手动提交: + kubectl exec -n {{ .Release.Namespace }} sts/{{ $stName }} -- /opt/seatunnel/bin/seatunnel.sh --master cluster --config /opt/seatunnel/config/hugegraph-sink.conf + kubectl exec -n {{ .Release.Namespace }} sts/{{ $stName }} -- /opt/seatunnel/bin/seatunnel.sh --master cluster --config /opt/seatunnel/config/hugegraph-source.conf + Helm 不自动建图或执行作业；跨环境 clone 仍需用户提供 HG_SOURCE_* / HG_TARGET_*。 {{- end }} {{- range $component := list "server" "hubble" "pd" }} {{- if or (ne $component "hubble") $hubbleEnabled }} diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index 28bced02f1..d849f62a2a 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -25,7 +25,7 @@ {{- end -}} {{- $strict := eq (.Values.mode | default "easy") "strict" -}} {{- $serviceType := ternary "ClusterIP" "NodePort" (or $strict $legacy) -}} -{{- range $comp := list "server" "hubble" -}} +{{- range $comp := list "server" "hubble" "seatunnel" -}} {{- $component := get $.Values $comp -}} {{- $svc := get $component "service" | default dict -}} {{- $_ := set $component "service" $svc -}} @@ -743,6 +743,18 @@ Cross-field validation that JSON Schema draft-07 cannot express. {{- end -}} {{- include "hugegraph.resolveValues" . -}} +{{- if (.Values.seatunnel | default dict).enabled -}} +{{- $svc := .Values.seatunnel.service -}} +{{- if and (get $svc "nodePort") (not (has $svc.type (list "NodePort" "LoadBalancer"))) -}} +{{- fail "seatunnel.service.nodePort requires seatunnel.service.type to be NodePort or LoadBalancer" -}} +{{- end -}} +{{- range $key := list "username" "password" -}} +{{- $value := get $.Values.seatunnel.auth $key | default "" -}} +{{- if or (ne $value (trim $value)) (contains "${" $value) -}} +{{- fail (printf "seatunnel.auth.%s cannot contain ${...} placeholders or leading/trailing whitespace: native Engine YAML expands or trims them" $key) -}} +{{- end -}} +{{- end -}} +{{- end -}} {{- $startupBudget := include "hugegraph.server.startupProbeBudget" . | int -}} {{- if lt $startupBudget (add (int .Values.server.storageWaitTimeoutSeconds) 120) -}} {{- fail "server.probes.startup.failureThreshold and periodSeconds must leave at least server.storageWaitTimeoutSeconds + 120 seconds to wait for storage and start Server; increase the probe budget or reduce server.storageWaitTimeoutSeconds" -}} diff --git a/helm/hugegraph/templates/seatunnel-auth-secret.yaml b/helm/hugegraph/templates/seatunnel-auth-secret.yaml new file mode 100644 index 0000000000..918f1a5ac9 --- /dev/null +++ b/helm/hugegraph/templates/seatunnel-auth-secret.yaml @@ -0,0 +1,74 @@ +{{/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "hugegraph.seatunnel.authSecretName" -}} +{{- printf "%s-auth" (include "hugegraph.seatunnel.name" .) -}} +{{- end -}} + +{{/* Resolve a base64 credential and reuse the live Secret during upgrades. */}} +{{- define "hugegraph.seatunnel.authPassword" -}} +{{- $auth := get .Values.seatunnel "auth" | default dict -}} +{{- $password := get $auth "password" | default "" -}} +{{- if $password -}} +{{- $password | b64enc -}} +{{- else -}} +{{- $name := include "hugegraph.seatunnel.authSecretName" . -}} +{{- $secret := lookup "v1" "Secret" .Release.Namespace $name -}} +{{- if and $secret (hasKey $secret "data") (hasKey (get $secret "data") "password") -}} +{{- get (get $secret "data") "password" -}} +{{- else -}} +{{- include "hugegraph.generatedCredential" (dict "root" . "name" "seatunnel-admin") | b64enc -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* Engine YAML uses JVM properties, not Pod env; mount its password in a Secret. */}} +{{- define "hugegraph.seatunnel.engineConfig" -}} +{{- $files := include "hugegraph.seatunnel.config" . | fromYaml -}} +{{- $config := get $files "seatunnel.yaml" | fromYaml | default dict -}} +{{- if hasKey $config "Error" -}} +{{- fail "seatunnel.config[seatunnel.yaml] must contain valid native YAML" -}} +{{- end -}} +{{- $seatunnel := get $config "seatunnel" | default dict -}} +{{- $engine := get $seatunnel "engine" | default dict -}} +{{- $http := get $engine "http" | default dict -}} +{{- $auth := get .Values.seatunnel "auth" | default dict -}} +{{- $_ := set $http "enable-basic-auth" true -}} +{{- $_ := set $http "basic-auth-username" (get $auth "username" | default "admin") -}} +{{- $_ := set $http "basic-auth-password" (include "hugegraph.seatunnel.authPassword" . | b64dec) -}} +{{- $_ := set $engine "http" $http -}} +{{- $_ := set $seatunnel "engine" $engine -}} +{{- $_ := set $config "seatunnel" $seatunnel -}} +{{- $config | toYaml -}} +{{- end -}} + +{{- if (.Values.seatunnel | default dict).enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "hugegraph.seatunnel.authSecretName" . }} + labels: + {{- include "hugegraph.labels" . | nindent 4 }} + app.kubernetes.io/component: seatunnel + annotations: + helm.sh/resource-policy: keep +type: Opaque +data: + username: {{ (get (get .Values.seatunnel "auth" | default dict) "username" | default "admin") | b64enc | quote }} + password: {{ include "hugegraph.seatunnel.authPassword" . | quote }} + seatunnel.yaml: {{ include "hugegraph.seatunnel.engineConfig" . | b64enc | quote }} +{{- end }} diff --git a/helm/hugegraph/templates/seatunnel.yaml b/helm/hugegraph/templates/seatunnel.yaml index 73b1934028..75c425b48f 100644 --- a/helm/hugegraph/templates/seatunnel.yaml +++ b/helm/hugegraph/templates/seatunnel.yaml @@ -48,6 +48,9 @@ Use its DNS discovery alternative, so Kubernetes API RBAC is unnecessary. {{- $_ := set $data "seatunnel.yaml" ($engine | toYaml) -}} {{/* The upstream script appends jvm_options after JAVA_OPTS; set heap in that file. */}} {{- $_ := set $data "jvm_options" (.Values.seatunnel.javaOpts | replace " " "\n") -}} +{{- range $path, $_ := .Files.Glob "examples/seatunnel/hugegraph-*.conf" -}} +{{- $_ := set $data (base $path) ($.Files.Get $path) -}} +{{- end -}} {{- range $filename, $content := .Values.seatunnel.config -}} {{- $_ := set $data $filename $content -}} {{- end -}} @@ -58,7 +61,27 @@ Use its DNS discovery alternative, so Kubernetes API RBAC is unnecessary. {{- include "hugegraph.resolveValues" . }} {{- include "hugegraph.validateValues" . }} {{- $name := include "hugegraph.seatunnel.name" . }} -{{- $config := include "hugegraph.seatunnel.config" . | fromYaml }} +{{- $config := omit (include "hugegraph.seatunnel.config" . | fromYaml) "seatunnel.yaml" }} +{{- $svc := .Values.seatunnel.service }} +{{- $env := list + (dict "name" "SEATUNNEL_HOME" "value" "/opt/seatunnel") + (dict "name" "HG_HOST" "value" (printf "%s.%s.svc" (include "hugegraph.server.name" .) .Release.Namespace)) + (dict "name" "HG_PORT" "value" (.Values.server.port | toString)) + (dict "name" "HG_PROTOCOL" "value" "http") + (dict "name" "HG_USERNAME" "value" "admin") + (dict "name" "HG_GRAPH_SPACE" "value" .Values.server.graphSpace) + (dict "name" "HG_GRAPH_NAME" "value" "graph_a") }} +{{- $password := dict "name" "HG_PASSWORD" "value" "" }} +{{- if .Values.server.auth.enabled }} +{{- $password = dict "name" "HG_PASSWORD" "valueFrom" (dict "secretKeyRef" (dict + "name" (include "hugegraph.server.authSecretName" .) + "key" (include "hugegraph.server.authSecretKey" .))) }} +{{- end }} +{{- $env = append $env $password }} +{{- $overrides := dict }} +{{- range .Values.seatunnel.env }} +{{- $_ := set $overrides .name true }} +{{- end }} apiVersion: v1 kind: ConfigMap metadata: @@ -94,12 +117,29 @@ metadata: labels: {{- include "hugegraph.labels" . | nindent 4 }} app.kubernetes.io/component: seatunnel + {{- with $svc.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: - type: ClusterIP + type: {{ $svc.type }} + {{- range $field := list "externalTrafficPolicy" "loadBalancerClass" "loadBalancerIP" "loadBalancerSourceRanges" "externalIPs" }} + {{- with get $svc $field }} + {{ $field }}: {{ toYaml . | nindent 4 }} + {{- end }} + {{- end }} + {{- if hasKey $svc "allocateLoadBalancerNodePorts" }} + {{- if ne (get $svc "allocateLoadBalancerNodePorts") nil }} + allocateLoadBalancerNodePorts: {{ $svc.allocateLoadBalancerNodePorts }} + {{- end }} + {{- end }} ports: - name: http port: 8080 targetPort: http + {{- with $svc.nodePort }} + nodePort: {{ . }} + {{- end }} selector: {{- include "hugegraph.selectorLabels" . | nindent 4 }} app.kubernetes.io/component: seatunnel @@ -126,6 +166,7 @@ spec: app.kubernetes.io/component: seatunnel annotations: checksum/config: {{ $config | toYaml | sha256sum }} + checksum/auth: {{ include "hugegraph.seatunnel.engineConfig" . | sha256sum }} spec: automountServiceAccountToken: false terminationGracePeriodSeconds: 120 @@ -140,8 +181,11 @@ spec: # The default upstream role is hybrid; an explicit -r is unnecessary. command: ["/opt/seatunnel/bin/seatunnel-cluster.sh"] env: - - name: SEATUNNEL_HOME - value: /opt/seatunnel + {{- range $env }} + {{- if not (hasKey $overrides .name) }} + {{- toYaml (list .) | nindent 12 }} + {{- end }} + {{- end }} {{- with .Values.seatunnel.env }} {{- toYaml . | nindent 12 }} {{- end }} @@ -174,12 +218,21 @@ spec: exec: command: ["/opt/seatunnel/bin/stop-seatunnel-cluster.sh"] volumeMounts: + - name: seatunnel-auth + mountPath: /opt/seatunnel/config/seatunnel.yaml + subPath: seatunnel.yaml {{- range $filename, $_ := $config }} - name: seatunnel-configs mountPath: /opt/seatunnel/config/{{ $filename }} subPath: {{ $filename }} {{- end }} volumes: + - name: seatunnel-auth + secret: + secretName: {{ include "hugegraph.seatunnel.authSecretName" . }} + items: + - key: seatunnel.yaml + path: seatunnel.yaml - name: seatunnel-configs configMap: name: {{ $name }}-configs diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 99a5e03fe6..75006a0a52 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -76,6 +76,23 @@ "replicas": { "type": "integer", "minimum": 1 + }, + "service": { + "$ref": "#/definitions/service" + }, + "auth": { + "type": "object", + "additionalProperties": false, + "properties": { + "username": { + "type": "string", + "minLength": 1, + "pattern": "^[^:]+$" + }, + "password": { + "type": "string" + } + } } } } diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index d6fc5f350c..e6be800629 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -544,6 +544,18 @@ seatunnel: repository: apache/seatunnel tag: "3.0.0" pullPolicy: IfNotPresent + service: + # Unset inherits mode (easy: NodePort, strict: ClusterIP). + type: + # Omitted/0 allocates automatically; only the HTTP/UI port is exposed. + nodePort: + annotations: {} + auth: + # Native HTTP Basic authentication for the Web UI and all REST endpoints. + # Independent of the HugeGraph credential used by source/sink connectors. + username: admin + # Empty generates a kept Secret; upgrades reuse the existing password. + password: "" javaOpts: "-Xms512m -Xmx1g" resources: requests: @@ -551,7 +563,9 @@ seatunnel: memory: 1536Mi limits: memory: 2Gi - # Native Kubernetes env entries, including Secret keyRef credentials. + # Native Kubernetes env entries; matching names override automatic HG_* values. + # HG_HOST/PORT/PROTOCOL, HG_USERNAME/PASSWORD and HG_GRAPH_SPACE/NAME default to + # this release's Server, existing admin Secret, graph space and graph_a. env: [] # Full native configuration files, keyed by filename (e.g. seatunnel.yaml). # Keep member port 5801 and HTTP 8080 when overriding discovery/engine files. diff --git a/helm/test_helm_seatunnel.py b/helm/test_helm_seatunnel.py index 42823efe41..beca8eb38a 100644 --- a/helm/test_helm_seatunnel.py +++ b/helm/test_helm_seatunnel.py @@ -16,6 +16,7 @@ """Check native hybrid deployment, release isolation, connectivity and packaging.""" +import base64 from pathlib import Path import shutil import subprocess @@ -44,6 +45,19 @@ def pod(workload): return workload["spec"]["template"] +def engine_auth_secret(documents): + return next(doc for doc in engine_resources(documents, "Secret") + if "seatunnel.yaml" in doc.get("data", {})) + + +def engine_config(documents): + data = {} + for doc in engine_resources(documents, "ConfigMap"): + data.update(doc["data"]) + data["seatunnel.yaml"] = base64.b64decode(engine_auth_secret(documents)["data"]["seatunnel.yaml"]).decode() + return data + + class SeaTunnelChartTest(unittest.TestCase): @classmethod def setUpClass(cls): @@ -58,19 +72,22 @@ def run_helm(self, arguments, timeout, check=False): except subprocess.TimeoutExpired as error: self.fail(f"{' '.join(command)} timed out after {timeout} seconds: {error}") - def render(self, seatunnel=None, chart=CHART, namespace="default", **values): + def render(self, seatunnel=None, chart=CHART, namespace="default", release="fixture", **values): if seatunnel is not None: values["seatunnel"] = {"enabled": True, **seatunnel} + # Keep independent render comparisons deterministic; generation is tested separately. + values["seatunnel"]["auth"] = {"password": "native-test-admin-password", + **seatunnel.get("auth", {})} with tempfile.TemporaryDirectory(prefix="seatunnel-values-") as directory: configuration = Path(directory) / "values.yaml" configuration.write_text(yaml.safe_dump(values)) - result = self.run_helm(["template", "fixture", str(chart), + result = self.run_helm(["template", release, str(chart), "--namespace", namespace, "-f", str(configuration)], HELM_TEMPLATE_TIMEOUT_SECONDS) return result - def documents(self, seatunnel=None, chart=CHART, namespace="default", **values): - result = self.render(seatunnel, chart=chart, namespace=namespace, **values) + def documents(self, seatunnel=None, chart=CHART, namespace="default", release="fixture", **values): + result = self.render(seatunnel, chart=chart, namespace=namespace, release=release, **values) self.assertEqual(0, result.returncode, result.stderr) return [doc for doc in yaml.safe_load_all(result.stdout) if doc] @@ -91,7 +108,7 @@ def test_modes_share_one_native_hybrid_workload_and_services(self): launch = " ".join(container.get("command", []) + container.get("args", [])) self.assertIn("/opt/seatunnel/bin/seatunnel-cluster.sh", launch) self.assertNotIn(" -r ", launch) - data = engine_resources(documents, "ConfigMap")[0]["data"] + data = engine_config(documents) engine = yaml.safe_load(data["seatunnel.yaml"])["seatunnel"]["engine"] self.assertEqual(backups, engine["backup-count"]) self.assertTrue(engine["http"]["enable-http"]) @@ -120,7 +137,7 @@ def test_dns_discovery_and_services_are_scoped_to_engine_and_release(self): for other in documents: if other["kind"] in ("Deployment", "StatefulSet") and other is not workload: self.assertFalse(matches(selector, pod(other)["metadata"]["labels"])) - data = engine_resources(documents, "ConfigMap")[0]["data"] + data = engine_config(documents) discovery = yaml.safe_load(data["hazelcast.yaml"])["hazelcast"]["network"]["join"]["kubernetes"] self.assertTrue(discovery["enabled"]) self.assertTrue(discovery["service-dns"].startswith(headless["metadata"]["name"] + ".graphs.svc")) @@ -135,6 +152,150 @@ def test_dns_discovery_and_services_are_scoped_to_engine_and_release(self): self.assertIn(discovery["service-dns"] + ":5801", client["network"]["cluster-members"]) self.assertFalse(any(doc["kind"] in ("Role", "RoleBinding") for doc in engine_resources(documents))) + def test_http_entry_inherits_mode_and_preserves_explicit_service_settings(self): + for mode, service_type in (("easy", "NodePort"), ("strict", "ClusterIP")): + with self.subTest(mode=mode): + documents = self.documents({}, mode=mode) + api, = [doc for doc in engine_resources(documents, "Service") + if doc["spec"].get("clusterIP") != "None"] + self.assertEqual(service_type, api["spec"]["type"]) + self.assertNotIn("nodePort", api["spec"]["ports"][0]) + for mode, service in (("easy", {"type": "ClusterIP"}), + ("strict", {"type": "NodePort", "nodePort": 31081})): + with self.subTest(mode=mode, service=service): + documents = self.documents({"service": service}, mode=mode) + headless, = [doc for doc in engine_resources(documents, "Service") + if doc["spec"].get("clusterIP") == "None"] + api, = [doc for doc in engine_resources(documents, "Service") if doc is not headless] + self.assertEqual(service["type"], api["spec"]["type"]) + if "nodePort" in service: + self.assertEqual(service["nodePort"], api["spec"]["ports"][0]["nodePort"]) + self.assertNotIn("nodePort", headless["spec"]["ports"][0]) + result = self.render({"service": {"type": "ClusterIP", "nodePort": 31081}}) + self.assertNotEqual(0, result.returncode) + + def test_local_graph_defaults_follow_rendered_server_and_admin_secret(self): + documents = self.documents({}, namespace="graphs", release="fixture-" + "r" * 40, + fullnameOverride="graph-copy-" + "n" * 55, + server={"port": 8182, "graphSpace": "tenant_a"}) + server, = [doc for doc in documents if doc["kind"] == "Service" + and doc["metadata"]["labels"].get("app.kubernetes.io/component") == "server" + and doc["spec"].get("clusterIP") != "None"] + server_workload, = [doc for doc in documents if doc["kind"] == "Deployment" + and doc["metadata"]["labels"].get("app.kubernetes.io/component") == "server"] + server_container = next(entry for entry in pod(server_workload)["spec"]["containers"] + if entry["name"] == "server") + server_password = next(entry["valueFrom"] for entry in server_container["env"] + if entry["name"] == "PASSWORD") + workload, = engine_resources(documents, "StatefulSet") + container, = pod(workload)["spec"]["containers"] + env = {entry["name"]: entry for entry in container["env"]} + self.assertEqual(server["metadata"]["name"] + ".graphs.svc", env["HG_HOST"]["value"]) + self.assertEqual(str(server["spec"]["ports"][0]["port"]), env["HG_PORT"]["value"]) + self.assertEqual("http", env["HG_PROTOCOL"]["value"]) + self.assertEqual("tenant_a", env["HG_GRAPH_SPACE"]["value"]) + self.assertEqual("graph_a", env["HG_GRAPH_NAME"]["value"]) + self.assertEqual("admin", env["HG_USERNAME"]["value"]) + self.assertEqual(server_password, env["HG_PASSWORD"]["valueFrom"]) + + def test_existing_admin_secret_and_disabled_auth_reach_local_graph_environment(self): + cases = (({"enabled": True, "admin": {"existingSecret": "graph-admin", "key": "admin-pass"}}, + {"name": "HG_PASSWORD", "valueFrom": { + "secretKeyRef": {"name": "graph-admin", "key": "admin-pass"}}}), + ({"enabled": False}, {"name": "HG_PASSWORD", "value": ""})) + for auth, password in cases: + with self.subTest(auth=auth): + documents = self.documents({}, server={"auth": auth}, hubble={"enabled": False}) + workload, = engine_resources(documents, "StatefulSet") + container, = pod(workload)["spec"]["containers"] + self.assertIn(password, container["env"]) + + def test_explicit_environment_overrides_defaults_without_duplicate_names(self): + overrides = [{"name": "HG_HOST", "value": "external-graph.example.com"}, + {"name": "HG_PORT", "value": "8443"}, + {"name": "HG_PROTOCOL", "value": "https"}, + {"name": "HG_GRAPH_SPACE", "value": "other"}, + {"name": "HG_GRAPH_NAME", "value": "existing_graph"}, + {"name": "HG_USERNAME", "value": "writer"}, + {"name": "HG_PASSWORD", "valueFrom": { + "secretKeyRef": {"name": "writer-auth", "key": "password"}}}, + {"name": "TZ", "value": "UTC"}] + documents = self.documents({"env": overrides}) + workload, = engine_resources(documents, "StatefulSet") + container, = pod(workload)["spec"]["containers"] + env = {entry["name"]: entry for entry in container["env"]} + self.assertEqual(len(env), len(container["env"])) + for expected in overrides: + self.assertEqual(expected, env[expected["name"]]) + + def test_demo_jobs_are_mounted_and_keep_credentials_as_placeholders(self): + password = "render-test-password" + documents = self.documents({}, server={"auth": {"admin": {"password": password}}}) + configmaps = engine_resources(documents, "ConfigMap") + jobs = {doc["metadata"]["name"]: doc["data"] for doc in configmaps + if any(filename.endswith(".conf") for filename in doc["data"])} + self.assertEqual(2, sum(filename.endswith(".conf") for data in jobs.values() for filename in data)) + workload, = engine_resources(documents, "StatefulSet") + spec = pod(workload)["spec"] + container, = spec["containers"] + for configmap_name, data in jobs.items(): + volume_names = {volume["name"] for volume in spec["volumes"] + if volume.get("configMap", {}).get("name") == configmap_name} + self.assertTrue(volume_names) + mounts = [mount for mount in container["volumeMounts"] if mount["name"] in volume_names] + self.assertTrue(mounts) + for filename, content in data.items(): + if filename.endswith(".conf"): + self.assertTrue(any(mount["mountPath"] == "/opt/seatunnel/config/" + filename + for mount in mounts)) + for name in ("HG_HOST", "HG_PORT", "HG_PROTOCOL", "HG_GRAPH_SPACE", "HG_GRAPH_NAME", + "HG_USERNAME", "HG_PASSWORD"): + self.assertIn("${" + name + "}", content) + for data in (doc["data"] for doc in configmaps): + self.assertFalse(any(password in content for content in data.values())) + + def test_seatunnel_admin_is_generated_separately_and_kept_out_of_public_configuration(self): + graph_password = "graph-only-admin-password" + documents = self.documents({"auth": {"password": ""}}, + server={"auth": {"admin": {"password": graph_password}}}) + secret = engine_auth_secret(documents) + username = base64.b64decode(secret["data"]["username"]).decode() + password = base64.b64decode(secret["data"]["password"]).decode() + self.assertEqual("admin", username) + self.assertGreaterEqual(len(password), 32) + self.assertNotEqual(graph_password, password) + http = yaml.safe_load(engine_config(documents)["seatunnel.yaml"])["seatunnel"]["engine"]["http"] + self.assertTrue(http["enable-basic-auth"]) + self.assertEqual(username, http["basic-auth-username"]) + self.assertEqual(password, http["basic-auth-password"]) + for configmap in engine_resources(documents, "ConfigMap"): + self.assertNotIn("seatunnel.yaml", configmap["data"]) + self.assertNotIn(password, yaml.safe_dump(configmap["data"])) + workload, = engine_resources(documents, "StatefulSet") + self.assertNotIn(password, yaml.safe_dump(pod(workload))) + volumes = {volume["name"] for volume in pod(workload)["spec"]["volumes"] + if volume.get("secret", {}).get("secretName") == secret["metadata"]["name"]} + container, = pod(workload)["spec"]["containers"] + self.assertTrue(any(mount["name"] in volumes + and mount["mountPath"] == "/opt/seatunnel/config/seatunnel.yaml" + for mount in container["volumeMounts"])) + + def test_native_basic_auth_preserves_special_credentials_and_cannot_be_disabled_by_config(self): + username = "ops'\"\\$" + password = "quote'\" backslash\\ dollar$ colon:" + native = {"seatunnel": {"engine": {"backup-count": 0, "http": { + "enable-http": True, "port": 8080, "enable-basic-auth": False, + "basic-auth-username": "ignored-user", "basic-auth-password": "ignored-password"}}}} + documents = self.documents({"auth": {"username": username, "password": password}, + "config": {"seatunnel.yaml": yaml.safe_dump(native)}}) + secret = engine_auth_secret(documents) + self.assertEqual(username, base64.b64decode(secret["data"]["username"]).decode()) + self.assertEqual(password, base64.b64decode(secret["data"]["password"]).decode()) + http = yaml.safe_load(engine_config(documents)["seatunnel.yaml"])["seatunnel"]["engine"]["http"] + self.assertTrue(http["enable-basic-auth"]) + self.assertEqual(username, http["basic-auth-username"]) + self.assertEqual(password, http["basic-auth-password"]) + def test_image_secret_environment_and_native_file_overrides_reach_engine(self): custom = {"jvm_options": "-Xms128m\n-Xmx384m\n", "log4j2.properties": "status=warn\n"} secret_env = {"name": "HG_PASSWORD", "valueFrom": {"secretKeyRef": {"name": "graph-auth", "key": "password"}}} @@ -149,7 +310,7 @@ def test_image_secret_environment_and_native_file_overrides_reach_engine(self): self.assertEqual("Never", container["imagePullPolicy"]) self.assertEqual([{"name": "registry-auth"}], spec["imagePullSecrets"]) self.assertIn(secret_env, container["env"]) - data = engine_resources(documents, "ConfigMap")[0]["data"] + data = engine_config(documents) mounts = {entry["subPath"] for entry in container["volumeMounts"]} for filename, content in custom.items(): self.assertEqual(content, data[filename]) @@ -160,7 +321,12 @@ def test_image_secret_environment_and_native_file_overrides_reach_engine(self): def test_invalid_enabled_mode_and_cluster_size_fail_before_install(self): for values in ({"mode": "invalid"}, {"mode": "cluster", "replicas": 0}, - {"mode": "cluster", "replicas": 1}): + {"mode": "cluster", "replicas": 1}, + {"auth": {"username": "${user.name}"}}, + {"auth": {"password": "${user.name}"}}, + {"auth": {"username": " admin"}}, + {"auth": {"password": "password "}}, + {"auth": {"username": "admin:operator"}}): with self.subTest(values=values): self.assertNotEqual(0, self.render(values).returncode) From 301dd725aafdd41a27f80b3d414e1356646264a3 Mon Sep 17 00:00:00 2001 From: imbajin Date: Sat, 24 Oct 2026 01:27:50 +0800 Subject: [PATCH 10/17] fix(helm): preserve legacy values without SeaTunnel --- helm/hugegraph/templates/_helpers.tpl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index d849f62a2a..7322d2c6c0 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -23,6 +23,9 @@ {{- if not (hasKey .Values "hubble") -}} {{- $_ := set .Values "hubble" (dict "enabled" false "service" dict "persistence" dict) -}} {{- end -}} +{{- if not (hasKey .Values "seatunnel") -}} +{{- $_ := set .Values "seatunnel" (dict "enabled" false "service" dict) -}} +{{- end -}} {{- $strict := eq (.Values.mode | default "easy") "strict" -}} {{- $serviceType := ternary "ClusterIP" "NodePort" (or $strict $legacy) -}} {{- range $comp := list "server" "hubble" "seatunnel" -}} From 051d1fe425dea600ff785da4e45390bc618a2746 Mon Sep 17 00:00:00 2001 From: orion Date: Fri, 23 Oct 2026 17:50:16 +0000 Subject: [PATCH 11/17] docs(helm): remove one-off SeaTunnel validation receipts --- helm/hugegraph/docs/seatunnel.md | 47 ++++---------------------------- 1 file changed, 5 insertions(+), 42 deletions(-) diff --git a/helm/hugegraph/docs/seatunnel.md b/helm/hugegraph/docs/seatunnel.md index 5628a61ed2..2128f89abe 100644 --- a/helm/hugegraph/docs/seatunnel.md +++ b/helm/hugegraph/docs/seatunnel.md @@ -235,47 +235,10 @@ done 验收应同时核对属性键类型、基数、标签、主键、边端点定义、索引和 Schema ID，以及全部顶点 ID/属性、边 ID/端点/属性；只比较数量或作业状态不够。相关选项见官方 [HugeGraph Source](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/source/HugeGraph.md) 和 [Sink](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/sink/HugeGraph.md)。 -## 已完成实测记录：原生 hybrid 基线 +## 使用限制 -2026 年 10 月 23–24 日（Asia/Singapore）完成以下验证，均使用本页的一套原生 hybrid StatefulSet。源、目标是两个独立的 Helm 环境，各有 **1 PD + 1 Store + 1 Server，后端为 HStore**，分别使用自己的 PVC 和认证。源 `graph_a` 与目标 `graph_b` 不共享后端；另在目标环境的 `graph_a` 测试十条数据直接导入。 +集群模式下，应逐个 hybrid Pod 确认 Workers 和成员数与副本数一致；仅 Pod Ready 不能说明成员已组成同一个集群。Hazelcast DNS 发现使用模板中的 `service-port: 5801`，应与引擎监听端口保持一致。 -SeaTunnel 使用原始 `apache/seatunnel:3.0.0` 镜像。实际校验了 OCI 清单、配置及全部九层的大小和 SHA-256，并在容器中确认 HugeGraph、FakeSource、Console 连接器及内置 UI 均存在；无需自建镜像。镜像索引 digest 为 `sha256:914fcf37b2d5acdf9f95f7dbbd7d4c62d150fcbc5ff3e5f59957d65840fea8dd`，实际运行 amd64 清单 digest 为 `sha256:dd8f357dfed2192a68184aeac41507e5529eb06c208b12aae4bf6466ed0a0bb0`。 - -测试环境资源有限，kind 节点使用 FUSE overlay 承载官方镜像原始层；PD、Store、Server 则复用此前从源码构建的缓存运行镜像。以上均为测试基础设施设置，不是 Chart 的部署依赖。HugeGraph 测试运行产物的源码版本为 `f60ad3e3bfc71d4404956fd2dbfb62c7bc002f13`；本轮同时对最新 `master` 基础上的改动完成了编译检查。 - -| 验证项 | 实际结果 | -| --- | --- | -| 直接导入 | 原始 `hugegraph-sink.conf` 未修改，作业 `1166045557131575297` 为 `FINISHED`，读 10、写 10、失败 0；目标图查询及逐条内容核对均为 10 条 | -| Schema 复制 | 空目标先复制 10 个属性键、2 个顶点标签、2 个边标签、3 个索引；保留原 Schema ID，逐项核对类型、基数、ID 策略、nullable、索引和自定义元数据 | -| 单机复制 | 00:08:49 完成，1 个 hybrid 节点；4 个作业全部 `FINISHED`，1000 个顶点、2400 条边的 ID、全部属性和边端点与源图相同 | -| 集群复制 | 00:18:51 完成，2 个 hybrid 节点；人为扰动目标一个属性后重新运行 4 个作业，目标恢复到与源图完全一致；每个 Pod 在作业前后均报告 Workers=2 | -| 切回单机 | 00:21:16 完成只读复查，Workers=1，运行中作业为 0；全部 Schema、1000 个顶点、2400 条边及另图的 10 条导入数据保持一致 | -| 升级影响 | 单机 → 集群 → 单机使用同一个 StatefulSet；两环境原有 6 个 HugeGraph Pod UID、4 个 PVC、6 个认证 Secret 均未变化，Pod 全部 Ready 且重启为 0；匿名访问 401、有效账号 200 | -| 内置前端 | 真实 Chromium（Asia/Singapore）验证两种模式的 Overview、作业列表和 DAG：Workers 分别为 1、2；选中作业的 DAG 均为 2 节点、1 条连接，显示读写各 800；浏览器 JS、关键 API、静态资源错误均为 0 | - -复制作业的逐项结果如下；两种模式每个作业的失败数均为 0。 - -| 标签 | 类型 | 读取 / 写入 | 单机作业 ID | 集群作业 ID | -| --- | --- | --- | --- | --- | -| person | PRIMARY_KEY 顶点 | 800 / 800 | 1166045485111181313 | 1166048035369648130 | -| company | CUSTOMIZE_NUMBER 顶点 | 200 / 200 | 1166045504388202497 | 1166048053988163586 | -| works_at | 边 | 800 / 800 | 1166045519504474113 | 1166048070853459969 | -| knows | MULTIPLE 边 | 1600 / 1600 | 1166045539670687745 | 1166048087110582273 | - -验收使用全量数据比较，单机、集群和切回单机后的 Schema、顶点、边三组规范化哈希分别完全一致。Schema 比较仅忽略元素顶层的运行状态及系统 `user_data.~create_time`，保留业务自定义的 `status/create_time` 元数据。另验证了 SET 内容、LIST 顺序和重复元素、可空属性、自定义元数据、同端点且 sort key 为 2020/2021 的两条多重边；三个索引均为 `CREATED`，范围查询 `age >= 80` 返回 88 条，二级索引查询 `company_123` 返回 1 条，源、目标查询结果相同。 - -集群测试曾发现两个 Pod 虽然 Ready，却各自形成单节点集群。原因是 Hazelcast DNS 发现未指定端口时使用 5701，而引擎监听 5801；模板已明确配置 `service-port: 5801`。修复后逐 Pod 确认 Workers=2、成员数为 2，持续观察一分钟，再重新执行上述集群作业和前端验证。仅 Pod Ready 不作为集群组建成功的依据。 - -上述基线还通过了 22 项 Python Helm 回归、309 项 Helm 单元测试（36 个 suite、4 个快照）、两种模式的 strict Helm lint，以及目标环境的 `helm test --logs`。`mvn editorconfig:format` 和 `mvn clean compile -Dmaven.javadoc.skip=true` 的 38 个模块均通过。 - -本次范围与已知限制： - -- 已覆盖 `PRIMARY_KEY`、`CUSTOMIZE_NUMBER`，暂不覆盖 `AUTOMATIC` 顶点的无损复制；后者需要解决新旧顶点 ID 与边端点的对应关系。 -- 官方 3.0.0 UI 的 Duration 显示存在上游问题：无时区时间字符串在新加坡浏览器中会产生 8 小时偏差，且已完成作业的显示时长仍随当前时间增长。以 CLI/REST 的完成状态及开始、结束时间核对实际结果；本次未修改上游 UI 或硬编码时区。 -- 测试覆盖停止作业后的模式升级和数据保留，未验证运行中作业不中断切换、生产 HA、持久化恢复或 CNI 的实际网络隔离。默认临时 checkpoint 目录不能据此视为持久化恢复方案。 - -## 本轮增量验证记录 - -本轮新增 SeaTunnel HTTP 入口继承 easy/strict、独立原生 HTTP Basic 认证、同 release 的 `HG_*` 默认连接信息，以及启动时挂载 Source/Sink 演示文件。上一节的原生 hybrid 基线记录验证了引擎、跨环境 Schema／点／边复制及内置 UI，不作为这些新增行为的运行验证结论。 - -本轮增量实测结果将在完成后补充。 +- 示例支持 `PRIMARY_KEY`、`CUSTOMIZE_NUMBER`，不支持 `AUTOMATIC` 顶点的无损复制；后者需要解决新旧顶点 ID 与边端点的对应关系。 +- 官方 3.0.0 UI 的 Duration 显示存在上游问题：无时区时间字符串在新加坡浏览器中会产生 8 小时偏差，且已完成作业的显示时长仍随当前时间增长。以 CLI/REST 的完成状态及开始、结束时间核对实际结果。 +- 模式切换前应停止作业；本集成不承诺运行中作业无中断切换、生产 HA 或持久化恢复。默认临时 checkpoint 目录不是持久化恢复方案，NetworkPolicy 的实际隔离依赖 CNI 支持。 From 8fde247089ced2b2438710596697c3547e0a701c Mon Sep 17 00:00:00 2001 From: orion Date: Fri, 23 Oct 2026 17:50:26 +0000 Subject: [PATCH 12/17] fix(helm): retain legacy values compatibility Default absent optional SeaTunnel maps for legacy release renders. Cover earlier install/upgrade values and registry overrides. Record authenticated NodePort, dual-member jobs and password rotation tests. --- helm/hugegraph/docs/seatunnel.md | 42 +++++++++++++++++++++++++-- helm/hugegraph/templates/_helpers.tpl | 6 ++-- helm/test_helm_seatunnel.py | 23 +++++++++++++++ 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/helm/hugegraph/docs/seatunnel.md b/helm/hugegraph/docs/seatunnel.md index 5628a61ed2..84f659df8c 100644 --- a/helm/hugegraph/docs/seatunnel.md +++ b/helm/hugegraph/docs/seatunnel.md @@ -84,7 +84,9 @@ helm upgrade hugegraph ./helm/hugegraph -n hugegraph -f my-values.yaml \ ## SeaTunnel 入口认证 -前端、静态资源和 REST 提交接口默认使用引擎原生 HTTP Basic 认证，不新增认证代理或独立登录服务。SeaTunnel 使用独立入口管理员：`seatunnel.auth.username` 默认为 `admin`；密码未显式设置时，Chart 自动生成强密码并保存在 `-auth` Secret，升级时复用已有密码。也可通过受控 values 文件显式设置 `seatunnel.auth.username/password`，用于自定义账号或轮换。自定义值不能有首尾空白或 `${...}` 占位符；内部空格和引号仍可使用。只做离线模板或 GitOps 渲染时，应显式固定 `seatunnel.auth.password`；默认密码的升级复用依赖在线 Helm lookup。 +前端、静态资源和 REST 提交接口默认使用引擎原生 HTTP Basic 认证，不新增认证代理或独立登录服务。SeaTunnel 使用独立入口管理员：`seatunnel.auth.username` 默认为 `admin`；密码未显式设置时，Chart 自动生成强密码并保存在 `-auth` Secret，升级时复用已有密码。 + +也可通过受控 values 文件显式设置 `seatunnel.auth.username/password`，用于自定义账号或轮换。自定义值不能有首尾空白或 `${...}` 占位符，用户名另不能包含冒号；内部空格和引号仍可使用。只做离线模板或 GitOps 渲染时，应显式固定 `seatunnel.auth.password`；默认密码的升级复用依赖在线 Helm lookup。 默认 release 的 Secret 为 `hugegraph-seatunnel-auth`。保持前述端口转发，在本机读取账号和密码后调用 REST： @@ -278,4 +280,40 @@ SeaTunnel 使用原始 `apache/seatunnel:3.0.0` 镜像。实际校验了 OCI 清 本轮新增 SeaTunnel HTTP 入口继承 easy/strict、独立原生 HTTP Basic 认证、同 release 的 `HG_*` 默认连接信息，以及启动时挂载 Source/Sink 演示文件。上一节的原生 hybrid 基线记录验证了引擎、跨环境 Schema／点／边复制及内置 UI，不作为这些新增行为的运行验证结论。 -本轮增量实测结果将在完成后补充。 +2026 年 10 月 24 日（Asia/Singapore）完成增量实测。使用一套独立的 **1 PD + 1 Store + 1 Server/HStore**，先保持 SeaTunnel 关闭，再同 release 开启，随后切换 strict 双节点并轮换入口密码。为验证配置来自 Helm 实际值，使用 `fullnameOverride: bridge`、Server REST 端口 `8184`，以及外部 HugeGraph 管理员 Secret `graph-admin` 的自定义键 `credential`；SeaTunnel 未手动设置任何 `HG_*`。 + +SeaTunnel 继续使用上节相同 digest 的官方原镜像；本轮重新下载并验证九层及配置、清单，在独立节点再次核对全部 SHA-256 和大小。HugeGraph 继续使用上节注明版本的缓存运行产物；当前分支已合入 remote master 的 `75ea5c7dc`，并另行通过全部 38 模块编译。 + +| 项目 | 实测结果 | +|---|---| +| easy 真实入口 | HTTP Service 自动继承 NodePort，指定 `32080`，经真实节点端口访问；只有 HTTP 对外公开，5801 保留内部 | +| strict 双节点 | Service 自动改为 ClusterIP、移除 NodePort，原 `32080` 入口实际不可访问；每个 Pod 均报告 Workers=2、Hazelcast Members=2，持续稳定超过一分钟 | +| 入口认证 | 每阶段检查首页、`index.html`、真实静态资源、`/overview` 及 `POST /submit-job`；匿名、错误用户名、错误密码、HugeGraph 管理员凭据共 60 个负例全部 401，正确 SeaTunnel 凭据返回 200；SeaTunnel 凭据访问 HugeGraph 同样被拒绝 | +| 默认连接与文件 | 自动连接 `bridge-server..svc:8184`，正确引用 `graph-admin/credential`；挂载的 Source/Sink 配置与仓库文件字节一致，无上传或手动补齐步骤 | +| Source/Sink | easy 单节点、strict 双节点分别直接运行两个挂载示例，均 `FINISHED`、读 10、写 10、失败 0；Source Console 中十组姓名及年龄与图数据逐条一致 | +| REST 提交 | 三阶段各通过带认证的 REST 提交一次 FakeSource → Console，均 `FINISHED`，读取、写入和提交计数均为 10 | +| 升级与轮换 | 普通在线升级复用独立密码；显式轮换后旧密码 401、新密码 200，Secret UID 与用户名保留。两个引擎 Pod 滚动更新后仍稳定为双成员 | +| HugeGraph 保持 | 启用 SeaTunnel、切换 strict 双节点和密码轮换后，原 3 个 HugeGraph Pod UID、2 个 PVC、3 个原有凭据 Secret 均未变，Pod Ready/0 重启，图仍为准确的 10 个顶点、0 条边；启用前及 strict 后原生 `helm test` 均通过 | +| 凭据存放 | 原生认证配置与独立 Secret 一致，`seatunnel.yaml` 只从 Secret 挂载，认证 checksum 正确；SeaTunnel 密码未进入 ConfigMap、Pod 环境变量或 JVM 参数。业务及浏览器证据中的实际密码、Basic token 扫描均无泄漏 | + +七个实际作业的记录如下： + +| 阶段 | 作业 | ID | 状态 / 读写 | +|---|---|---|---| +| easy 单节点 | 挂载 Sink | 1166060683054284801 | FINISHED / 10、10 | +| easy 单节点 | 挂载 Source | 1166060702146756609 | FINISHED / 10、10 | +| easy 单节点 | REST FakeSource → Console | 1166060687785721857 | FINISHED / 10、10 | +| strict 双节点 | 挂载 Sink | 1166061637581406210 | FINISHED / 10、10 | +| strict 双节点 | 挂载 Source | 1166061657428852738 | FINISHED / 10、10 | +| strict 双节点 | REST FakeSource → Console | 1166061642451255297 | FINISHED / 10、10 | +| 轮换后双节点 | REST FakeSource → Console | 1166062548156416001 | FINISHED / 10、10 | + +真实 Chromium 使用新加坡时区，分别在 01:09:12、01:13:01、01:16:46 完成 easy 单节点、strict 双节点、轮换后双节点的 Basic 登录、Overview、Jobs 和 DAG 验证，Workers 分别为 1、2、2。每次选中作业的 DAG 都有两个节点、一条连接，页面显示读写各 10；累计 21 次关键 API、51 次静态资源响应全部 200，JS、控制台及请求错误均为 0。九张截图均人工检查，未打开可能展示作业凭据的配置页。 + +预热时曾选择 REST 端口 `8182`，与 HugeGraph 默认 Gremlin 端口冲突；改为 `8184` 并通过原生 Helm test 后才建立资源基线。strict 后从未获放行的 Server Pod 探测 SeaTunnel 被阻断，最终成员检查改为逐引擎 Pod 本地认证访问；没有为测试扩大默认放行范围。 + +最终本地回归为 30 项 Python 测试（含 15 项 SeaTunnel 集成）、309 项 Helm 单元测试，formatter 和最新 master 基础的 38 模块编译通过。CI 发现的旧 release values 缺少 `seatunnel` 字段兼容问题已补上空值处理，并新增真实旧 values 的安装、升级及自定义镜像仓库渲染回归。 + +验证结束后已清理本轮专用 kind 集群、网络及卷，日志、结果和截图保留。原有测试环境的节点身份、启动时间及运行状态均未变化。 + +验证结束后已清理本轮专用 kind 集群、网络及卷，日志、结果和截图保留。原有测试环境的节点身份、启动时间及运行状态均未变化。 diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index d849f62a2a..9e901d08ce 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -26,7 +26,7 @@ {{- $strict := eq (.Values.mode | default "easy") "strict" -}} {{- $serviceType := ternary "ClusterIP" "NodePort" (or $strict $legacy) -}} {{- range $comp := list "server" "hubble" "seatunnel" -}} -{{- $component := get $.Values $comp -}} +{{- $component := get $.Values $comp | default dict -}} {{- $svc := get $component "service" | default dict -}} {{- $_ := set $component "service" $svc -}} {{- $typeInherited := or (not (hasKey $svc "type")) (eq (get $svc "type" | toString) "") -}} @@ -49,7 +49,7 @@ {{- if contains "://" $registry -}}{{- fail "global.imageRegistry must be a registry host/path without http:// or https://" -}}{{- end -}} {{- if $registry -}} {{- range $comp := list "pd" "store" "server" "hubble" "seatunnel" -}} -{{- $image := get (get $.Values $comp) "image" | default dict -}} +{{- $image := get (get $.Values $comp | default dict) "image" | default dict -}} {{- if $image -}} {{- $_ := set $image "repository" (include "hugegraph.registryImage" (dict "reference" $image.repository "registry" $registry)) -}} {{- end -}} @@ -749,7 +749,7 @@ Cross-field validation that JSON Schema draft-07 cannot express. {{- fail "seatunnel.service.nodePort requires seatunnel.service.type to be NodePort or LoadBalancer" -}} {{- end -}} {{- range $key := list "username" "password" -}} -{{- $value := get $.Values.seatunnel.auth $key | default "" -}} +{{- $value := get ($.Values.seatunnel.auth | default dict) $key | default "" -}} {{- if or (ne $value (trim $value)) (contains "${" $value) -}} {{- fail (printf "seatunnel.auth.%s cannot contain ${...} placeholders or leading/trailing whitespace: native Engine YAML expands or trims them" $key) -}} {{- end -}} diff --git a/helm/test_helm_seatunnel.py b/helm/test_helm_seatunnel.py index beca8eb38a..a0f340be13 100644 --- a/helm/test_helm_seatunnel.py +++ b/helm/test_helm_seatunnel.py @@ -96,6 +96,29 @@ def test_disabled_has_no_engine_resources(self): self.assertEqual([], engine_resources(documents)) self.assertFalse(any("seatunnel" in doc["metadata"]["name"] for doc in documents)) + def test_legacy_release_without_seatunnel_values_still_renders(self): + with tempfile.TemporaryDirectory(prefix="seatunnel-legacy-") as directory: + chart = Path(directory) / "chart" + shutil.copytree(CHART, chart) + (chart / "values.yaml").write_bytes((CHART / "testdata/values-pre-hardening.yaml").read_bytes()) + for arguments in ([], ["--is-upgrade"], ["--set", "global.imageRegistry=registry.example.com"]): + with self.subTest(arguments=arguments): + result = self.run_helm(["template", "legacy", str(chart), *arguments], + HELM_TEMPLATE_TIMEOUT_SECONDS) + self.assertEqual(0, result.returncode, result.stderr) + documents = [doc for doc in yaml.safe_load_all(result.stdout) if doc] + self.assertEqual([], engine_resources(documents)) + # An earlier enabled release also predates the entry/auth fields. + previous = yaml.safe_load((CHART / "values.yaml").read_text()) + previous["seatunnel"]["enabled"] = True + for field in ("service", "auth"): + previous["seatunnel"].pop(field) + (chart / "values.yaml").write_text(yaml.safe_dump(previous)) + documents = self.documents(chart=chart) + http = yaml.safe_load(engine_config(documents)["seatunnel.yaml"])["seatunnel"]["engine"]["http"] + self.assertTrue(http["enable-basic-auth"]) + self.assertEqual("admin", http["basic-auth-username"]) + def test_modes_share_one_native_hybrid_workload_and_services(self): modes = [self.documents({}), self.documents({"mode": "cluster", "replicas": 3})] workloads = [] From 6a1ed2ac163b7e1a89cfee3f8ff7cbdd2653216a Mon Sep 17 00:00:00 2001 From: orion Date: Sun, 25 Oct 2026 05:18:26 +0000 Subject: [PATCH 13/17] fix(helm): guard graph clone examples Reject inherited or unknown edge label types before contacting the target. Pin clone mapping time zones instead of relying on the worker default. Document source timezone assumptions and cover the early rejection path. --- helm/hugegraph/docs/seatunnel.md | 6 ++++++ .../examples/seatunnel/clone/company.conf | 1 + .../examples/seatunnel/clone/copy-schema.py | 11 ++++++++-- .../examples/seatunnel/clone/knows.conf | 1 + .../examples/seatunnel/clone/person.conf | 1 + .../examples/seatunnel/clone/works_at.conf | 1 + helm/test_helm_seatunnel.py | 21 +++++++++++++++++++ 7 files changed, 40 insertions(+), 2 deletions(-) diff --git a/helm/hugegraph/docs/seatunnel.md b/helm/hugegraph/docs/seatunnel.md index 5ca29b1f84..bcaede9db1 100644 --- a/helm/hugegraph/docs/seatunnel.md +++ b/helm/hugegraph/docs/seatunnel.md @@ -208,6 +208,8 @@ curl -fsS -u "$HG_USERNAME:$HG_PASSWORD" -H 'Content-Type: application/json' \ 先通过 HugeGraph 原生 API 创建空的 `graph_b`，再执行 [copy-schema.py](../examples/seatunnel/clone/copy-schema.py)。脚本在目标图的 RESTORING 状态下保留源 Schema ID，复制属性键、顶点标签、边标签和索引；不传输图数据。源、目标应使用兼容的 HugeGraph 版本，账号需有相应管理权限，目标 Schema 必须为空。 +这份示例只支持普通边标签。遇到 `PARENT`、`SUB` 或未知边标签类型时，脚本会在访问目标图前退出；继承边需要按父子关系另行迁移。 + ```bash # 两个 URL 均包含图空间及图名；对应账号密码由本机环境提供 export HG_SOURCE_GRAPH_URL=https://source.example.com/graphspaces/DEFAULT/graphs/graph_a @@ -235,6 +237,10 @@ done 这组示例验证 `PRIMARY_KEY` 和 `CUSTOMIZE_NUMBER` 顶点 ID 策略。现有连接器对 `AUTOMATIC` 顶点会生成新 ID，不能用这组映射直接无损复制其边端点；迁移前先核对源图的 ID 策略。 +四份映射均显式设置 `timeZone = UTC`，假设源 Server 的 `DATE` 字符串按 UTC 序列化，避免复制结果随 SeaTunnel worker 的默认时区变化。 +源 Server 使用其他时区时，应把每个 Sink `mappings` 条目中的 `timeZone` 改为源 Server 的实际时区。 +Source 的 `time_zone` 不能修正 REST 返回的无时区日期字符串；含 `DATE` 属性时，验收还需比较原始时间戳。 + 验收应同时核对属性键类型、基数、标签、主键、边端点定义、索引和 Schema ID，以及全部顶点 ID/属性、边 ID/端点/属性；只比较数量或作业状态不够。相关选项见官方 [HugeGraph Source](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/source/HugeGraph.md) 和 [Sink](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/sink/HugeGraph.md)。 ## 使用限制 diff --git a/helm/hugegraph/examples/seatunnel/clone/company.conf b/helm/hugegraph/examples/seatunnel/clone/company.conf index a2cbd1174c..aaf4b66d45 100644 --- a/helm/hugegraph/examples/seatunnel/clone/company.conf +++ b/helm/hugegraph/examples/seatunnel/clone/company.conf @@ -46,6 +46,7 @@ sink { { type = VERTEX label = company + timeZone = UTC idStrategy = CUSTOMIZE_NUMBER idFields = ["~id"] } diff --git a/helm/hugegraph/examples/seatunnel/clone/copy-schema.py b/helm/hugegraph/examples/seatunnel/clone/copy-schema.py index b4ec6e8547..0fb19bc83b 100644 --- a/helm/hugegraph/examples/seatunnel/clone/copy-schema.py +++ b/helm/hugegraph/examples/seatunnel/clone/copy-schema.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Copy schema to an existing empty graph, preserving IDs for subsequent data copy.""" +"""Copy ordinary-label schema to an empty graph, preserving IDs for data copy.""" import argparse import base64 import copy @@ -61,8 +61,15 @@ def main(): parser.add_argument('--target', required=True, help='Target graph URL, including graph name') args = parser.parse_args() source = client(args.source, 'SOURCE') - target = client(args.target, 'TARGET') schema = source('GET', '/schema') + for label in schema['edgelabels']: + label_type = label.get('edgelabel_type', 'NORMAL') + if label_type != 'NORMAL': + raise SystemExit( + f"Unsupported edge label type {label_type!r} for {label['name']!r}: " + 'this example only supports NORMAL edge labels; ' + 'inherited PARENT/SUB edge labels are not supported.') + target = client(args.target, 'TARGET') target_schema = target('GET', '/schema') if any(target_schema[kind] for kind in KINDS): raise SystemExit('Target schema must be empty; use a new graph.') diff --git a/helm/hugegraph/examples/seatunnel/clone/knows.conf b/helm/hugegraph/examples/seatunnel/clone/knows.conf index 8ba2259138..af3960743b 100644 --- a/helm/hugegraph/examples/seatunnel/clone/knows.conf +++ b/helm/hugegraph/examples/seatunnel/clone/knows.conf @@ -46,6 +46,7 @@ sink { { type = EDGE label = knows + timeZone = UTC frequency = MULTIPLE sortKeys = [since] sourceConfig { label = person, idFields = ["~source_id"] } diff --git a/helm/hugegraph/examples/seatunnel/clone/person.conf b/helm/hugegraph/examples/seatunnel/clone/person.conf index 91e37053b3..945739ad8a 100644 --- a/helm/hugegraph/examples/seatunnel/clone/person.conf +++ b/helm/hugegraph/examples/seatunnel/clone/person.conf @@ -46,6 +46,7 @@ sink { { type = VERTEX label = person + timeZone = UTC idStrategy = PRIMARY_KEY idFields = [name] } diff --git a/helm/hugegraph/examples/seatunnel/clone/works_at.conf b/helm/hugegraph/examples/seatunnel/clone/works_at.conf index 2575cf1be4..b8a4d5f937 100644 --- a/helm/hugegraph/examples/seatunnel/clone/works_at.conf +++ b/helm/hugegraph/examples/seatunnel/clone/works_at.conf @@ -46,6 +46,7 @@ sink { { type = EDGE label = works_at + timeZone = UTC frequency = SINGLE sourceConfig { label = person, idFields = ["~source_id"] } targetConfig { label = company, idFields = ["~target_id"] } diff --git a/helm/test_helm_seatunnel.py b/helm/test_helm_seatunnel.py index a0f340be13..ee83eee6cd 100644 --- a/helm/test_helm_seatunnel.py +++ b/helm/test_helm_seatunnel.py @@ -17,12 +17,14 @@ """Check native hybrid deployment, release isolation, connectivity and packaging.""" import base64 +import importlib.util from pathlib import Path import shutil import subprocess import tarfile import tempfile import unittest +from unittest import mock import yaml @@ -96,6 +98,25 @@ def test_disabled_has_no_engine_resources(self): self.assertEqual([], engine_resources(documents)) self.assertFalse(any("seatunnel" in doc["metadata"]["name"] for doc in documents)) + def test_schema_copy_rejects_unsupported_edges_before_contacting_target(self): + path = CHART / "examples/seatunnel/clone/copy-schema.py" + spec = importlib.util.spec_from_file_location("copy_schema", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + for label_type in ("PARENT", "SUB", "UNKNOWN"): + with self.subTest(label_type=label_type): + source = mock.Mock(return_value={"edgelabels": [ + {"name": "relation", "edgelabel_type": label_type}]}) + target = mock.Mock() + with mock.patch.object(module, "client", side_effect=[source, target]) as factory, \ + mock.patch("sys.argv", [str(path), "--source", "http://source/graph", + "--target", "http://target/graph"]): + with self.assertRaisesRegex(SystemExit, "only supports NORMAL edge labels"): + module.main() + source.assert_called_once_with("GET", "/schema") + factory.assert_called_once_with("http://source/graph", "SOURCE") + target.assert_not_called() + def test_legacy_release_without_seatunnel_values_still_renders(self): with tempfile.TemporaryDirectory(prefix="seatunnel-legacy-") as directory: chart = Path(directory) / "chart" From e79c97b13bcea0a617626a9ec96f363252590b43 Mon Sep 17 00:00:00 2001 From: orion Date: Sun, 25 Oct 2026 06:36:34 +0000 Subject: [PATCH 14/17] fix(helm): secure native SeaTunnel members Restrict the unauthenticated member port in every chart mode while keeping easy-mode HTTP and egress behavior intact. Expose the same container security and scheduling settings as other workloads without forcing an image UID or read-only filesystem. Use English operator instructions and cover policy isolation plus configuration overrides in rendered chart regressions. --- helm/hugegraph/README.md | 33 +- helm/hugegraph/docs/seatunnel.md | 326 ++++++++++++------ .../examples/seatunnel/hugegraph-sink.conf | 4 +- .../examples/seatunnel/hugegraph-source.conf | 2 +- .../examples/values-seatunnel-cluster.yaml | 4 +- .../examples/values-seatunnel-standalone.yaml | 4 +- helm/hugegraph/templates/NOTES.txt | 47 +-- .../templates/seatunnel-networkpolicy.yaml | 11 +- helm/hugegraph/templates/seatunnel.yaml | 23 ++ helm/hugegraph/values.schema.json | 18 + helm/hugegraph/values.yaml | 23 +- helm/test_helm_seatunnel.py | 49 +++ 12 files changed, 386 insertions(+), 158 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 406bae362a..e75f6c708c 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -81,18 +81,27 @@ helm install hugegraph ./helm/hugegraph -n hugegraph --create-namespace \ Inactive credential values may remain in the values file when auth is disabled. -### 可选 SeaTunnel - -增加 `--set seatunnel.enabled=true`，与 HugeGraph 一起启动一个常驻 hybrid 节点； -增加 `--set seatunnel.mode=cluster`，使用同一套官方原生 Kubernetes 资源扩为多个节点（默认两个）。 -SeaTunnel 默认关闭，拓扑独立于 HugeGraph 的单机/集群预设。两种模式均提供引擎内置的任务可视化前端； -HTTP 入口继承 `easy` 的 NodePort 或 `strict` 的 ClusterIP/NetworkPolicy，可用 `seatunnel.service` 显式覆盖并指定 NodePort。 - -Chart 自动为同 release 的演示作业注入 Server 地址、图空间和管理员 Secret 引用， -并挂载 Source/Sink 配置；先创建目标图，即可手动提交，无需复制密码或上传作业文件。 -Helm 不自动建图或导入数据。前端和 REST 提交接口使用独立管理员的原生 HTTP Basic 认证， -默认密码自动生成在 SeaTunnel Secret 中并随升级保留；它与作业连接 HugeGraph 的凭据分别配置。 -启动、访问、演示作业与跨环境 Schema／顶点／边复制见 [SeaTunnel 联合部署指南](docs/seatunnel.md)。 +### Optional SeaTunnel + +Set `seatunnel.enabled=true` to start one persistent hybrid member alongside HugeGraph; +set `seatunnel.mode=cluster` to use multiple members of the same native StatefulSet +(two by default). SeaTunnel is disabled by default and its topology is independent +of the HugeGraph single-/multi-node presets. Both modes include the engine's task UI. +The HTTP Service inherits NodePort in `easy` mode or ClusterIP in `strict` mode; +`seatunnel.service` can explicitly override the type and NodePort. + +Member/CLI port 5801 is restricted to same-release peers by a SeaTunnel ingress +NetworkPolicy in every mode, including when the chart-wide policy flag is disabled. +Easy mode leaves the authenticated HTTP entry open. Actual isolation requires CNI +support and trusted namespace RBAC; full isolation additionally limits egress. + +The chart injects the same-release Server address, GraphSpace and administrator +Secret references into the mounted Source/Sink examples. Create the target graph +before submitting them manually; Helm does not create graphs or import data. +UI and REST submission use a separate native HTTP Basic administrator account, +whose generated Secret is retained and reused during online upgrades. Container +security and scheduling options follow the other components; see the +[SeaTunnel deployment guide](docs/seatunnel.md) for configuration and limits. ### Internal image registry diff --git a/helm/hugegraph/docs/seatunnel.md b/helm/hugegraph/docs/seatunnel.md index bcaede9db1..84f36150d4 100644 --- a/helm/hugegraph/docs/seatunnel.md +++ b/helm/hugegraph/docs/seatunnel.md @@ -1,78 +1,86 @@ -# HugeGraph 与 SeaTunnel 联合部署 +# Deploy HugeGraph with SeaTunnel -本 Chart 直接复用 [SeaTunnel 3.0.0 官方 hybrid Kubernetes 方案](https://github.com/apache/seatunnel/blob/v3.0.0/docs/zh/getting-started/kubernetes/hybrid-cluster-mode.md)，在 HugeGraph Helm 中适配资源名称、DNS、镜像、配置和副本数，不修改 SeaTunnel Helm，也不引入子 Chart、Operator 或 CRD。 +This chart adapts the [SeaTunnel 3.0.0 native hybrid Kubernetes deployment](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/getting-started/kubernetes/hybrid-cluster-mode.md) +for HugeGraph resource names, DNS, images, configuration and replica counts. +It does not introduce a subchart, operator, CRD or derived SeaTunnel image. -SeaTunnel 默认关闭。启用后，一个常驻 hybrid 节点同时承担调度和执行，持续接收任务；集群模式只是把同一个 StatefulSet 扩为多个 hybrid 节点。它与执行完即退出的 `local-mode` Pod/Job 不同。 +SeaTunnel is disabled by default. When enabled, a persistent hybrid member both +schedules and executes jobs. Cluster mode scales the same StatefulSet to multiple +members; it is not a `local-mode` Pod or Job that exits after one submission. -| 配置 | 含义 | +| Setting | Meaning | |---|---| -| `values-single.yaml` | HugeGraph 1 PD + 1 Store + 1 Server，使用 HStore | -| `values-cluster.yaml` | HugeGraph 自身的多副本拓扑 | -| `seatunnel.mode: standalone` | 一个常驻 hybrid 节点 | -| `seatunnel.mode: cluster`、`replicas: 2` | 两个 hybrid 节点，副本数至少为 2 | +| `values-single.yaml` | HugeGraph 1 PD + 1 Store + 1 Server using HStore | +| `values-cluster.yaml` | HugeGraph's own multi-replica topology | +| `seatunnel.mode: standalone` | One persistent hybrid member | +| `seatunnel.mode: cluster`, `replicas: 2` | Two hybrid members; at least two are required | -HugeGraph 与 SeaTunnel 的拓扑独立选择；顶层 `mode: easy/strict` 控制同一 release 的 HTTP 入口和 NetworkPolicy 默认值。SeaTunnel 单机和集群均提供相同的内置前端与 API。 +Choose HugeGraph and SeaTunnel topologies independently. Top-level `mode: easy/strict` +controls HTTP entry and full NetworkPolicy defaults. Both SeaTunnel modes provide +the same built-in UI and API. -## 启动、访问前端与切换 +## Install, access and switch modes -在 HugeGraph 仓库根目录执行，以下两种安装方式选择一种： +From the HugeGraph repository root, choose one installation: ```bash -# HugeGraph 1 PD + 1 Store + 1 Server，加一个常驻 SeaTunnel 节点 +# HugeGraph 1 PD + 1 Store + 1 Server, plus one persistent SeaTunnel member helm install hugegraph ./helm/hugegraph -n hugegraph --create-namespace \ -f helm/hugegraph/values-single.yaml \ -f helm/hugegraph/examples/values-seatunnel-standalone.yaml \ --wait --timeout 30m -# 同样的 HugeGraph 拓扑，加两个 SeaTunnel hybrid 节点 +# The same HugeGraph topology, plus two SeaTunnel hybrid members helm install hugegraph ./helm/hugegraph -n hugegraph --create-namespace \ -f helm/hugegraph/values-single.yaml \ -f helm/hugegraph/examples/values-seatunnel-cluster.yaml \ --wait --timeout 30m ``` -默认使用 `apache/seatunnel:3.0.0`。两种模式都开启引擎自带的 HTTP 服务和任务可视化前端，无需另部署 Web 服务或数据库。 - -SeaTunnel HTTP Service 沿用 HugeGraph 的访问模式；显式设置 `seatunnel.service.type` 时优先使用指定值： - -| 配置 | HTTP Service 默认值 | 访问方式 | -|---|---|---| -| `mode: easy` | `NodePort` | 可达节点 IP 加分配的 NodePort | -| `mode: strict` | `ClusterIP`，开启 NetworkPolicy | 集群内访问，或由运维人员临时端口转发 | -| 显式 `seatunnel.service.type` | 使用指定的 `ClusterIP`、`NodePort` 或 `LoadBalancer` | 按实际 Service 地址访问 | - -例如指定 NodePort；不设置 `nodePort` 时由 Kubernetes 分配： - -```yaml -seatunnel: - enabled: true - service: - type: NodePort - nodePort: 30081 - annotations: {} -``` +The default image is `apache/seatunnel:3.0.0`. Both modes expose the UI and REST +API on HTTP 8080. Easy mode inherits a NodePort, while strict mode inherits +ClusterIP. Override `seatunnel.service.type/nodePort` explicitly when needed. +`helm install` prints access instructions; retrieve an automatically allocated +HTTP NodePort with: ```bash kubectl -n hugegraph get svc hugegraph-seatunnel \ -o jsonpath='{.spec.ports[?(@.name=="http")].nodePort}{"\n"}' -kubectl get nodes -o wide ``` -浏览器入口为 `http://<可达节点IP>:/#/overview`，API 使用同一地址的相应 REST 路径。使用 LoadBalancer 时，通过 `kubectl -n hugegraph get svc hugegraph-seatunnel -o wide` 查看分配的地址，再访问 `http://<负载均衡地址>:8080/#/overview`。kind、minikube 等本地环境需按其网络方式提供可达入口。 - -所有 Service 类型均可使用端口转发： +For any Service type, port forwarding provides a local entry: ```bash kubectl -n hugegraph port-forward svc/hugegraph-seatunnel 8081:8080 ``` -对应前端地址为 ，使用下节的独立 SeaTunnel 账号完成浏览器 Basic 登录。前端用于查看集群、作业、DAG 和日志；作业通过 SeaTunnel CLI 或受保护的 REST API 提交。Helm 只启动引擎并准备示例配置，不自动建图或执行作业。 - -外部入口只包含 HTTP 8080。Hazelcast 5801 仍仅通过内部 Headless Service 供成员发现及 CLI 通信，不随 NodePort 或 LoadBalancer 一起公开。strict 模式下，额外的应用、监控或入口代理需通过 `networkPolicy.seatunnel.extraIngress` 配置原生规则；本 Chart 不额外管理 SeaTunnel Ingress。 - -release 为 `hugegraph` 时，资源为 `hugegraph-seatunnel` StatefulSet、同名 HTTP Service、`hugegraph-seatunnel-headless` 发现 Service，以及 `hugegraph-seatunnel-configs` ConfigMap。名称遵循 HugeGraph 的 `fullnameOverride`；成员通过 Headless Service 的 DNS 发现，不访问 Kubernetes API，因此不需要发现用的 RBAC。 - -切换前先停止全部作业，保留安装使用的完整 values，再升级： +Open and use the separate SeaTunnel credentials +described below. The UI shows cluster members, jobs, DAGs and logs. Submit jobs +through the CLI or authenticated REST API. Helm starts the engine and prepares +examples; it does not create graphs or run jobs. + +Only HTTP 8080 is exposed through NodePort or LoadBalancer. A separate ingress +NetworkPolicy always restricts Hazelcast member/CLI port 5801 to same-release +SeaTunnel peers, even in easy mode or with `networkPolicy.enabled=false`. +Easy mode permits HTTP callers without adding an ingress rule and leaves egress +unrestricted. When full NetworkPolicy isolation is enabled, explicitly allow +HTTP callers and external source/sink destinations with +`networkPolicy.seatunnel.extraIngress/extraEgress`. Explicit extra ingress rules +can also broaden member access, so restrict them to trusted callers. + +Actual isolation requires a NetworkPolicy-capable CNI. Pod selectors are not +native Hazelcast authentication: use trusted namespace RBAC/admission controls +for creating or relabeling Pods and for `kubectl exec`. Do not expose 5801 outside +that administrative boundary. The chart does not manage a separate SeaTunnel Ingress. + +For release `hugegraph`, resources are the `hugegraph-seatunnel` StatefulSet and +HTTP Service, `hugegraph-seatunnel-headless` discovery Service, and +`hugegraph-seatunnel-configs` ConfigMap. Names honor `fullnameOverride`. +Members use Headless Service DNS rather than the Kubernetes API, so discovery +requires no Kubernetes API token or RBAC permission. + +Stop all jobs before switching modes. Retain the complete installation values +and upgrade with: ```bash helm upgrade hugegraph ./helm/hugegraph -n hugegraph -f my-values.yaml \ @@ -80,15 +88,28 @@ helm upgrade hugegraph ./helm/hugegraph -n hugegraph -f my-values.yaml \ --set seatunnel.replicas=2 --wait --timeout 30m ``` -切回单机设置 `seatunnel.mode=standalone`。两种模式使用同一个 StatefulSet，但副本和引擎配置会变化；不承诺无中断切换，也不会自动恢复或重新提交作业。设置 `seatunnel.enabled=false` 移除本 release 的 SeaTunnel 工作负载和 Service，已经写入 HugeGraph 的数据保留。认证 Secret 带有保留策略，禁用或卸载时仍会保留，便于再次启用时复用凭据。 +Set `seatunnel.mode=standalone` to return to one member. The StatefulSet stays the +same, but replica count and engine configuration change. Uninterrupted switching, +automatic job recovery and resubmission are not guaranteed. Setting +`seatunnel.enabled=false` removes the engine workloads and Services without +deleting data already written to HugeGraph. The retained auth Secret can be +reused after re-enabling or reinstalling. -## SeaTunnel 入口认证 +## SeaTunnel entry authentication -前端、静态资源和 REST 提交接口默认使用引擎原生 HTTP Basic 认证，不新增认证代理或独立登录服务。SeaTunnel 使用独立入口管理员：`seatunnel.auth.username` 默认为 `admin`；密码未显式设置时，Chart 自动生成强密码并保存在 `-auth` Secret，升级时复用已有密码。 +The UI, static resources and REST submission endpoints use native HTTP Basic +authentication. There is no added proxy or login service. `seatunnel.auth.username` +defaults to `admin`. An empty password generates a strong password in +`-auth`; online upgrades reuse the existing password. -也可通过受控 values 文件显式设置 `seatunnel.auth.username/password`，用于自定义账号或轮换。自定义值不能有首尾空白或 `${...}` 占位符，用户名另不能包含冒号；内部空格和引号仍可使用。只做离线模板或 GitOps 渲染时，应显式固定 `seatunnel.auth.password`；默认密码的升级复用依赖在线 Helm lookup。 +Use a controlled values file to set or rotate `seatunnel.auth.username/password`. +Values cannot contain leading/trailing whitespace or `${...}` placeholders, and +usernames cannot contain a colon. Internal spaces and quotes are accepted. +Pin `seatunnel.auth.password` for offline templates or GitOps rendering; automatic +password reuse requires an online Helm lookup. -默认 release 的 Secret 为 `hugegraph-seatunnel-auth`。保持前述端口转发，在本机读取账号和密码后调用 REST： +The default release uses Secret `hugegraph-seatunnel-auth`. With port forwarding +active, read its credentials into local variables and call REST: ```bash ST_USERNAME="$(kubectl -n hugegraph get secret hugegraph-seatunnel-auth \ @@ -99,31 +120,55 @@ ST_PASSWORD="$(kubectl -n hugegraph get secret hugegraph-seatunnel-auth \ curl -fsS --user "$ST_USERNAME:$ST_PASSWORD" http://127.0.0.1:8081/overview ``` -浏览器打开 `/#/overview` 时，使用 Secret 中的同一组账号密码完成 HTTP Basic 登录；访问其他 NodePort 或 LoadBalancer 地址也使用这组入口凭据。外部提交作业使用同一受保护 HTTP 入口，并按 [官方 REST API](https://github.com/apache/seatunnel/blob/v3.0.0/docs/zh/engines/zeta/rest-api-and-web-ui.md) 的格式携带认证信息。 - -入口 Secret 包含 `username`、`password` 及最终的 `seatunnel.yaml`；包含认证信息的引擎文件直接从 Secret 挂载，不进入 ConfigMap、Java 参数或日志。轮换时在受控 values 中指定新密码并执行 `helm upgrade`，同步更新 Secret 内容并滚动重启；只修改 Secret 的 `data.password` 不会同步修改其中的引擎文件。`HG_USERNAME/HG_PASSWORD` 是作业连接 HugeGraph 的另一组凭据，不会替代入口认证。即使 `server.auth.enabled=false`，SeaTunnel 入口仍使用独立认证。 - -HTTP Basic 不覆盖 Hazelcast 5801 的内部成员和 CLI 协议。下文在 Pod 内执行的 `seatunnel.sh --master cluster` 属于可信 Kubernetes/RBAC 管理范围；外部客户端应使用受保护的 REST 入口，不能把 5801 对外公开。本配置提供供可信管理人员使用的单一管理员入口，不提供按用户或作业划分的细粒度权限，也不复用 HugeGraph 的多用户权限、RBAC 或 SSO。需要对外服务时可接入已有 TLS 入口；本 Chart 不新增 Ingress 管理。 - -## 镜像、凭据与原生配置 - -本轮已直接核实官方 `apache/seatunnel:3.0.0` 镜像自带 HugeGraph、FakeSource、Console 连接器和内置 UI，直接复用即可，无需维护派生镜像。更换版本或自定义镜像时再核对插件与依赖。镜像通过 `seatunnel.image.repository/tag/pullPolicy` 选择，继承 HugeGraph 的 `global.imageRegistry` 和 `imagePullSecrets`；Helm 不在启动时下载插件。官方镜像包含大量连接器，本次 amd64 镜像压缩后约 3.7GB；首次拉取较慢时可延长安装 timeout，或预拉取、镜像到内部仓库。 - -启用 SeaTunnel 后，Chart 为同 release 的 HugeGraph 演示作业准备以下环境变量，无需手写 Server 地址或复制管理员密码： - -| 变量 | 默认来源 | +Use the same credentials for the browser Basic login and any NodePort or +LoadBalancer entry. External job submissions must authenticate to the protected +HTTP entry using the [official REST API](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/engines/zeta/rest-api-and-web-ui.md). + +The Secret contains `username`, `password` and the final `seatunnel.yaml`. +The engine configuration is mounted directly from the Secret, not embedded in +a ConfigMap, Java arguments or logs. Rotate through controlled values and +`helm upgrade`, which updates the Secret and rolls the engine Pods. Editing only +`data.password` does not update credentials inside the engine configuration. +`HG_USERNAME/HG_PASSWORD` are separate HugeGraph connector credentials and cannot +replace entry authentication. Disabling `server.auth.enabled` does not disable +SeaTunnel authentication. + +HTTP Basic does not authenticate the Hazelcast member/CLI protocol on 5801. +The examples below execute the CLI inside an engine Pod under Kubernetes RBAC; +external clients should use REST. This is a single administrator entry, not +per-user/per-job authorization or HugeGraph RBAC/SSO. Use an existing TLS entry +when exposing the HTTP service outside a trusted network. + +## Images, credentials and native configuration + +The official `apache/seatunnel:3.0.0` image used by this chart includes the +HugeGraph, FakeSource and Console connectors and the built-in UI. Verify plugin +availability when changing the version or image. Configure +`seatunnel.image.repository/tag/pullPolicy`; it inherits `global.imageRegistry` +and `imagePullSecrets`. Helm does not download plugins at startup. The upstream +image contains many connectors, so allow sufficient pull time or pre-pull/mirror +it into an internal registry. + +The chart provides these defaults for same-release HugeGraph example jobs: + +| Variable | Default source | |---|---| -| `HG_HOST` | 当前 release 的 HugeGraph Server Service DNS | +| `HG_HOST` | Same-release HugeGraph Server Service DNS | | `HG_PORT` | `server.port` | -| `HG_PROTOCOL` | `http`，访问集群内部 Server | +| `HG_PROTOCOL` | `http` for the internal Server connection | | `HG_USERNAME` | `admin` | -| `HG_PASSWORD` | 当前 Server 管理员 Secret 的实际名称与键；认证关闭时为空密码 | +| `HG_PASSWORD` | Actual Server administrator Secret name/key, or empty when auth is disabled | | `HG_GRAPH_SPACE` | `server.graphSpace` | | `HG_GRAPH_NAME` | `graph_a` | -`HG_PASSWORD` 使用原生 `secretKeyRef`，兼容 `server.auth.admin.existingSecret` 及其自定义键，不把密码写入演示文件。Chart 不因此创建额外的 HugeGraph 账号；更新密码 Secret 后，按 Server 账号同步流程处理，再重启相关 SeaTunnel Pod 刷新环境变量。 +`HG_PASSWORD` uses a native `secretKeyRef`, including +`server.auth.admin.existingSecret` and custom keys. It is not copied into example +job files. The chart does not create an extra HugeGraph account. Follow the +Server account synchronization procedure after rotating its credential Secret, +then restart relevant engine Pods to refresh their environment. -`seatunnel.env` 中同名条目显式覆盖自动值，最终每个变量只有一个有效条目。例如更换演示图，或使用已有的专用作业账号： +A matching name in `seatunnel.env` overrides an automatic value without creating +duplicate entries. For example, select another graph and an existing job account: ```yaml seatunnel: @@ -139,9 +184,22 @@ seatunnel: key: password ``` -专用账号需预先存在，并具备所用图的数据和 Schema 权限。这组 `HG_*` 默认值只服务于本 release 的简单 Source/Sink 示例；跨环境复制仍由用户提供下文的 `HG_SOURCE_*`、`HG_TARGET_*`，不会自动发现或配置远端图。SeaTunnel 前端/API 自身的认证也不能由这组连接器变量替代。 +The account must already exist and have the required data/Schema permissions. +These defaults serve only the local Source/Sink examples. Cross-environment jobs +require the explicit `HG_SOURCE_*` and `HG_TARGET_*` values below; the chart does +not discover remote graphs. Connector credentials do not replace entry auth. -`seatunnel.javaOpts` 调整引擎 JVM，`resources` 调整容器资源。高级配置保留 SeaTunnel 原生文件格式，通过 `seatunnel.config` 按文件名完整覆盖，不逐项增加 Helm 开关，例如： +Tune the JVM with `seatunnel.javaOpts` and container resources with +`seatunnel.resources`. `seatunnel.podSecurityContext` and +`seatunnel.securityContext` follow the other chart components. The container +starts with privilege escalation disabled, all capabilities dropped and +`RuntimeDefault` seccomp. The upstream image runs as root: configure a compatible +UID and writable paths when enforcing `runAsNonRoot` or a restricted Pod Security +namespace; these defaults alone do not establish compatibility with that policy. +Use `seatunnel.nodeSelector`, `tolerations`, `affinity` and `priorityClassName` for +node placement and scheduling. + +Override complete native configuration files through `seatunnel.config`, for example: ```yaml seatunnel: @@ -152,23 +210,33 @@ seatunnel: -Xmx256m ``` -默认在 ConfigMap 中准备 `hazelcast.yaml`、`hazelcast-client.yaml`、JVM 和演示作业文件；包含入口认证的 `seatunnel.yaml` 从独立 Secret 挂载，其余配置沿用镜像。`seatunnel.config` 可覆盖引擎文件，但最终 HTTP Basic 开关和凭据始终由 `seatunnel.auth` 管理。覆盖发现或引擎文件时保留 TCP 5801、HTTP 8080 和 `enable-http: true`，才能继续使用现有 Service、探针及前端。默认 checkpoint 使用临时本地目录；本集成不提供持久化作业恢复保证。 - -strict 模式允许 SeaTunnel 成员通信、DNS，以及访问同 release 的 HugeGraph Server。远端图 A、其他数据源及调用方，通过 `networkPolicy.seatunnel.extraEgress/extraIngress` 补充原生规则；实际隔离依赖 CNI 支持。HugeGraph 的凭据、存储和访问方式见 [Chart README](../README.md)。 - -## 同 release 演示：直接导入并读取 - -Chart 启动时把 [hugegraph-sink.conf](../examples/seatunnel/hugegraph-sink.conf) 和 [hugegraph-source.conf](../examples/seatunnel/hugegraph-source.conf) 放入现有的 SeaTunnel ConfigMap，分别挂载为： - -- `/opt/seatunnel/config/hugegraph-sink.conf`：`FakeSource → HugeGraph`，向 `seatunnel_demo_person` 标签写入十条固定数据。 -- `/opt/seatunnel/config/hugegraph-source.conf`：读取同一图中的演示标签，输出到 Console。 - -两个文件均使用前述自动 `HG_*` 环境变量；无需上传临时文件或手动补全当前 Server 地址、端口、图空间及密码。需要调整作业时，可在 `seatunnel.config` 中按同名文件完整覆盖。 - -先通过 Hubble 或 HugeGraph API 创建 `graph_a`。如果改写了 `HG_GRAPH_SPACE/HG_GRAPH_NAME`，则创建对应图。以下命令按默认 release、命名空间及图空间演示建图；本机的 `HG_USERNAME/HG_PASSWORD` 由凭据管理器提供，Pod 环境变量不会传到本机： +The ConfigMap supplies discovery, JVM and example job files. The authenticated +`seatunnel.yaml` is mounted from its separate Secret; other files use the image +defaults. Final HTTP Basic settings remain managed by `seatunnel.auth` even when +overriding engine configuration. Retain member port 5801, HTTP 8080 and +`enable-http: true` so Services, probes and UI remain usable. Checkpoints use an +ephemeral local directory; this integration does not guarantee durable recovery. +Full NetworkPolicy isolation allows peer traffic, DNS and the same-release +Server API by default. See the [Chart README](../README.md) for HugeGraph access, +credentials and storage. + +## Same-release examples: import and read + +The chart mounts [hugegraph-sink.conf](../examples/seatunnel/hugegraph-sink.conf) +and [hugegraph-source.conf](../examples/seatunnel/hugegraph-source.conf) as +`/opt/seatunnel/config/hugegraph-sink.conf` and +`/opt/seatunnel/config/hugegraph-source.conf`. The Sink writes ten fixed vertices +with label `seatunnel_demo_person`; the Source reads that label into Console. +Both use the automatic `HG_*` environment values. Override a file under the same +name in `seatunnel.config` to customize a job. + +Create `graph_a` through Hubble or the HugeGraph API first, or create the graph +selected by explicit `HG_GRAPH_SPACE/HG_GRAPH_NAME` values. The default release, +namespace and GraphSpace are used below. Local `HG_USERNAME/HG_PASSWORD` must +come from your credential manager; Pod environment values do not reach your shell. ```bash -# 在另一个终端保持 Server 端口转发 +# Keep Server port forwarding active in a separate terminal kubectl -n hugegraph port-forward svc/hugegraph-server 8082:8080 ``` @@ -178,7 +246,7 @@ curl -fsS -u "$HG_USERNAME:$HG_PASSWORD" -H 'Content-Type: application/json' \ -X POST "$HG_URL/graphspaces/DEFAULT/graphs/graph_a" -d '{}' ``` -已有图不重复建图。随后直接提交已挂载的 Sink，再提交 Source： +Do not recreate an existing graph. Submit the mounted Sink, then Source: ```bash kubectl -n hugegraph exec sts/hugegraph-seatunnel -- \ @@ -190,7 +258,10 @@ kubectl -n hugegraph exec sts/hugegraph-seatunnel -- \ --config /opt/seatunnel/config/hugegraph-source.conf ``` -`--master cluster` 表示提交给常驻引擎，单节点也使用该参数。首次空图导入后，Sink 作业应为 `FINISHED`，读取、写入各十条；Source 应读取十条并输出到执行节点的日志。可先在前端查看作业状态和 DAG，再检查各 hybrid Pod 日志及图中实际数据： +`--master cluster` submits to the persistent engine, including standalone mode. +On an initially empty graph, expect a finished Sink with ten rows read/written, +and a Source with ten rows read into the executing member's log. Inspect UI +status/DAG, all member logs and the actual graph data: ```bash kubectl -n hugegraph logs -l app.kubernetes.io/instance=hugegraph,app.kubernetes.io/component=seatunnel \ @@ -200,27 +271,42 @@ curl -fsS -u "$HG_USERNAME:$HG_PASSWORD" -H 'Content-Type: application/json' \ "$HG_URL/gremlin" -d '{"gremlin":"graph.traversal().V().hasLabel(\"seatunnel_demo_person\").count()","aliases":{"graph":"DEFAULT-graph_a"}}' ``` -数量查询的 `result.data` 应为 `[10]`。集群模式下作业可能由任一 hybrid 节点执行，因此 Console 输出不限定在 `hugegraph-seatunnel-0`。这两个演示作业也不会在 Helm 安装或升级时自动执行。 +The count query should return `result.data: [10]`. A cluster job can execute on +any member, so Console output is not limited to `hugegraph-seatunnel-0`. Neither +example runs automatically during Helm installation or upgrade. -## 新环境复制：Schema、顶点和边 +## Cross-environment copy: Schema, vertices and edges -这里使用已有图 A 和另一个环境中的空图 B，不接续上面的十条数据示例。复制顺序是 **Schema → 顶点 → 边**。SeaTunnel Sink 的自动建 Schema 只能根据映射和行类型创建所需结构，不能替代源图完整 Schema、索引及 ID 的复制。 +Use an existing source graph A and an empty graph B in another environment. +Copy **Schema, then vertices, then edges**. Sink auto-schema mapping cannot +replace copying the complete source Schema, indexes and IDs. -先通过 HugeGraph 原生 API 创建空的 `graph_b`，再执行 [copy-schema.py](../examples/seatunnel/clone/copy-schema.py)。脚本在目标图的 RESTORING 状态下保留源 Schema ID，复制属性键、顶点标签、边标签和索引；不传输图数据。源、目标应使用兼容的 HugeGraph 版本，账号需有相应管理权限，目标 Schema 必须为空。 +Create `graph_b` through the native API, then run +[copy-schema.py](../examples/seatunnel/clone/copy-schema.py). It copies property +keys, vertex labels, edge labels and indexes with source Schema IDs while the +target is in RESTORING mode. It does not copy graph data. Use compatible HugeGraph +versions and accounts with administrative permissions; target Schema must be empty. -这份示例只支持普通边标签。遇到 `PARENT`、`SUB` 或未知边标签类型时，脚本会在访问目标图前退出；继承边需要按父子关系另行迁移。 +Only NORMAL edge labels are supported. The script rejects PARENT, SUB or unknown +edge types before contacting the target; inherited Schema needs a separate +migration procedure. ```bash -# 两个 URL 均包含图空间及图名；对应账号密码由本机环境提供 +# Both URLs include GraphSpace and graph; credentials come from your local environment export HG_SOURCE_GRAPH_URL=https://source.example.com/graphspaces/DEFAULT/graphs/graph_a export HG_TARGET_GRAPH_URL=https://target.example.com/graphspaces/DEFAULT/graphs/graph_b python3 helm/hugegraph/examples/seatunnel/clone/copy-schema.py \ --source "$HG_SOURCE_GRAPH_URL" --target "$HG_TARGET_GRAPH_URL" ``` -脚本读取 `HG_SOURCE_USERNAME/HG_SOURCE_PASSWORD` 和 `HG_TARGET_USERNAME/HG_TARGET_PASSWORD`。随后在 SeaTunnel Pod 中配置相同的两组凭据，以及 `HG_SOURCE_HOST/PORT/PROTOCOL`、`HG_TARGET_HOST/PORT/PROTOCOL`；密码继续用 Secret 注入。源、目标地址必须能从 SeaTunnel Pod 访问，不能填写仅在本机可达的端口转发地址。 +The script reads `HG_SOURCE_USERNAME/HG_SOURCE_PASSWORD` and +`HG_TARGET_USERNAME/HG_TARGET_PASSWORD`. Configure corresponding credentials in +the engine Pods through Secrets, plus `HG_SOURCE_HOST/PORT/PROTOCOL` and +`HG_TARGET_HOST/PORT/PROTOCOL`. Addresses must be reachable from the engine Pods, +not merely local port-forward addresses. -四个示例作业读取 `graph_a`、写入 `graph_b`，先复制两个顶点标签，再复制两类边： +The four example jobs read `graph_a` and write `graph_b`, copying both vertex +labels before either edge label: ```bash for clone_job in person company works_at knows; do @@ -233,20 +319,38 @@ for clone_job in person company works_at knows; do done ``` -作业使用 `ERROR_WHEN_SCHEMA_NOT_EXIST`，复用已经复制的 Schema；边依赖先前写入的端点顶点。这组映射覆盖 [person](../examples/seatunnel/clone/person.conf)、[company](../examples/seatunnel/clone/company.conf)、[works_at](../examples/seatunnel/clone/works_at.conf) 和 [knows](../examples/seatunnel/clone/knows.conf)。官方 Sink 要求显式提供 `mappings` 或 `schema_config`。复制其他图时需按实际标签、属性和 ID 策略调整原生映射，不会自动枚举全图并提交迁移作业。 - -这组示例验证 `PRIMARY_KEY` 和 `CUSTOMIZE_NUMBER` 顶点 ID 策略。现有连接器对 `AUTOMATIC` 顶点会生成新 ID，不能用这组映射直接无损复制其边端点；迁移前先核对源图的 ID 策略。 - -四份映射均显式设置 `timeZone = UTC`，假设源 Server 的 `DATE` 字符串按 UTC 序列化，避免复制结果随 SeaTunnel worker 的默认时区变化。 -源 Server 使用其他时区时，应把每个 Sink `mappings` 条目中的 `timeZone` 改为源 Server 的实际时区。 -Source 的 `time_zone` 不能修正 REST 返回的无时区日期字符串；含 `DATE` 属性时，验收还需比较原始时间戳。 - -验收应同时核对属性键类型、基数、标签、主键、边端点定义、索引和 Schema ID，以及全部顶点 ID/属性、边 ID/端点/属性；只比较数量或作业状态不够。相关选项见官方 [HugeGraph Source](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/source/HugeGraph.md) 和 [Sink](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/sink/HugeGraph.md)。 - -## 使用限制 - -集群模式下，应逐个 hybrid Pod 确认 Workers 和成员数与副本数一致；仅 Pod Ready 不能说明成员已组成同一个集群。Hazelcast DNS 发现使用模板中的 `service-port: 5801`，应与引擎监听端口保持一致。 - -- 示例支持 `PRIMARY_KEY`、`CUSTOMIZE_NUMBER`，不支持 `AUTOMATIC` 顶点的无损复制；后者需要解决新旧顶点 ID 与边端点的对应关系。 -- 官方 3.0.0 UI 的 Duration 显示存在上游问题：无时区时间字符串在新加坡浏览器中会产生 8 小时偏差，且已完成作业的显示时长仍随当前时间增长。以 CLI/REST 的完成状态及开始、结束时间核对实际结果。 -- 模式切换前应停止作业；本集成不承诺运行中作业无中断切换、生产 HA 或持久化恢复。默认临时 checkpoint 目录不是持久化恢复方案，NetworkPolicy 的实际隔离依赖 CNI 支持。 +Mappings use `ERROR_WHEN_SCHEMA_NOT_EXIST` and the already copied Schema. Edges +require their endpoint vertices first. See [person](../examples/seatunnel/clone/person.conf), +[company](../examples/seatunnel/clone/company.conf), [works_at](../examples/seatunnel/clone/works_at.conf) +and [knows](../examples/seatunnel/clone/knows.conf). The official Sink requires +explicit `mappings` or `schema_config`; adapt labels, properties and ID strategies +to your graph. These examples do not enumerate or migrate every graph automatically. + +The mappings support PRIMARY_KEY and CUSTOMIZE_NUMBER vertex IDs. AUTOMATIC +vertices receive new IDs, so these mappings cannot preserve their edge endpoints +losslessly. Check source ID strategies before migration. + +Every Sink mapping sets `timeZone = UTC`, assuming UTC-serialized DATE strings +from the source Server. Set each mapping to the source Server's actual time zone +when different. Source `time_zone` cannot repair timezone-free REST date strings; +include original timestamps in acceptance checks for DATE properties. + +Compare Schema types, cardinalities, labels, primary keys, endpoint definitions, +indexes and Schema IDs, plus every vertex ID/property and edge ID/endpoint/property. +Counts or job status alone are insufficient. See the official +[HugeGraph Source](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/source/HugeGraph.md) +and [Sink](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/sink/HugeGraph.md) options. + +## Limits + +Check the Workers and member counts on every hybrid Pod against the configured +replicas; Pod Ready alone does not establish a shared engine cluster. DNS +discovery's `service-port: 5801` must match the listening member port. + +- PRIMARY_KEY and CUSTOMIZE_NUMBER examples do not provide lossless AUTOMATIC-ID + copies; these require an explicit mapping of generated IDs and edge endpoints. +- The upstream 3.0.0 UI Duration display has a known timezone/completed-job issue. + Use CLI/REST status and start/end timestamps to assess actual completion. +- Stop jobs before mode changes. Uninterrupted switching, production HA and + persistent recovery are not guaranteed. Temporary checkpoints are not a durable + recovery design, and actual NetworkPolicy enforcement depends on the CNI. diff --git a/helm/hugegraph/examples/seatunnel/hugegraph-sink.conf b/helm/hugegraph/examples/seatunnel/hugegraph-sink.conf index fbfb685b1f..7b408a2c14 100644 --- a/helm/hugegraph/examples/seatunnel/hugegraph-sink.conf +++ b/helm/hugegraph/examples/seatunnel/hugegraph-sink.conf @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# 先创建 HG_GRAPH_SPACE/HG_GRAPH_NAME 指定的图，默认 DEFAULT/graph_a。 -# Helm 自动配置同 release 的 HugeGraph 地址与 Secret 凭据。 +# First create the graph selected by HG_GRAPH_SPACE/HG_GRAPH_NAME (default DEFAULT/graph_a). +# Helm configures the same-release HugeGraph address and Secret credential references. env { job.mode = "BATCH" parallelism = 1 diff --git a/helm/hugegraph/examples/seatunnel/hugegraph-source.conf b/helm/hugegraph/examples/seatunnel/hugegraph-source.conf index 622e5effa6..e28a8fa148 100644 --- a/helm/hugegraph/examples/seatunnel/hugegraph-source.conf +++ b/helm/hugegraph/examples/seatunnel/hugegraph-source.conf @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# 先运行 hugegraph-sink.conf，再读取同一图中的十条演示顶点。 +# Run hugegraph-sink.conf first, then read the ten example vertices from the same graph. env { job.mode = BATCH parallelism = 1 diff --git a/helm/hugegraph/examples/values-seatunnel-cluster.yaml b/helm/hugegraph/examples/values-seatunnel-cluster.yaml index 9e6fe30cb0..25308032af 100644 --- a/helm/hugegraph/examples/values-seatunnel-cluster.yaml +++ b/helm/hugegraph/examples/values-seatunnel-cluster.yaml @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# 复用官方原生 hybrid StatefulSet；HTTP8080 同时提供 REST 和任务可视化前端。 -# HugeGraph 拓扑独立选择，1 PD + 1 Store + 1 Server 可用 ../values-single.yaml。 +# Use the official native hybrid StatefulSet; HTTP 8080 serves REST and the task UI. +# Choose HugeGraph topology independently; ../values-single.yaml uses 1 PD + 1 Store + 1 Server. seatunnel: enabled: true mode: cluster diff --git a/helm/hugegraph/examples/values-seatunnel-standalone.yaml b/helm/hugegraph/examples/values-seatunnel-standalone.yaml index e75efd6a58..91a79ba511 100644 --- a/helm/hugegraph/examples/values-seatunnel-standalone.yaml +++ b/helm/hugegraph/examples/values-seatunnel-standalone.yaml @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# 复用官方原生 hybrid StatefulSet；HTTP8080 同时提供 REST 和任务可视化前端。 -# HugeGraph 拓扑独立选择，1 PD + 1 Store + 1 Server 可用 ../values-single.yaml。 +# Use the official native hybrid StatefulSet; HTTP 8080 serves REST and the task UI. +# Choose HugeGraph topology independently; ../values-single.yaml uses 1 PD + 1 Store + 1 Server. seatunnel: enabled: true mode: standalone diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt index 0db01562bc..3ec7e7bc8d 100644 --- a/helm/hugegraph/templates/NOTES.txt +++ b/helm/hugegraph/templates/NOTES.txt @@ -64,50 +64,51 @@ Access: SeaTunnel: {{- $stName := include "hugegraph.seatunnel.name" . }} {{- $stService := .Values.seatunnel.service | default dict }} - 集群内 API: http://{{ $stName }}.{{ .Release.Namespace }}.svc:8080 + In-cluster API: http://{{ $stName }}.{{ .Release.Namespace }}.svc:8080 {{- if eq $stService.type "NodePort" }} HTTP NodePort: kubectl get svc -n {{ .Release.Namespace }} {{ $stName }} -o jsonpath='{.spec.ports[?(@.name=="http")].nodePort}{"\n"}' kubectl get nodes -o wide - 前端: http://<可达节点IP>:{{ $stService.nodePort | default "" }}/#/overview - API: http://<可达节点IP>:{{ $stService.nodePort | default "" }} + UI: http://:{{ $stService.nodePort | default "" }}/#/overview + API: http://:{{ $stService.nodePort | default "" }} {{- else if eq $stService.type "LoadBalancer" }} - 负载均衡地址: + LoadBalancer address: kubectl get svc -n {{ .Release.Namespace }} {{ $stName }} -o wide kubectl get svc -n {{ .Release.Namespace }} {{ $stName }} -o jsonpath='{.status.loadBalancer.ingress}{"\n"}' - 地址分配后访问前端 http://<负载均衡地址>:8080/#/overview，API 使用同一 HTTP 地址。 + Once assigned, open http://:8080/#/overview; the API uses the same HTTP address. {{- else }} - HTTP Service 为 ClusterIP；集群外访问可使用下方端口转发。 + HTTP Service is ClusterIP; use port forwarding below for access outside the cluster. {{- end }} - 所有 Service 类型均可端口转发: + Port forwarding works with every Service type: kubectl port-forward -n {{ .Release.Namespace }} svc/{{ $stName }} 8081:8080 - 本机前端: http://127.0.0.1:8081/#/overview - 本机 API: http://127.0.0.1:8081 - Hazelcast 5801 仅用于集群内部发现及 CLI 通信，不通过以上外部入口公开。 - 前端、静态资源和 REST 提交接口使用独立 SeaTunnel 账号的 HTTP Basic 认证。 - 以下命令读取凭据到本机变量，不在安装输出中显示密码: + Local UI: http://127.0.0.1:8081/#/overview + Local API: http://127.0.0.1:8081 + The SeaTunnel NetworkPolicy restricts member/CLI port 5801 to same-release peers in every mode. + Enforcement requires a NetworkPolicy-capable CNI; keep namespace labels, pod creation and exec under trusted RBAC. + The UI, static resources and REST submission API use a separate SeaTunnel HTTP Basic account. + Read credentials into local variables without printing the password in installation output: ST_USERNAME="$(kubectl get secret -n {{ .Release.Namespace }} {{ $stName }}-auth -o jsonpath='{.data.username}' | base64 --decode)" ST_PASSWORD="$(kubectl get secret -n {{ .Release.Namespace }} {{ $stName }}-auth -o jsonpath='{.data.password}' | base64 --decode)" - 浏览器 Basic 登录使用此 Secret 中的账号密码；保持端口转发后可验证 REST: + Use this account for the browser Basic login; with port forwarding active, check REST with: curl --user "${ST_USERNAME}:${ST_PASSWORD}" http://127.0.0.1:8081/overview - 该入口账号独立于 HugeGraph 连接器账号；关闭 HugeGraph 认证也不会关闭 SeaTunnel 入口认证。 - 内部 CLI/5801 不经 HTTP Basic，仅在可信 Kubernetes/RBAC 范围使用，外部提交走受保护的 REST。 - 默认凭据随在线 Helm 升级保留；轮换用受控 values 的 seatunnel.auth.password，再 helm upgrade。 - GitOps/template-only 渲染请显式固定密码；认证 Secret 在禁用或卸载后仍保留。 - 账号、轮换及作业说明: helm/hugegraph/docs/seatunnel.md + These entry credentials are separate from HugeGraph connector credentials, even when HugeGraph auth is disabled. + Internal CLI/5801 has no HTTP Basic authentication. External submissions must use the protected REST API. + Online Helm upgrades retain the default credentials; rotate using seatunnel.auth.password in controlled values. + Pin a password for GitOps/template-only rendering. The auth Secret is retained after disabling or uninstalling. + Authentication, rotation and jobs: helm/hugegraph/docs/seatunnel.md {{- if and .Values.networkPolicy.enabled (has $stService.type (list "NodePort" "LoadBalancer")) }} {{- $stRules := get .Values.networkPolicy "seatunnel" | default dict }} {{- if empty (get $stRules "extraIngress") }} - NetworkPolicy 已开启；外部 HTTP 调用方还需通过 networkPolicy.seatunnel.extraIngress 允许。 + Full NetworkPolicy isolation is enabled; allow external HTTP callers through networkPolicy.seatunnel.extraIngress. {{- end }} {{- end }} - 同 release 演示作业: - 已注入 Server 地址、端口、图空间及凭据引用，示例配置已挂载，无需上传临时文件。 - 先创建目标图（默认 {{ .Values.server.graphSpace }}/graph_a；HG_* 显式覆盖时以覆盖值为准），再手动提交: + Same-release example jobs: + Server address, port, GraphSpace and credential references are injected; the job files are already mounted. + First create the target graph (default {{ .Values.server.graphSpace }}/graph_a; explicit HG_* values override it), then submit: kubectl exec -n {{ .Release.Namespace }} sts/{{ $stName }} -- /opt/seatunnel/bin/seatunnel.sh --master cluster --config /opt/seatunnel/config/hugegraph-sink.conf kubectl exec -n {{ .Release.Namespace }} sts/{{ $stName }} -- /opt/seatunnel/bin/seatunnel.sh --master cluster --config /opt/seatunnel/config/hugegraph-source.conf - Helm 不自动建图或执行作业；跨环境 clone 仍需用户提供 HG_SOURCE_* / HG_TARGET_*。 + Helm does not create graphs or run jobs. Cross-environment copies still require HG_SOURCE_* / HG_TARGET_* values. {{- end }} {{- range $component := list "server" "hubble" "pd" }} {{- if or (ne $component "hubble") $hubbleEnabled }} diff --git a/helm/hugegraph/templates/seatunnel-networkpolicy.yaml b/helm/hugegraph/templates/seatunnel-networkpolicy.yaml index b9746c95f5..faddf24c86 100644 --- a/helm/hugegraph/templates/seatunnel-networkpolicy.yaml +++ b/helm/hugegraph/templates/seatunnel-networkpolicy.yaml @@ -14,7 +14,7 @@ # limitations under the License. {{ include "hugegraph.resolveValues" . }} -{{- if and (.Values.seatunnel | default dict).enabled .Values.networkPolicy.enabled }} +{{- if (.Values.seatunnel | default dict).enabled }} {{- $rules := get .Values.networkPolicy "seatunnel" | default dict }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy @@ -29,15 +29,23 @@ spec: {{- include "hugegraph.netpol.seatunnelLabels" . | nindent 6 }} policyTypes: - Ingress + {{- if .Values.networkPolicy.enabled }} - Egress + {{- end }} ingress: - from: {{- include "hugegraph.netpol.seatunnelPeer" . | nindent 8 }} ports: {{- include "hugegraph.netpol.ports" (list 5801 8080) | nindent 8 }} + {{- if not .Values.networkPolicy.enabled }} + # Easy mode keeps the authenticated HTTP entry open, never the member port. + - ports: + {{- include "hugegraph.netpol.ports" (list 8080) | nindent 8 }} + {{- end }} {{- with get $rules "extraIngress" }} {{- toYaml . | nindent 4 }} {{- end }} + {{- if .Values.networkPolicy.enabled }} egress: - to: {{- include "hugegraph.netpol.seatunnelPeer" . | nindent 8 }} @@ -51,4 +59,5 @@ spec: {{- with get $rules "extraEgress" }} {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} {{- end }} diff --git a/helm/hugegraph/templates/seatunnel.yaml b/helm/hugegraph/templates/seatunnel.yaml index 75c425b48f..fabe1f269e 100644 --- a/helm/hugegraph/templates/seatunnel.yaml +++ b/helm/hugegraph/templates/seatunnel.yaml @@ -174,10 +174,33 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.seatunnel.priorityClassName }} + priorityClassName: {{ . | quote }} + {{- end }} + {{- with .Values.seatunnel.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.seatunnel.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.seatunnel.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.seatunnel.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: seatunnel image: {{ include "hugegraph.image" (dict "image" .Values.seatunnel.image "appVersion" "3.0.0") | quote }} imagePullPolicy: {{ .Values.seatunnel.image.pullPolicy }} + {{- with .Values.seatunnel.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} # The default upstream role is hybrid; an explicit -r is unnecessary. command: ["/opt/seatunnel/bin/seatunnel-cluster.sh"] env: diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 75006a0a52..4c0713c864 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -93,6 +93,24 @@ "type": "string" } } + }, + "podSecurityContext": { + "type": "object" + }, + "securityContext": { + "type": "object" + }, + "nodeSelector": { + "type": "object" + }, + "tolerations": { + "type": "array" + }, + "affinity": { + "type": "object" + }, + "priorityClassName": { + "type": "string" } } } diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index e6be800629..15a2784086 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -563,6 +563,19 @@ seatunnel: memory: 1536Mi limits: memory: 2Gi + # Match the other workloads. The upstream image runs as root; configure a + # compatible UID and writable paths before enabling runAsNonRoot. + podSecurityContext: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + nodeSelector: {} + tolerations: [] + affinity: {} + priorityClassName: "" # Native Kubernetes env entries; matching names override automatic HG_* values. # HG_HOST/PORT/PROTOCOL, HG_USERNAME/PASSWORD and HG_GRAPH_SPACE/NAME default to # this release's Server, existing admin Secret, graph space and graph_a. @@ -572,8 +585,9 @@ seatunnel: config: {} # Kubernetes NetworkPolicy: mode default plus explicit per-component rules. -# easy does not isolate clients; strict isolates release traffic and DNS. Only -# enforced when the network plugin implements NetworkPolicy (kind v0.25+, +# easy keeps HTTP clients open; strict isolates release traffic and DNS. +# Enabled SeaTunnel always restricts member port 5801 to its release peers. +# Enforced only when the network plugin implements NetworkPolicy (kind v0.25+, # k3s, Calico, Cilium); elsewhere the objects are accepted and do nothing. # Nothing outside the release is admitted unless it is listed in # .extraIngress as standard NetworkPolicy ingress rules: apps in @@ -595,8 +609,9 @@ networkPolicy: # through extraEnv). PD, Store and Server reach only the release and DNS. extraEgress: [] seatunnel: - # SeaTunnel peers, DNS and the HugeGraph Server API are allowed by default. - # Add submitters/monitoring here using native NetworkPolicy rules. + # Member port 5801 is restricted even when networkPolicy.enabled is false. + # With full isolation, peers, DNS and the Server API are allowed by default. + # Add trusted submitters/monitoring using native NetworkPolicy rules. extraIngress: [] # External source/sink systems and shared checkpoint storage need egress. extraEgress: [] diff --git a/helm/test_helm_seatunnel.py b/helm/test_helm_seatunnel.py index ee83eee6cd..a2f4741574 100644 --- a/helm/test_helm_seatunnel.py +++ b/helm/test_helm_seatunnel.py @@ -401,6 +401,55 @@ def allows(rules, direction, selector, port): dns = [port for rule in engine["egress"] if not rule.get("to") for port in rule["ports"]] self.assertTrue({("UDP", 53), ("TCP", 53)} <= {(p["protocol"], p["port"]) for p in dns}) + def test_member_ingress_is_isolated_when_full_network_policy_is_disabled(self): + for mode in ("easy", "strict"): + with self.subTest(mode=mode): + documents = self.documents({}, mode=mode, networkPolicy={"enabled": False}) + policy, = engine_resources(documents, "NetworkPolicy") + spec = policy["spec"] + self.assertEqual(["Ingress"], spec["policyTypes"]) + self.assertNotIn("egress", spec) + workload, = engine_resources(documents, "StatefulSet") + labels = pod(workload)["metadata"]["labels"] + self.assertTrue(matches(spec["podSelector"]["matchLabels"], labels)) + member_rules = [rule for rule in spec["ingress"] + if any(port["port"] == 5801 for port in rule["ports"])] + self.assertEqual(1, len(member_rules)) + self.assertEqual([{"podSelector": {"matchLabels": spec["podSelector"]["matchLabels"]}}], + member_rules[0]["from"]) + selector = member_rules[0]["from"][0]["podSelector"]["matchLabels"] + self.assertFalse(matches(selector, {**labels, "app.kubernetes.io/component": "server"})) + self.assertFalse(matches(selector, {**labels, "app.kubernetes.io/instance": "other"})) + open_rules = [rule for rule in spec["ingress"] if not rule.get("from")] + self.assertEqual([{"ports": [{"protocol": "TCP", "port": 8080}]}], open_rules) + + def test_engine_security_defaults_and_native_scheduling_overrides(self): + workload, = engine_resources(self.documents({}), "StatefulSet") + container, = pod(workload)["spec"]["containers"] + self.assertEqual({"allowPrivilegeEscalation": False, + "capabilities": {"drop": ["ALL"]}, + "seccompProfile": {"type": "RuntimeDefault"}}, container["securityContext"]) + overrides = { + "podSecurityContext": {"runAsUser": 1000, "fsGroup": 1000}, + "securityContext": {"runAsNonRoot": True}, + "nodeSelector": {"workload": "ingestion"}, + "tolerations": [{"key": "ingestion", "operator": "Exists", "effect": "NoSchedule"}], + "affinity": {"podAntiAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": []}}, + "priorityClassName": "ingestion", + } + workload, = engine_resources(self.documents(overrides), "StatefulSet") + spec = pod(workload)["spec"] + self.assertEqual(overrides["podSecurityContext"], spec["securityContext"]) + for field in ("nodeSelector", "tolerations", "affinity", "priorityClassName"): + self.assertEqual(overrides[field], spec[field]) + container, = spec["containers"] + self.assertTrue(container["securityContext"]["runAsNonRoot"]) + self.assertFalse(container["securityContext"]["allowPrivilegeEscalation"]) + for invalid in ({"securityContext": []}, {"nodeSelector": []}, {"tolerations": {}}, + {"affinity": []}, {"priorityClassName": 1}): + with self.subTest(invalid=invalid): + self.assertNotEqual(0, self.render(invalid).returncode) + def test_packaged_chart_has_both_modes_without_seatunnel_dependency(self): with tempfile.TemporaryDirectory(prefix="seatunnel-package-") as directory: self.run_helm(["package", str(CHART), "--destination", directory], From 0051b78a074da08e1abe59f8add4cc7d6f722576 Mon Sep 17 00:00:00 2001 From: imbajin Date: Sun, 25 Oct 2026 14:42:03 +0800 Subject: [PATCH 15/17] fix(helm): secure native SeaTunnel members Restrict the unauthenticated member port in every chart mode while keeping easy-mode HTTP and egress behavior intact. Expose the same container security and scheduling settings as other workloads without forcing an image UID or read-only filesystem. Use English operator instructions and cover policy isolation plus configuration overrides in rendered chart regressions. --- helm/hugegraph/README.md | 33 +- helm/hugegraph/docs/seatunnel.md | 326 ++++++++++++------ .../examples/seatunnel/hugegraph-sink.conf | 4 +- .../examples/seatunnel/hugegraph-source.conf | 2 +- .../examples/values-seatunnel-cluster.yaml | 4 +- .../examples/values-seatunnel-standalone.yaml | 4 +- helm/hugegraph/templates/NOTES.txt | 47 +-- .../templates/seatunnel-networkpolicy.yaml | 11 +- helm/hugegraph/templates/seatunnel.yaml | 23 ++ helm/hugegraph/values.schema.json | 18 + helm/hugegraph/values.yaml | 23 +- helm/test_helm_seatunnel.py | 49 +++ 12 files changed, 386 insertions(+), 158 deletions(-) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 406bae362a..e75f6c708c 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -81,18 +81,27 @@ helm install hugegraph ./helm/hugegraph -n hugegraph --create-namespace \ Inactive credential values may remain in the values file when auth is disabled. -### 可选 SeaTunnel - -增加 `--set seatunnel.enabled=true`，与 HugeGraph 一起启动一个常驻 hybrid 节点； -增加 `--set seatunnel.mode=cluster`，使用同一套官方原生 Kubernetes 资源扩为多个节点（默认两个）。 -SeaTunnel 默认关闭，拓扑独立于 HugeGraph 的单机/集群预设。两种模式均提供引擎内置的任务可视化前端； -HTTP 入口继承 `easy` 的 NodePort 或 `strict` 的 ClusterIP/NetworkPolicy，可用 `seatunnel.service` 显式覆盖并指定 NodePort。 - -Chart 自动为同 release 的演示作业注入 Server 地址、图空间和管理员 Secret 引用， -并挂载 Source/Sink 配置；先创建目标图，即可手动提交，无需复制密码或上传作业文件。 -Helm 不自动建图或导入数据。前端和 REST 提交接口使用独立管理员的原生 HTTP Basic 认证， -默认密码自动生成在 SeaTunnel Secret 中并随升级保留；它与作业连接 HugeGraph 的凭据分别配置。 -启动、访问、演示作业与跨环境 Schema／顶点／边复制见 [SeaTunnel 联合部署指南](docs/seatunnel.md)。 +### Optional SeaTunnel + +Set `seatunnel.enabled=true` to start one persistent hybrid member alongside HugeGraph; +set `seatunnel.mode=cluster` to use multiple members of the same native StatefulSet +(two by default). SeaTunnel is disabled by default and its topology is independent +of the HugeGraph single-/multi-node presets. Both modes include the engine's task UI. +The HTTP Service inherits NodePort in `easy` mode or ClusterIP in `strict` mode; +`seatunnel.service` can explicitly override the type and NodePort. + +Member/CLI port 5801 is restricted to same-release peers by a SeaTunnel ingress +NetworkPolicy in every mode, including when the chart-wide policy flag is disabled. +Easy mode leaves the authenticated HTTP entry open. Actual isolation requires CNI +support and trusted namespace RBAC; full isolation additionally limits egress. + +The chart injects the same-release Server address, GraphSpace and administrator +Secret references into the mounted Source/Sink examples. Create the target graph +before submitting them manually; Helm does not create graphs or import data. +UI and REST submission use a separate native HTTP Basic administrator account, +whose generated Secret is retained and reused during online upgrades. Container +security and scheduling options follow the other components; see the +[SeaTunnel deployment guide](docs/seatunnel.md) for configuration and limits. ### Internal image registry diff --git a/helm/hugegraph/docs/seatunnel.md b/helm/hugegraph/docs/seatunnel.md index bcaede9db1..84f36150d4 100644 --- a/helm/hugegraph/docs/seatunnel.md +++ b/helm/hugegraph/docs/seatunnel.md @@ -1,78 +1,86 @@ -# HugeGraph 与 SeaTunnel 联合部署 +# Deploy HugeGraph with SeaTunnel -本 Chart 直接复用 [SeaTunnel 3.0.0 官方 hybrid Kubernetes 方案](https://github.com/apache/seatunnel/blob/v3.0.0/docs/zh/getting-started/kubernetes/hybrid-cluster-mode.md)，在 HugeGraph Helm 中适配资源名称、DNS、镜像、配置和副本数，不修改 SeaTunnel Helm，也不引入子 Chart、Operator 或 CRD。 +This chart adapts the [SeaTunnel 3.0.0 native hybrid Kubernetes deployment](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/getting-started/kubernetes/hybrid-cluster-mode.md) +for HugeGraph resource names, DNS, images, configuration and replica counts. +It does not introduce a subchart, operator, CRD or derived SeaTunnel image. -SeaTunnel 默认关闭。启用后，一个常驻 hybrid 节点同时承担调度和执行，持续接收任务；集群模式只是把同一个 StatefulSet 扩为多个 hybrid 节点。它与执行完即退出的 `local-mode` Pod/Job 不同。 +SeaTunnel is disabled by default. When enabled, a persistent hybrid member both +schedules and executes jobs. Cluster mode scales the same StatefulSet to multiple +members; it is not a `local-mode` Pod or Job that exits after one submission. -| 配置 | 含义 | +| Setting | Meaning | |---|---| -| `values-single.yaml` | HugeGraph 1 PD + 1 Store + 1 Server，使用 HStore | -| `values-cluster.yaml` | HugeGraph 自身的多副本拓扑 | -| `seatunnel.mode: standalone` | 一个常驻 hybrid 节点 | -| `seatunnel.mode: cluster`、`replicas: 2` | 两个 hybrid 节点，副本数至少为 2 | +| `values-single.yaml` | HugeGraph 1 PD + 1 Store + 1 Server using HStore | +| `values-cluster.yaml` | HugeGraph's own multi-replica topology | +| `seatunnel.mode: standalone` | One persistent hybrid member | +| `seatunnel.mode: cluster`, `replicas: 2` | Two hybrid members; at least two are required | -HugeGraph 与 SeaTunnel 的拓扑独立选择；顶层 `mode: easy/strict` 控制同一 release 的 HTTP 入口和 NetworkPolicy 默认值。SeaTunnel 单机和集群均提供相同的内置前端与 API。 +Choose HugeGraph and SeaTunnel topologies independently. Top-level `mode: easy/strict` +controls HTTP entry and full NetworkPolicy defaults. Both SeaTunnel modes provide +the same built-in UI and API. -## 启动、访问前端与切换 +## Install, access and switch modes -在 HugeGraph 仓库根目录执行，以下两种安装方式选择一种： +From the HugeGraph repository root, choose one installation: ```bash -# HugeGraph 1 PD + 1 Store + 1 Server，加一个常驻 SeaTunnel 节点 +# HugeGraph 1 PD + 1 Store + 1 Server, plus one persistent SeaTunnel member helm install hugegraph ./helm/hugegraph -n hugegraph --create-namespace \ -f helm/hugegraph/values-single.yaml \ -f helm/hugegraph/examples/values-seatunnel-standalone.yaml \ --wait --timeout 30m -# 同样的 HugeGraph 拓扑，加两个 SeaTunnel hybrid 节点 +# The same HugeGraph topology, plus two SeaTunnel hybrid members helm install hugegraph ./helm/hugegraph -n hugegraph --create-namespace \ -f helm/hugegraph/values-single.yaml \ -f helm/hugegraph/examples/values-seatunnel-cluster.yaml \ --wait --timeout 30m ``` -默认使用 `apache/seatunnel:3.0.0`。两种模式都开启引擎自带的 HTTP 服务和任务可视化前端，无需另部署 Web 服务或数据库。 - -SeaTunnel HTTP Service 沿用 HugeGraph 的访问模式；显式设置 `seatunnel.service.type` 时优先使用指定值： - -| 配置 | HTTP Service 默认值 | 访问方式 | -|---|---|---| -| `mode: easy` | `NodePort` | 可达节点 IP 加分配的 NodePort | -| `mode: strict` | `ClusterIP`，开启 NetworkPolicy | 集群内访问，或由运维人员临时端口转发 | -| 显式 `seatunnel.service.type` | 使用指定的 `ClusterIP`、`NodePort` 或 `LoadBalancer` | 按实际 Service 地址访问 | - -例如指定 NodePort；不设置 `nodePort` 时由 Kubernetes 分配： - -```yaml -seatunnel: - enabled: true - service: - type: NodePort - nodePort: 30081 - annotations: {} -``` +The default image is `apache/seatunnel:3.0.0`. Both modes expose the UI and REST +API on HTTP 8080. Easy mode inherits a NodePort, while strict mode inherits +ClusterIP. Override `seatunnel.service.type/nodePort` explicitly when needed. +`helm install` prints access instructions; retrieve an automatically allocated +HTTP NodePort with: ```bash kubectl -n hugegraph get svc hugegraph-seatunnel \ -o jsonpath='{.spec.ports[?(@.name=="http")].nodePort}{"\n"}' -kubectl get nodes -o wide ``` -浏览器入口为 `http://<可达节点IP>:/#/overview`，API 使用同一地址的相应 REST 路径。使用 LoadBalancer 时，通过 `kubectl -n hugegraph get svc hugegraph-seatunnel -o wide` 查看分配的地址，再访问 `http://<负载均衡地址>:8080/#/overview`。kind、minikube 等本地环境需按其网络方式提供可达入口。 - -所有 Service 类型均可使用端口转发： +For any Service type, port forwarding provides a local entry: ```bash kubectl -n hugegraph port-forward svc/hugegraph-seatunnel 8081:8080 ``` -对应前端地址为 ，使用下节的独立 SeaTunnel 账号完成浏览器 Basic 登录。前端用于查看集群、作业、DAG 和日志；作业通过 SeaTunnel CLI 或受保护的 REST API 提交。Helm 只启动引擎并准备示例配置，不自动建图或执行作业。 - -外部入口只包含 HTTP 8080。Hazelcast 5801 仍仅通过内部 Headless Service 供成员发现及 CLI 通信，不随 NodePort 或 LoadBalancer 一起公开。strict 模式下，额外的应用、监控或入口代理需通过 `networkPolicy.seatunnel.extraIngress` 配置原生规则；本 Chart 不额外管理 SeaTunnel Ingress。 - -release 为 `hugegraph` 时，资源为 `hugegraph-seatunnel` StatefulSet、同名 HTTP Service、`hugegraph-seatunnel-headless` 发现 Service，以及 `hugegraph-seatunnel-configs` ConfigMap。名称遵循 HugeGraph 的 `fullnameOverride`；成员通过 Headless Service 的 DNS 发现，不访问 Kubernetes API，因此不需要发现用的 RBAC。 - -切换前先停止全部作业，保留安装使用的完整 values，再升级： +Open and use the separate SeaTunnel credentials +described below. The UI shows cluster members, jobs, DAGs and logs. Submit jobs +through the CLI or authenticated REST API. Helm starts the engine and prepares +examples; it does not create graphs or run jobs. + +Only HTTP 8080 is exposed through NodePort or LoadBalancer. A separate ingress +NetworkPolicy always restricts Hazelcast member/CLI port 5801 to same-release +SeaTunnel peers, even in easy mode or with `networkPolicy.enabled=false`. +Easy mode permits HTTP callers without adding an ingress rule and leaves egress +unrestricted. When full NetworkPolicy isolation is enabled, explicitly allow +HTTP callers and external source/sink destinations with +`networkPolicy.seatunnel.extraIngress/extraEgress`. Explicit extra ingress rules +can also broaden member access, so restrict them to trusted callers. + +Actual isolation requires a NetworkPolicy-capable CNI. Pod selectors are not +native Hazelcast authentication: use trusted namespace RBAC/admission controls +for creating or relabeling Pods and for `kubectl exec`. Do not expose 5801 outside +that administrative boundary. The chart does not manage a separate SeaTunnel Ingress. + +For release `hugegraph`, resources are the `hugegraph-seatunnel` StatefulSet and +HTTP Service, `hugegraph-seatunnel-headless` discovery Service, and +`hugegraph-seatunnel-configs` ConfigMap. Names honor `fullnameOverride`. +Members use Headless Service DNS rather than the Kubernetes API, so discovery +requires no Kubernetes API token or RBAC permission. + +Stop all jobs before switching modes. Retain the complete installation values +and upgrade with: ```bash helm upgrade hugegraph ./helm/hugegraph -n hugegraph -f my-values.yaml \ @@ -80,15 +88,28 @@ helm upgrade hugegraph ./helm/hugegraph -n hugegraph -f my-values.yaml \ --set seatunnel.replicas=2 --wait --timeout 30m ``` -切回单机设置 `seatunnel.mode=standalone`。两种模式使用同一个 StatefulSet，但副本和引擎配置会变化；不承诺无中断切换，也不会自动恢复或重新提交作业。设置 `seatunnel.enabled=false` 移除本 release 的 SeaTunnel 工作负载和 Service，已经写入 HugeGraph 的数据保留。认证 Secret 带有保留策略，禁用或卸载时仍会保留，便于再次启用时复用凭据。 +Set `seatunnel.mode=standalone` to return to one member. The StatefulSet stays the +same, but replica count and engine configuration change. Uninterrupted switching, +automatic job recovery and resubmission are not guaranteed. Setting +`seatunnel.enabled=false` removes the engine workloads and Services without +deleting data already written to HugeGraph. The retained auth Secret can be +reused after re-enabling or reinstalling. -## SeaTunnel 入口认证 +## SeaTunnel entry authentication -前端、静态资源和 REST 提交接口默认使用引擎原生 HTTP Basic 认证，不新增认证代理或独立登录服务。SeaTunnel 使用独立入口管理员：`seatunnel.auth.username` 默认为 `admin`；密码未显式设置时，Chart 自动生成强密码并保存在 `-auth` Secret，升级时复用已有密码。 +The UI, static resources and REST submission endpoints use native HTTP Basic +authentication. There is no added proxy or login service. `seatunnel.auth.username` +defaults to `admin`. An empty password generates a strong password in +`-auth`; online upgrades reuse the existing password. -也可通过受控 values 文件显式设置 `seatunnel.auth.username/password`，用于自定义账号或轮换。自定义值不能有首尾空白或 `${...}` 占位符，用户名另不能包含冒号；内部空格和引号仍可使用。只做离线模板或 GitOps 渲染时，应显式固定 `seatunnel.auth.password`；默认密码的升级复用依赖在线 Helm lookup。 +Use a controlled values file to set or rotate `seatunnel.auth.username/password`. +Values cannot contain leading/trailing whitespace or `${...}` placeholders, and +usernames cannot contain a colon. Internal spaces and quotes are accepted. +Pin `seatunnel.auth.password` for offline templates or GitOps rendering; automatic +password reuse requires an online Helm lookup. -默认 release 的 Secret 为 `hugegraph-seatunnel-auth`。保持前述端口转发，在本机读取账号和密码后调用 REST： +The default release uses Secret `hugegraph-seatunnel-auth`. With port forwarding +active, read its credentials into local variables and call REST: ```bash ST_USERNAME="$(kubectl -n hugegraph get secret hugegraph-seatunnel-auth \ @@ -99,31 +120,55 @@ ST_PASSWORD="$(kubectl -n hugegraph get secret hugegraph-seatunnel-auth \ curl -fsS --user "$ST_USERNAME:$ST_PASSWORD" http://127.0.0.1:8081/overview ``` -浏览器打开 `/#/overview` 时，使用 Secret 中的同一组账号密码完成 HTTP Basic 登录；访问其他 NodePort 或 LoadBalancer 地址也使用这组入口凭据。外部提交作业使用同一受保护 HTTP 入口，并按 [官方 REST API](https://github.com/apache/seatunnel/blob/v3.0.0/docs/zh/engines/zeta/rest-api-and-web-ui.md) 的格式携带认证信息。 - -入口 Secret 包含 `username`、`password` 及最终的 `seatunnel.yaml`；包含认证信息的引擎文件直接从 Secret 挂载，不进入 ConfigMap、Java 参数或日志。轮换时在受控 values 中指定新密码并执行 `helm upgrade`，同步更新 Secret 内容并滚动重启；只修改 Secret 的 `data.password` 不会同步修改其中的引擎文件。`HG_USERNAME/HG_PASSWORD` 是作业连接 HugeGraph 的另一组凭据，不会替代入口认证。即使 `server.auth.enabled=false`，SeaTunnel 入口仍使用独立认证。 - -HTTP Basic 不覆盖 Hazelcast 5801 的内部成员和 CLI 协议。下文在 Pod 内执行的 `seatunnel.sh --master cluster` 属于可信 Kubernetes/RBAC 管理范围；外部客户端应使用受保护的 REST 入口，不能把 5801 对外公开。本配置提供供可信管理人员使用的单一管理员入口，不提供按用户或作业划分的细粒度权限，也不复用 HugeGraph 的多用户权限、RBAC 或 SSO。需要对外服务时可接入已有 TLS 入口；本 Chart 不新增 Ingress 管理。 - -## 镜像、凭据与原生配置 - -本轮已直接核实官方 `apache/seatunnel:3.0.0` 镜像自带 HugeGraph、FakeSource、Console 连接器和内置 UI，直接复用即可，无需维护派生镜像。更换版本或自定义镜像时再核对插件与依赖。镜像通过 `seatunnel.image.repository/tag/pullPolicy` 选择，继承 HugeGraph 的 `global.imageRegistry` 和 `imagePullSecrets`；Helm 不在启动时下载插件。官方镜像包含大量连接器，本次 amd64 镜像压缩后约 3.7GB；首次拉取较慢时可延长安装 timeout，或预拉取、镜像到内部仓库。 - -启用 SeaTunnel 后，Chart 为同 release 的 HugeGraph 演示作业准备以下环境变量，无需手写 Server 地址或复制管理员密码： - -| 变量 | 默认来源 | +Use the same credentials for the browser Basic login and any NodePort or +LoadBalancer entry. External job submissions must authenticate to the protected +HTTP entry using the [official REST API](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/engines/zeta/rest-api-and-web-ui.md). + +The Secret contains `username`, `password` and the final `seatunnel.yaml`. +The engine configuration is mounted directly from the Secret, not embedded in +a ConfigMap, Java arguments or logs. Rotate through controlled values and +`helm upgrade`, which updates the Secret and rolls the engine Pods. Editing only +`data.password` does not update credentials inside the engine configuration. +`HG_USERNAME/HG_PASSWORD` are separate HugeGraph connector credentials and cannot +replace entry authentication. Disabling `server.auth.enabled` does not disable +SeaTunnel authentication. + +HTTP Basic does not authenticate the Hazelcast member/CLI protocol on 5801. +The examples below execute the CLI inside an engine Pod under Kubernetes RBAC; +external clients should use REST. This is a single administrator entry, not +per-user/per-job authorization or HugeGraph RBAC/SSO. Use an existing TLS entry +when exposing the HTTP service outside a trusted network. + +## Images, credentials and native configuration + +The official `apache/seatunnel:3.0.0` image used by this chart includes the +HugeGraph, FakeSource and Console connectors and the built-in UI. Verify plugin +availability when changing the version or image. Configure +`seatunnel.image.repository/tag/pullPolicy`; it inherits `global.imageRegistry` +and `imagePullSecrets`. Helm does not download plugins at startup. The upstream +image contains many connectors, so allow sufficient pull time or pre-pull/mirror +it into an internal registry. + +The chart provides these defaults for same-release HugeGraph example jobs: + +| Variable | Default source | |---|---| -| `HG_HOST` | 当前 release 的 HugeGraph Server Service DNS | +| `HG_HOST` | Same-release HugeGraph Server Service DNS | | `HG_PORT` | `server.port` | -| `HG_PROTOCOL` | `http`，访问集群内部 Server | +| `HG_PROTOCOL` | `http` for the internal Server connection | | `HG_USERNAME` | `admin` | -| `HG_PASSWORD` | 当前 Server 管理员 Secret 的实际名称与键；认证关闭时为空密码 | +| `HG_PASSWORD` | Actual Server administrator Secret name/key, or empty when auth is disabled | | `HG_GRAPH_SPACE` | `server.graphSpace` | | `HG_GRAPH_NAME` | `graph_a` | -`HG_PASSWORD` 使用原生 `secretKeyRef`，兼容 `server.auth.admin.existingSecret` 及其自定义键，不把密码写入演示文件。Chart 不因此创建额外的 HugeGraph 账号；更新密码 Secret 后，按 Server 账号同步流程处理，再重启相关 SeaTunnel Pod 刷新环境变量。 +`HG_PASSWORD` uses a native `secretKeyRef`, including +`server.auth.admin.existingSecret` and custom keys. It is not copied into example +job files. The chart does not create an extra HugeGraph account. Follow the +Server account synchronization procedure after rotating its credential Secret, +then restart relevant engine Pods to refresh their environment. -`seatunnel.env` 中同名条目显式覆盖自动值，最终每个变量只有一个有效条目。例如更换演示图，或使用已有的专用作业账号： +A matching name in `seatunnel.env` overrides an automatic value without creating +duplicate entries. For example, select another graph and an existing job account: ```yaml seatunnel: @@ -139,9 +184,22 @@ seatunnel: key: password ``` -专用账号需预先存在，并具备所用图的数据和 Schema 权限。这组 `HG_*` 默认值只服务于本 release 的简单 Source/Sink 示例；跨环境复制仍由用户提供下文的 `HG_SOURCE_*`、`HG_TARGET_*`，不会自动发现或配置远端图。SeaTunnel 前端/API 自身的认证也不能由这组连接器变量替代。 +The account must already exist and have the required data/Schema permissions. +These defaults serve only the local Source/Sink examples. Cross-environment jobs +require the explicit `HG_SOURCE_*` and `HG_TARGET_*` values below; the chart does +not discover remote graphs. Connector credentials do not replace entry auth. -`seatunnel.javaOpts` 调整引擎 JVM，`resources` 调整容器资源。高级配置保留 SeaTunnel 原生文件格式，通过 `seatunnel.config` 按文件名完整覆盖，不逐项增加 Helm 开关，例如： +Tune the JVM with `seatunnel.javaOpts` and container resources with +`seatunnel.resources`. `seatunnel.podSecurityContext` and +`seatunnel.securityContext` follow the other chart components. The container +starts with privilege escalation disabled, all capabilities dropped and +`RuntimeDefault` seccomp. The upstream image runs as root: configure a compatible +UID and writable paths when enforcing `runAsNonRoot` or a restricted Pod Security +namespace; these defaults alone do not establish compatibility with that policy. +Use `seatunnel.nodeSelector`, `tolerations`, `affinity` and `priorityClassName` for +node placement and scheduling. + +Override complete native configuration files through `seatunnel.config`, for example: ```yaml seatunnel: @@ -152,23 +210,33 @@ seatunnel: -Xmx256m ``` -默认在 ConfigMap 中准备 `hazelcast.yaml`、`hazelcast-client.yaml`、JVM 和演示作业文件；包含入口认证的 `seatunnel.yaml` 从独立 Secret 挂载，其余配置沿用镜像。`seatunnel.config` 可覆盖引擎文件，但最终 HTTP Basic 开关和凭据始终由 `seatunnel.auth` 管理。覆盖发现或引擎文件时保留 TCP 5801、HTTP 8080 和 `enable-http: true`，才能继续使用现有 Service、探针及前端。默认 checkpoint 使用临时本地目录；本集成不提供持久化作业恢复保证。 - -strict 模式允许 SeaTunnel 成员通信、DNS，以及访问同 release 的 HugeGraph Server。远端图 A、其他数据源及调用方，通过 `networkPolicy.seatunnel.extraEgress/extraIngress` 补充原生规则；实际隔离依赖 CNI 支持。HugeGraph 的凭据、存储和访问方式见 [Chart README](../README.md)。 - -## 同 release 演示：直接导入并读取 - -Chart 启动时把 [hugegraph-sink.conf](../examples/seatunnel/hugegraph-sink.conf) 和 [hugegraph-source.conf](../examples/seatunnel/hugegraph-source.conf) 放入现有的 SeaTunnel ConfigMap，分别挂载为： - -- `/opt/seatunnel/config/hugegraph-sink.conf`：`FakeSource → HugeGraph`，向 `seatunnel_demo_person` 标签写入十条固定数据。 -- `/opt/seatunnel/config/hugegraph-source.conf`：读取同一图中的演示标签，输出到 Console。 - -两个文件均使用前述自动 `HG_*` 环境变量；无需上传临时文件或手动补全当前 Server 地址、端口、图空间及密码。需要调整作业时，可在 `seatunnel.config` 中按同名文件完整覆盖。 - -先通过 Hubble 或 HugeGraph API 创建 `graph_a`。如果改写了 `HG_GRAPH_SPACE/HG_GRAPH_NAME`，则创建对应图。以下命令按默认 release、命名空间及图空间演示建图；本机的 `HG_USERNAME/HG_PASSWORD` 由凭据管理器提供，Pod 环境变量不会传到本机： +The ConfigMap supplies discovery, JVM and example job files. The authenticated +`seatunnel.yaml` is mounted from its separate Secret; other files use the image +defaults. Final HTTP Basic settings remain managed by `seatunnel.auth` even when +overriding engine configuration. Retain member port 5801, HTTP 8080 and +`enable-http: true` so Services, probes and UI remain usable. Checkpoints use an +ephemeral local directory; this integration does not guarantee durable recovery. +Full NetworkPolicy isolation allows peer traffic, DNS and the same-release +Server API by default. See the [Chart README](../README.md) for HugeGraph access, +credentials and storage. + +## Same-release examples: import and read + +The chart mounts [hugegraph-sink.conf](../examples/seatunnel/hugegraph-sink.conf) +and [hugegraph-source.conf](../examples/seatunnel/hugegraph-source.conf) as +`/opt/seatunnel/config/hugegraph-sink.conf` and +`/opt/seatunnel/config/hugegraph-source.conf`. The Sink writes ten fixed vertices +with label `seatunnel_demo_person`; the Source reads that label into Console. +Both use the automatic `HG_*` environment values. Override a file under the same +name in `seatunnel.config` to customize a job. + +Create `graph_a` through Hubble or the HugeGraph API first, or create the graph +selected by explicit `HG_GRAPH_SPACE/HG_GRAPH_NAME` values. The default release, +namespace and GraphSpace are used below. Local `HG_USERNAME/HG_PASSWORD` must +come from your credential manager; Pod environment values do not reach your shell. ```bash -# 在另一个终端保持 Server 端口转发 +# Keep Server port forwarding active in a separate terminal kubectl -n hugegraph port-forward svc/hugegraph-server 8082:8080 ``` @@ -178,7 +246,7 @@ curl -fsS -u "$HG_USERNAME:$HG_PASSWORD" -H 'Content-Type: application/json' \ -X POST "$HG_URL/graphspaces/DEFAULT/graphs/graph_a" -d '{}' ``` -已有图不重复建图。随后直接提交已挂载的 Sink，再提交 Source： +Do not recreate an existing graph. Submit the mounted Sink, then Source: ```bash kubectl -n hugegraph exec sts/hugegraph-seatunnel -- \ @@ -190,7 +258,10 @@ kubectl -n hugegraph exec sts/hugegraph-seatunnel -- \ --config /opt/seatunnel/config/hugegraph-source.conf ``` -`--master cluster` 表示提交给常驻引擎，单节点也使用该参数。首次空图导入后，Sink 作业应为 `FINISHED`，读取、写入各十条；Source 应读取十条并输出到执行节点的日志。可先在前端查看作业状态和 DAG，再检查各 hybrid Pod 日志及图中实际数据： +`--master cluster` submits to the persistent engine, including standalone mode. +On an initially empty graph, expect a finished Sink with ten rows read/written, +and a Source with ten rows read into the executing member's log. Inspect UI +status/DAG, all member logs and the actual graph data: ```bash kubectl -n hugegraph logs -l app.kubernetes.io/instance=hugegraph,app.kubernetes.io/component=seatunnel \ @@ -200,27 +271,42 @@ curl -fsS -u "$HG_USERNAME:$HG_PASSWORD" -H 'Content-Type: application/json' \ "$HG_URL/gremlin" -d '{"gremlin":"graph.traversal().V().hasLabel(\"seatunnel_demo_person\").count()","aliases":{"graph":"DEFAULT-graph_a"}}' ``` -数量查询的 `result.data` 应为 `[10]`。集群模式下作业可能由任一 hybrid 节点执行，因此 Console 输出不限定在 `hugegraph-seatunnel-0`。这两个演示作业也不会在 Helm 安装或升级时自动执行。 +The count query should return `result.data: [10]`. A cluster job can execute on +any member, so Console output is not limited to `hugegraph-seatunnel-0`. Neither +example runs automatically during Helm installation or upgrade. -## 新环境复制：Schema、顶点和边 +## Cross-environment copy: Schema, vertices and edges -这里使用已有图 A 和另一个环境中的空图 B，不接续上面的十条数据示例。复制顺序是 **Schema → 顶点 → 边**。SeaTunnel Sink 的自动建 Schema 只能根据映射和行类型创建所需结构，不能替代源图完整 Schema、索引及 ID 的复制。 +Use an existing source graph A and an empty graph B in another environment. +Copy **Schema, then vertices, then edges**. Sink auto-schema mapping cannot +replace copying the complete source Schema, indexes and IDs. -先通过 HugeGraph 原生 API 创建空的 `graph_b`，再执行 [copy-schema.py](../examples/seatunnel/clone/copy-schema.py)。脚本在目标图的 RESTORING 状态下保留源 Schema ID，复制属性键、顶点标签、边标签和索引；不传输图数据。源、目标应使用兼容的 HugeGraph 版本，账号需有相应管理权限，目标 Schema 必须为空。 +Create `graph_b` through the native API, then run +[copy-schema.py](../examples/seatunnel/clone/copy-schema.py). It copies property +keys, vertex labels, edge labels and indexes with source Schema IDs while the +target is in RESTORING mode. It does not copy graph data. Use compatible HugeGraph +versions and accounts with administrative permissions; target Schema must be empty. -这份示例只支持普通边标签。遇到 `PARENT`、`SUB` 或未知边标签类型时，脚本会在访问目标图前退出；继承边需要按父子关系另行迁移。 +Only NORMAL edge labels are supported. The script rejects PARENT, SUB or unknown +edge types before contacting the target; inherited Schema needs a separate +migration procedure. ```bash -# 两个 URL 均包含图空间及图名；对应账号密码由本机环境提供 +# Both URLs include GraphSpace and graph; credentials come from your local environment export HG_SOURCE_GRAPH_URL=https://source.example.com/graphspaces/DEFAULT/graphs/graph_a export HG_TARGET_GRAPH_URL=https://target.example.com/graphspaces/DEFAULT/graphs/graph_b python3 helm/hugegraph/examples/seatunnel/clone/copy-schema.py \ --source "$HG_SOURCE_GRAPH_URL" --target "$HG_TARGET_GRAPH_URL" ``` -脚本读取 `HG_SOURCE_USERNAME/HG_SOURCE_PASSWORD` 和 `HG_TARGET_USERNAME/HG_TARGET_PASSWORD`。随后在 SeaTunnel Pod 中配置相同的两组凭据，以及 `HG_SOURCE_HOST/PORT/PROTOCOL`、`HG_TARGET_HOST/PORT/PROTOCOL`；密码继续用 Secret 注入。源、目标地址必须能从 SeaTunnel Pod 访问，不能填写仅在本机可达的端口转发地址。 +The script reads `HG_SOURCE_USERNAME/HG_SOURCE_PASSWORD` and +`HG_TARGET_USERNAME/HG_TARGET_PASSWORD`. Configure corresponding credentials in +the engine Pods through Secrets, plus `HG_SOURCE_HOST/PORT/PROTOCOL` and +`HG_TARGET_HOST/PORT/PROTOCOL`. Addresses must be reachable from the engine Pods, +not merely local port-forward addresses. -四个示例作业读取 `graph_a`、写入 `graph_b`，先复制两个顶点标签，再复制两类边： +The four example jobs read `graph_a` and write `graph_b`, copying both vertex +labels before either edge label: ```bash for clone_job in person company works_at knows; do @@ -233,20 +319,38 @@ for clone_job in person company works_at knows; do done ``` -作业使用 `ERROR_WHEN_SCHEMA_NOT_EXIST`，复用已经复制的 Schema；边依赖先前写入的端点顶点。这组映射覆盖 [person](../examples/seatunnel/clone/person.conf)、[company](../examples/seatunnel/clone/company.conf)、[works_at](../examples/seatunnel/clone/works_at.conf) 和 [knows](../examples/seatunnel/clone/knows.conf)。官方 Sink 要求显式提供 `mappings` 或 `schema_config`。复制其他图时需按实际标签、属性和 ID 策略调整原生映射，不会自动枚举全图并提交迁移作业。 - -这组示例验证 `PRIMARY_KEY` 和 `CUSTOMIZE_NUMBER` 顶点 ID 策略。现有连接器对 `AUTOMATIC` 顶点会生成新 ID，不能用这组映射直接无损复制其边端点；迁移前先核对源图的 ID 策略。 - -四份映射均显式设置 `timeZone = UTC`，假设源 Server 的 `DATE` 字符串按 UTC 序列化，避免复制结果随 SeaTunnel worker 的默认时区变化。 -源 Server 使用其他时区时，应把每个 Sink `mappings` 条目中的 `timeZone` 改为源 Server 的实际时区。 -Source 的 `time_zone` 不能修正 REST 返回的无时区日期字符串；含 `DATE` 属性时，验收还需比较原始时间戳。 - -验收应同时核对属性键类型、基数、标签、主键、边端点定义、索引和 Schema ID，以及全部顶点 ID/属性、边 ID/端点/属性；只比较数量或作业状态不够。相关选项见官方 [HugeGraph Source](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/source/HugeGraph.md) 和 [Sink](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/sink/HugeGraph.md)。 - -## 使用限制 - -集群模式下，应逐个 hybrid Pod 确认 Workers 和成员数与副本数一致；仅 Pod Ready 不能说明成员已组成同一个集群。Hazelcast DNS 发现使用模板中的 `service-port: 5801`，应与引擎监听端口保持一致。 - -- 示例支持 `PRIMARY_KEY`、`CUSTOMIZE_NUMBER`，不支持 `AUTOMATIC` 顶点的无损复制；后者需要解决新旧顶点 ID 与边端点的对应关系。 -- 官方 3.0.0 UI 的 Duration 显示存在上游问题：无时区时间字符串在新加坡浏览器中会产生 8 小时偏差，且已完成作业的显示时长仍随当前时间增长。以 CLI/REST 的完成状态及开始、结束时间核对实际结果。 -- 模式切换前应停止作业；本集成不承诺运行中作业无中断切换、生产 HA 或持久化恢复。默认临时 checkpoint 目录不是持久化恢复方案，NetworkPolicy 的实际隔离依赖 CNI 支持。 +Mappings use `ERROR_WHEN_SCHEMA_NOT_EXIST` and the already copied Schema. Edges +require their endpoint vertices first. See [person](../examples/seatunnel/clone/person.conf), +[company](../examples/seatunnel/clone/company.conf), [works_at](../examples/seatunnel/clone/works_at.conf) +and [knows](../examples/seatunnel/clone/knows.conf). The official Sink requires +explicit `mappings` or `schema_config`; adapt labels, properties and ID strategies +to your graph. These examples do not enumerate or migrate every graph automatically. + +The mappings support PRIMARY_KEY and CUSTOMIZE_NUMBER vertex IDs. AUTOMATIC +vertices receive new IDs, so these mappings cannot preserve their edge endpoints +losslessly. Check source ID strategies before migration. + +Every Sink mapping sets `timeZone = UTC`, assuming UTC-serialized DATE strings +from the source Server. Set each mapping to the source Server's actual time zone +when different. Source `time_zone` cannot repair timezone-free REST date strings; +include original timestamps in acceptance checks for DATE properties. + +Compare Schema types, cardinalities, labels, primary keys, endpoint definitions, +indexes and Schema IDs, plus every vertex ID/property and edge ID/endpoint/property. +Counts or job status alone are insufficient. See the official +[HugeGraph Source](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/source/HugeGraph.md) +and [Sink](https://github.com/apache/seatunnel/blob/v3.0.0/docs/en/connectors/sink/HugeGraph.md) options. + +## Limits + +Check the Workers and member counts on every hybrid Pod against the configured +replicas; Pod Ready alone does not establish a shared engine cluster. DNS +discovery's `service-port: 5801` must match the listening member port. + +- PRIMARY_KEY and CUSTOMIZE_NUMBER examples do not provide lossless AUTOMATIC-ID + copies; these require an explicit mapping of generated IDs and edge endpoints. +- The upstream 3.0.0 UI Duration display has a known timezone/completed-job issue. + Use CLI/REST status and start/end timestamps to assess actual completion. +- Stop jobs before mode changes. Uninterrupted switching, production HA and + persistent recovery are not guaranteed. Temporary checkpoints are not a durable + recovery design, and actual NetworkPolicy enforcement depends on the CNI. diff --git a/helm/hugegraph/examples/seatunnel/hugegraph-sink.conf b/helm/hugegraph/examples/seatunnel/hugegraph-sink.conf index fbfb685b1f..7b408a2c14 100644 --- a/helm/hugegraph/examples/seatunnel/hugegraph-sink.conf +++ b/helm/hugegraph/examples/seatunnel/hugegraph-sink.conf @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# 先创建 HG_GRAPH_SPACE/HG_GRAPH_NAME 指定的图，默认 DEFAULT/graph_a。 -# Helm 自动配置同 release 的 HugeGraph 地址与 Secret 凭据。 +# First create the graph selected by HG_GRAPH_SPACE/HG_GRAPH_NAME (default DEFAULT/graph_a). +# Helm configures the same-release HugeGraph address and Secret credential references. env { job.mode = "BATCH" parallelism = 1 diff --git a/helm/hugegraph/examples/seatunnel/hugegraph-source.conf b/helm/hugegraph/examples/seatunnel/hugegraph-source.conf index 622e5effa6..e28a8fa148 100644 --- a/helm/hugegraph/examples/seatunnel/hugegraph-source.conf +++ b/helm/hugegraph/examples/seatunnel/hugegraph-source.conf @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# 先运行 hugegraph-sink.conf，再读取同一图中的十条演示顶点。 +# Run hugegraph-sink.conf first, then read the ten example vertices from the same graph. env { job.mode = BATCH parallelism = 1 diff --git a/helm/hugegraph/examples/values-seatunnel-cluster.yaml b/helm/hugegraph/examples/values-seatunnel-cluster.yaml index 9e6fe30cb0..25308032af 100644 --- a/helm/hugegraph/examples/values-seatunnel-cluster.yaml +++ b/helm/hugegraph/examples/values-seatunnel-cluster.yaml @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# 复用官方原生 hybrid StatefulSet；HTTP8080 同时提供 REST 和任务可视化前端。 -# HugeGraph 拓扑独立选择，1 PD + 1 Store + 1 Server 可用 ../values-single.yaml。 +# Use the official native hybrid StatefulSet; HTTP 8080 serves REST and the task UI. +# Choose HugeGraph topology independently; ../values-single.yaml uses 1 PD + 1 Store + 1 Server. seatunnel: enabled: true mode: cluster diff --git a/helm/hugegraph/examples/values-seatunnel-standalone.yaml b/helm/hugegraph/examples/values-seatunnel-standalone.yaml index e75efd6a58..91a79ba511 100644 --- a/helm/hugegraph/examples/values-seatunnel-standalone.yaml +++ b/helm/hugegraph/examples/values-seatunnel-standalone.yaml @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# 复用官方原生 hybrid StatefulSet；HTTP8080 同时提供 REST 和任务可视化前端。 -# HugeGraph 拓扑独立选择，1 PD + 1 Store + 1 Server 可用 ../values-single.yaml。 +# Use the official native hybrid StatefulSet; HTTP 8080 serves REST and the task UI. +# Choose HugeGraph topology independently; ../values-single.yaml uses 1 PD + 1 Store + 1 Server. seatunnel: enabled: true mode: standalone diff --git a/helm/hugegraph/templates/NOTES.txt b/helm/hugegraph/templates/NOTES.txt index 0db01562bc..3ec7e7bc8d 100644 --- a/helm/hugegraph/templates/NOTES.txt +++ b/helm/hugegraph/templates/NOTES.txt @@ -64,50 +64,51 @@ Access: SeaTunnel: {{- $stName := include "hugegraph.seatunnel.name" . }} {{- $stService := .Values.seatunnel.service | default dict }} - 集群内 API: http://{{ $stName }}.{{ .Release.Namespace }}.svc:8080 + In-cluster API: http://{{ $stName }}.{{ .Release.Namespace }}.svc:8080 {{- if eq $stService.type "NodePort" }} HTTP NodePort: kubectl get svc -n {{ .Release.Namespace }} {{ $stName }} -o jsonpath='{.spec.ports[?(@.name=="http")].nodePort}{"\n"}' kubectl get nodes -o wide - 前端: http://<可达节点IP>:{{ $stService.nodePort | default "" }}/#/overview - API: http://<可达节点IP>:{{ $stService.nodePort | default "" }} + UI: http://:{{ $stService.nodePort | default "" }}/#/overview + API: http://:{{ $stService.nodePort | default "" }} {{- else if eq $stService.type "LoadBalancer" }} - 负载均衡地址: + LoadBalancer address: kubectl get svc -n {{ .Release.Namespace }} {{ $stName }} -o wide kubectl get svc -n {{ .Release.Namespace }} {{ $stName }} -o jsonpath='{.status.loadBalancer.ingress}{"\n"}' - 地址分配后访问前端 http://<负载均衡地址>:8080/#/overview，API 使用同一 HTTP 地址。 + Once assigned, open http://:8080/#/overview; the API uses the same HTTP address. {{- else }} - HTTP Service 为 ClusterIP；集群外访问可使用下方端口转发。 + HTTP Service is ClusterIP; use port forwarding below for access outside the cluster. {{- end }} - 所有 Service 类型均可端口转发: + Port forwarding works with every Service type: kubectl port-forward -n {{ .Release.Namespace }} svc/{{ $stName }} 8081:8080 - 本机前端: http://127.0.0.1:8081/#/overview - 本机 API: http://127.0.0.1:8081 - Hazelcast 5801 仅用于集群内部发现及 CLI 通信，不通过以上外部入口公开。 - 前端、静态资源和 REST 提交接口使用独立 SeaTunnel 账号的 HTTP Basic 认证。 - 以下命令读取凭据到本机变量，不在安装输出中显示密码: + Local UI: http://127.0.0.1:8081/#/overview + Local API: http://127.0.0.1:8081 + The SeaTunnel NetworkPolicy restricts member/CLI port 5801 to same-release peers in every mode. + Enforcement requires a NetworkPolicy-capable CNI; keep namespace labels, pod creation and exec under trusted RBAC. + The UI, static resources and REST submission API use a separate SeaTunnel HTTP Basic account. + Read credentials into local variables without printing the password in installation output: ST_USERNAME="$(kubectl get secret -n {{ .Release.Namespace }} {{ $stName }}-auth -o jsonpath='{.data.username}' | base64 --decode)" ST_PASSWORD="$(kubectl get secret -n {{ .Release.Namespace }} {{ $stName }}-auth -o jsonpath='{.data.password}' | base64 --decode)" - 浏览器 Basic 登录使用此 Secret 中的账号密码；保持端口转发后可验证 REST: + Use this account for the browser Basic login; with port forwarding active, check REST with: curl --user "${ST_USERNAME}:${ST_PASSWORD}" http://127.0.0.1:8081/overview - 该入口账号独立于 HugeGraph 连接器账号；关闭 HugeGraph 认证也不会关闭 SeaTunnel 入口认证。 - 内部 CLI/5801 不经 HTTP Basic，仅在可信 Kubernetes/RBAC 范围使用，外部提交走受保护的 REST。 - 默认凭据随在线 Helm 升级保留；轮换用受控 values 的 seatunnel.auth.password，再 helm upgrade。 - GitOps/template-only 渲染请显式固定密码；认证 Secret 在禁用或卸载后仍保留。 - 账号、轮换及作业说明: helm/hugegraph/docs/seatunnel.md + These entry credentials are separate from HugeGraph connector credentials, even when HugeGraph auth is disabled. + Internal CLI/5801 has no HTTP Basic authentication. External submissions must use the protected REST API. + Online Helm upgrades retain the default credentials; rotate using seatunnel.auth.password in controlled values. + Pin a password for GitOps/template-only rendering. The auth Secret is retained after disabling or uninstalling. + Authentication, rotation and jobs: helm/hugegraph/docs/seatunnel.md {{- if and .Values.networkPolicy.enabled (has $stService.type (list "NodePort" "LoadBalancer")) }} {{- $stRules := get .Values.networkPolicy "seatunnel" | default dict }} {{- if empty (get $stRules "extraIngress") }} - NetworkPolicy 已开启；外部 HTTP 调用方还需通过 networkPolicy.seatunnel.extraIngress 允许。 + Full NetworkPolicy isolation is enabled; allow external HTTP callers through networkPolicy.seatunnel.extraIngress. {{- end }} {{- end }} - 同 release 演示作业: - 已注入 Server 地址、端口、图空间及凭据引用，示例配置已挂载，无需上传临时文件。 - 先创建目标图（默认 {{ .Values.server.graphSpace }}/graph_a；HG_* 显式覆盖时以覆盖值为准），再手动提交: + Same-release example jobs: + Server address, port, GraphSpace and credential references are injected; the job files are already mounted. + First create the target graph (default {{ .Values.server.graphSpace }}/graph_a; explicit HG_* values override it), then submit: kubectl exec -n {{ .Release.Namespace }} sts/{{ $stName }} -- /opt/seatunnel/bin/seatunnel.sh --master cluster --config /opt/seatunnel/config/hugegraph-sink.conf kubectl exec -n {{ .Release.Namespace }} sts/{{ $stName }} -- /opt/seatunnel/bin/seatunnel.sh --master cluster --config /opt/seatunnel/config/hugegraph-source.conf - Helm 不自动建图或执行作业；跨环境 clone 仍需用户提供 HG_SOURCE_* / HG_TARGET_*。 + Helm does not create graphs or run jobs. Cross-environment copies still require HG_SOURCE_* / HG_TARGET_* values. {{- end }} {{- range $component := list "server" "hubble" "pd" }} {{- if or (ne $component "hubble") $hubbleEnabled }} diff --git a/helm/hugegraph/templates/seatunnel-networkpolicy.yaml b/helm/hugegraph/templates/seatunnel-networkpolicy.yaml index b9746c95f5..faddf24c86 100644 --- a/helm/hugegraph/templates/seatunnel-networkpolicy.yaml +++ b/helm/hugegraph/templates/seatunnel-networkpolicy.yaml @@ -14,7 +14,7 @@ # limitations under the License. {{ include "hugegraph.resolveValues" . }} -{{- if and (.Values.seatunnel | default dict).enabled .Values.networkPolicy.enabled }} +{{- if (.Values.seatunnel | default dict).enabled }} {{- $rules := get .Values.networkPolicy "seatunnel" | default dict }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy @@ -29,15 +29,23 @@ spec: {{- include "hugegraph.netpol.seatunnelLabels" . | nindent 6 }} policyTypes: - Ingress + {{- if .Values.networkPolicy.enabled }} - Egress + {{- end }} ingress: - from: {{- include "hugegraph.netpol.seatunnelPeer" . | nindent 8 }} ports: {{- include "hugegraph.netpol.ports" (list 5801 8080) | nindent 8 }} + {{- if not .Values.networkPolicy.enabled }} + # Easy mode keeps the authenticated HTTP entry open, never the member port. + - ports: + {{- include "hugegraph.netpol.ports" (list 8080) | nindent 8 }} + {{- end }} {{- with get $rules "extraIngress" }} {{- toYaml . | nindent 4 }} {{- end }} + {{- if .Values.networkPolicy.enabled }} egress: - to: {{- include "hugegraph.netpol.seatunnelPeer" . | nindent 8 }} @@ -51,4 +59,5 @@ spec: {{- with get $rules "extraEgress" }} {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} {{- end }} diff --git a/helm/hugegraph/templates/seatunnel.yaml b/helm/hugegraph/templates/seatunnel.yaml index 75c425b48f..fabe1f269e 100644 --- a/helm/hugegraph/templates/seatunnel.yaml +++ b/helm/hugegraph/templates/seatunnel.yaml @@ -174,10 +174,33 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.seatunnel.priorityClassName }} + priorityClassName: {{ . | quote }} + {{- end }} + {{- with .Values.seatunnel.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.seatunnel.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.seatunnel.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.seatunnel.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: seatunnel image: {{ include "hugegraph.image" (dict "image" .Values.seatunnel.image "appVersion" "3.0.0") | quote }} imagePullPolicy: {{ .Values.seatunnel.image.pullPolicy }} + {{- with .Values.seatunnel.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} # The default upstream role is hybrid; an explicit -r is unnecessary. command: ["/opt/seatunnel/bin/seatunnel-cluster.sh"] env: diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 75006a0a52..4c0713c864 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -93,6 +93,24 @@ "type": "string" } } + }, + "podSecurityContext": { + "type": "object" + }, + "securityContext": { + "type": "object" + }, + "nodeSelector": { + "type": "object" + }, + "tolerations": { + "type": "array" + }, + "affinity": { + "type": "object" + }, + "priorityClassName": { + "type": "string" } } } diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index e6be800629..15a2784086 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -563,6 +563,19 @@ seatunnel: memory: 1536Mi limits: memory: 2Gi + # Match the other workloads. The upstream image runs as root; configure a + # compatible UID and writable paths before enabling runAsNonRoot. + podSecurityContext: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + nodeSelector: {} + tolerations: [] + affinity: {} + priorityClassName: "" # Native Kubernetes env entries; matching names override automatic HG_* values. # HG_HOST/PORT/PROTOCOL, HG_USERNAME/PASSWORD and HG_GRAPH_SPACE/NAME default to # this release's Server, existing admin Secret, graph space and graph_a. @@ -572,8 +585,9 @@ seatunnel: config: {} # Kubernetes NetworkPolicy: mode default plus explicit per-component rules. -# easy does not isolate clients; strict isolates release traffic and DNS. Only -# enforced when the network plugin implements NetworkPolicy (kind v0.25+, +# easy keeps HTTP clients open; strict isolates release traffic and DNS. +# Enabled SeaTunnel always restricts member port 5801 to its release peers. +# Enforced only when the network plugin implements NetworkPolicy (kind v0.25+, # k3s, Calico, Cilium); elsewhere the objects are accepted and do nothing. # Nothing outside the release is admitted unless it is listed in # .extraIngress as standard NetworkPolicy ingress rules: apps in @@ -595,8 +609,9 @@ networkPolicy: # through extraEnv). PD, Store and Server reach only the release and DNS. extraEgress: [] seatunnel: - # SeaTunnel peers, DNS and the HugeGraph Server API are allowed by default. - # Add submitters/monitoring here using native NetworkPolicy rules. + # Member port 5801 is restricted even when networkPolicy.enabled is false. + # With full isolation, peers, DNS and the Server API are allowed by default. + # Add trusted submitters/monitoring using native NetworkPolicy rules. extraIngress: [] # External source/sink systems and shared checkpoint storage need egress. extraEgress: [] diff --git a/helm/test_helm_seatunnel.py b/helm/test_helm_seatunnel.py index ee83eee6cd..a2f4741574 100644 --- a/helm/test_helm_seatunnel.py +++ b/helm/test_helm_seatunnel.py @@ -401,6 +401,55 @@ def allows(rules, direction, selector, port): dns = [port for rule in engine["egress"] if not rule.get("to") for port in rule["ports"]] self.assertTrue({("UDP", 53), ("TCP", 53)} <= {(p["protocol"], p["port"]) for p in dns}) + def test_member_ingress_is_isolated_when_full_network_policy_is_disabled(self): + for mode in ("easy", "strict"): + with self.subTest(mode=mode): + documents = self.documents({}, mode=mode, networkPolicy={"enabled": False}) + policy, = engine_resources(documents, "NetworkPolicy") + spec = policy["spec"] + self.assertEqual(["Ingress"], spec["policyTypes"]) + self.assertNotIn("egress", spec) + workload, = engine_resources(documents, "StatefulSet") + labels = pod(workload)["metadata"]["labels"] + self.assertTrue(matches(spec["podSelector"]["matchLabels"], labels)) + member_rules = [rule for rule in spec["ingress"] + if any(port["port"] == 5801 for port in rule["ports"])] + self.assertEqual(1, len(member_rules)) + self.assertEqual([{"podSelector": {"matchLabels": spec["podSelector"]["matchLabels"]}}], + member_rules[0]["from"]) + selector = member_rules[0]["from"][0]["podSelector"]["matchLabels"] + self.assertFalse(matches(selector, {**labels, "app.kubernetes.io/component": "server"})) + self.assertFalse(matches(selector, {**labels, "app.kubernetes.io/instance": "other"})) + open_rules = [rule for rule in spec["ingress"] if not rule.get("from")] + self.assertEqual([{"ports": [{"protocol": "TCP", "port": 8080}]}], open_rules) + + def test_engine_security_defaults_and_native_scheduling_overrides(self): + workload, = engine_resources(self.documents({}), "StatefulSet") + container, = pod(workload)["spec"]["containers"] + self.assertEqual({"allowPrivilegeEscalation": False, + "capabilities": {"drop": ["ALL"]}, + "seccompProfile": {"type": "RuntimeDefault"}}, container["securityContext"]) + overrides = { + "podSecurityContext": {"runAsUser": 1000, "fsGroup": 1000}, + "securityContext": {"runAsNonRoot": True}, + "nodeSelector": {"workload": "ingestion"}, + "tolerations": [{"key": "ingestion", "operator": "Exists", "effect": "NoSchedule"}], + "affinity": {"podAntiAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": []}}, + "priorityClassName": "ingestion", + } + workload, = engine_resources(self.documents(overrides), "StatefulSet") + spec = pod(workload)["spec"] + self.assertEqual(overrides["podSecurityContext"], spec["securityContext"]) + for field in ("nodeSelector", "tolerations", "affinity", "priorityClassName"): + self.assertEqual(overrides[field], spec[field]) + container, = spec["containers"] + self.assertTrue(container["securityContext"]["runAsNonRoot"]) + self.assertFalse(container["securityContext"]["allowPrivilegeEscalation"]) + for invalid in ({"securityContext": []}, {"nodeSelector": []}, {"tolerations": {}}, + {"affinity": []}, {"priorityClassName": 1}): + with self.subTest(invalid=invalid): + self.assertNotEqual(0, self.render(invalid).returncode) + def test_packaged_chart_has_both_modes_without_seatunnel_dependency(self): with tempfile.TemporaryDirectory(prefix="seatunnel-package-") as directory: self.run_helm(["package", str(CHART), "--destination", directory], From d7363c10f1966a7212d718731bdd3c636382117a Mon Sep 17 00:00:00 2001 From: orion Date: Sun, 25 Oct 2026 07:42:07 +0000 Subject: [PATCH 16/17] fix(helm): retain SeaTunnel JVM defaults Preserve G1 and OOM diagnostics in the generated options. Bound metaspace for the default container budget and document full overrides. Check both native deployment modes. --- helm/hugegraph/docs/seatunnel.md | 7 +++++-- helm/hugegraph/values.yaml | 5 ++++- helm/test_helm_seatunnel.py | 3 +++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/helm/hugegraph/docs/seatunnel.md b/helm/hugegraph/docs/seatunnel.md index 84f36150d4..14a209a2c2 100644 --- a/helm/hugegraph/docs/seatunnel.md +++ b/helm/hugegraph/docs/seatunnel.md @@ -190,7 +190,11 @@ require the explicit `HG_SOURCE_*` and `HG_TARGET_*` values below; the chart doe not discover remote graphs. Connector credentials do not replace entry auth. Tune the JVM with `seatunnel.javaOpts` and container resources with -`seatunnel.resources`. `seatunnel.podSecurityContext` and +`seatunnel.resources`. Defaults retain G1 and OOM heap dumps, with a 1 GiB maximum +heap and 256 MiB metaspace cap to leave room for native memory under the 2 GiB limit. +Heap dumps use writable `/tmp/seatunnel.hprof` and are ephemeral. `javaOpts` and +`config.jvm_options` each replace the complete image JVM options file; keep the +required GC and diagnostic flags when overriding either. `seatunnel.podSecurityContext` and `seatunnel.securityContext` follow the other chart components. The container starts with privilege escalation disabled, all capabilities dropped and `RuntimeDefault` seccomp. The upstream image runs as root: configure a compatible @@ -203,7 +207,6 @@ Override complete native configuration files through `seatunnel.config`, for exa ```yaml seatunnel: - javaOpts: "-Xms512m -Xmx1g" config: jvm_client_options: | -Xms128m diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 15a2784086..358f8df00c 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -556,7 +556,10 @@ seatunnel: username: admin # Empty generates a kept Secret; upgrades reuse the existing password. password: "" - javaOpts: "-Xms512m -Xmx1g" + # Replaces the image's jvm_options; reserve memory outside the heap under 2Gi. + javaOpts: >- + -Xms512m -Xmx1g -XX:+UseG1GC -XX:MaxMetaspaceSize=256m + -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp/seatunnel.hprof resources: requests: cpu: 500m diff --git a/helm/test_helm_seatunnel.py b/helm/test_helm_seatunnel.py index a2f4741574..5f5e62e139 100644 --- a/helm/test_helm_seatunnel.py +++ b/helm/test_helm_seatunnel.py @@ -153,6 +153,9 @@ def test_modes_share_one_native_hybrid_workload_and_services(self): self.assertIn("/opt/seatunnel/bin/seatunnel-cluster.sh", launch) self.assertNotIn(" -r ", launch) data = engine_config(documents) + for option in ("-Xms512m", "-Xmx1g", "-XX:+UseG1GC", "-XX:MaxMetaspaceSize=256m", + "-XX:+HeapDumpOnOutOfMemoryError", "-XX:HeapDumpPath=/tmp/seatunnel.hprof"): + self.assertIn(option, data["jvm_options"].splitlines()) engine = yaml.safe_load(data["seatunnel.yaml"])["seatunnel"]["engine"] self.assertEqual(backups, engine["backup-count"]) self.assertTrue(engine["http"]["enable-http"]) From 806064d44f1ba8ee1f45cce8d6092dce6c863ca7 Mon Sep 17 00:00:00 2001 From: imbajin Date: Mon, 26 Oct 2026 01:09:35 +0800 Subject: [PATCH 17/17] fix(helm): keep failed schema copies isolated --- .../examples/seatunnel/clone/copy-schema.py | 16 ++- helm/schema_copy/__init__.py | 14 ++ helm/schema_copy/test_helm_seatunnel.py | 121 ++++++++++++++++++ 3 files changed, 148 insertions(+), 3 deletions(-) mode change 100644 => 100755 helm/hugegraph/examples/seatunnel/clone/copy-schema.py create mode 100644 helm/schema_copy/__init__.py create mode 100755 helm/schema_copy/test_helm_seatunnel.py diff --git a/helm/hugegraph/examples/seatunnel/clone/copy-schema.py b/helm/hugegraph/examples/seatunnel/clone/copy-schema.py old mode 100644 new mode 100755 index 0fb19bc83b..79ace9e43a --- a/helm/hugegraph/examples/seatunnel/clone/copy-schema.py +++ b/helm/hugegraph/examples/seatunnel/clone/copy-schema.py @@ -86,10 +86,20 @@ def main(): if kind == 'indexlabels': body['rebuild'] = False # Empty graph; subsequent data writes build indexes. target('POST', '/schema/' + kind, body) - finally: + if normalize(schema) != normalize(target('GET', '/schema')): + raise RuntimeError('Schema comparison failed') + except Exception as error: + raise SystemExit( + 'Schema copy failed; the target remains in RESTORING mode. ' + 'Inspect it or delete and recreate the graph before copying data: ' + f'{error}') from error + try: target('PUT', '/mode', original_mode) - if normalize(schema) != normalize(target('GET', '/schema')): - raise SystemExit('Schema comparison failed; inspect the new graph before copying data.') + except Exception as error: + raise SystemExit( + 'Schema copy verified, but restoring the target mode failed; ' + 'the target remains in RESTORING mode: ' + f'{error}') from error print(json.dumps({'schema_equal': True, 'ids_preserved': True, 'counts': {kind: len(schema[kind]) for kind in KINDS}})) diff --git a/helm/schema_copy/__init__.py b/helm/schema_copy/__init__.py new file mode 100644 index 0000000000..a6eb1b544f --- /dev/null +++ b/helm/schema_copy/__init__.py @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/helm/schema_copy/test_helm_seatunnel.py b/helm/schema_copy/test_helm_seatunnel.py new file mode 100755 index 0000000000..6db464873a --- /dev/null +++ b/helm/schema_copy/test_helm_seatunnel.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Failure-safety regression tests for the SeaTunnel schema-copy helper.""" + +import contextlib +import copy +import importlib.util +import io +from pathlib import Path +import sys +import unittest +from unittest import mock + + +SCRIPT = (Path(__file__).resolve().parents[1] / + "hugegraph/examples/seatunnel/clone/copy-schema.py") +SPEC = importlib.util.spec_from_file_location("seatunnel_copy_schema", SCRIPT) +COPY_SCHEMA = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(COPY_SCHEMA) + + +def schema(property_names=()): + result = {kind: [] for kind in COPY_SCHEMA.KINDS} + result["propertykeys"] = [ + {"id": index + 1, "name": name, "data_type": "TEXT", + "cardinality": "SINGLE", "user_data": {}} + for index, name in enumerate(property_names) + ] + return result + + +class SchemaCopyFailureSafetyTest(unittest.TestCase): + + def run_copy(self, source_schema, target): + source = mock.Mock(return_value=source_schema) + with mock.patch.object(COPY_SCHEMA, "client", + side_effect=[source, target]), \ + mock.patch.object(sys, "argv", [str(SCRIPT), + "--source", "http://source/graph", + "--target", "http://target/graph"]): + return COPY_SCHEMA.main() + + def test_partial_post_failure_keeps_target_in_restoring_mode(self): + source_schema = schema(("name", "city")) + calls = [] + posts = 0 + + def target(method, suffix, body=None): + nonlocal posts + calls.append((method, suffix, body)) + if (method, suffix) == ("GET", "/schema"): + return schema() + if (method, suffix) == ("GET", "/mode"): + return {"mode": "NONE"} + if method == "POST": + posts += 1 + if posts == 2: + raise RuntimeError("injected schema write failure") + return None + + with self.assertRaisesRegex(SystemExit, "remains in RESTORING mode"): + self.run_copy(source_schema, target) + self.assertEqual(["RESTORING"], [body for method, suffix, body in calls + if (method, suffix) == ("PUT", "/mode")]) + + def test_schema_mismatch_keeps_target_in_restoring_mode(self): + source_schema = schema(("name",)) + calls = [] + schema_reads = iter((schema(), schema())) + + def target(method, suffix, body=None): + calls.append((method, suffix, body)) + if (method, suffix) == ("GET", "/schema"): + return next(schema_reads) + if (method, suffix) == ("GET", "/mode"): + return {"mode": "NONE"} + return None + + with self.assertRaisesRegex(SystemExit, "Schema comparison failed"): + self.run_copy(source_schema, target) + self.assertEqual(["RESTORING"], [body for method, suffix, body in calls + if (method, suffix) == ("PUT", "/mode")]) + + def test_verified_copy_restores_original_mode(self): + source_schema = schema(("name",)) + calls = [] + schema_reads = iter((schema(), copy.deepcopy(source_schema))) + + def target(method, suffix, body=None): + calls.append((method, suffix, body)) + if (method, suffix) == ("GET", "/schema"): + return next(schema_reads) + if (method, suffix) == ("GET", "/mode"): + return {"mode": "NONE"} + return None + + output = io.StringIO() + with contextlib.redirect_stdout(output): + self.run_copy(source_schema, target) + self.assertEqual(["RESTORING", "NONE"], + [body for method, suffix, body in calls + if (method, suffix) == ("PUT", "/mode")]) + self.assertIn('"schema_equal": true', output.getvalue()) + + +if __name__ == "__main__": + unittest.main()