From 21c2eb751831372b63c35a74cdaf75396559215a Mon Sep 17 00:00:00 2001 From: orion Date: Fri, 23 Oct 2026 14:11:52 +0000 Subject: [PATCH 01/29] feat(docker): reuse deployment images with a selectable Topling runtime --- .github/workflows/pd-store-ci.yml | 1 + docker/README.md | 63 ++++++++ docker/bake.hcl | 23 ++- docker/docker-compose.dev.yml | 8 + docker/prepare-topling-image.sh | 52 +++++++ docker/test-compose.sh | 48 +++++- docker/test-topling-images.sh | 84 +++++++++++ docker/topling-runtime.env | 19 +++ docs/storage-lifecycle.md | 21 +++ helm/hugegraph/README.md | 33 ++++- helm/hugegraph/templates/_helpers.tpl | 26 ++++ helm/hugegraph/templates/pd-statefulset.yaml | 2 + .../templates/store-statefulset.yaml | 2 + .../templates/tests/test-connection.yaml | 5 + .../persistence_upgrade_persistent_test.yaml | 9 ++ .../hugegraph/tests/runtime_upgrade_test.yaml | 57 +++++++ .../hugegraph/tests/runtime_variant_test.yaml | 139 ++++++++++++++++++ .../hugegraph/tests/validate_values_test.yaml | 9 ++ helm/hugegraph/values.schema.json | 7 + helm/hugegraph/values.yaml | 3 + hugegraph-pd/Dockerfile | 62 +++++++- .../hg-pd-dist/docker/docker-entrypoint.sh | 15 ++ hugegraph-server/Dockerfile | 62 +++++++- hugegraph-server/Dockerfile-hstore | 32 +++- .../docker/docker-entrypoint-test.sh | 28 ++++ .../docker/docker-entrypoint.sh | 17 +++ hugegraph-store/Dockerfile | 62 +++++++- .../hg-store-dist/docker/docker-entrypoint.sh | 15 ++ 28 files changed, 878 insertions(+), 26 deletions(-) create mode 100644 docker/prepare-topling-image.sh create mode 100644 docker/test-topling-images.sh create mode 100644 docker/topling-runtime.env create mode 100644 helm/hugegraph/tests/runtime_upgrade_test.yaml create mode 100644 helm/hugegraph/tests/runtime_variant_test.yaml diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index e4dd1809ea..aab6c3cb2e 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -189,6 +189,7 @@ jobs: run: | bash $TRAVIS_DIR/test-topling-selection.sh bash hugegraph-store/hg-store-dist/src/assembly/travis/test-store-preloads.sh + bash docker/test-topling-images.sh - name: Run PD docker entrypoint secret override tests run: | diff --git a/docker/README.md b/docker/README.md index 84c6f14044..d04b11775b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -272,6 +272,69 @@ The Hubble `latest` image is expected to work with HugeGraph Server 1.7 and Serv +### ToplingDB images + +Select the ToplingDB variant with the existing image-version parameter. The same +Compose files, credentials, ports and lifecycle commands apply: + +```bash +export HUGEGRAPH_VERSION=latest-topling +# Standalone Server with embedded ToplingDB +# A distinct project creates fresh volumes for this runtime. +docker compose -p hugegraph-topling-standalone -f docker-compose.yml up -d --wait + +# Or PD + Store using ToplingDB, with an HStore Server +# Generate conf/hubble/hstore.local.properties as described above first. +docker compose -p hugegraph-topling-hstore -f docker-compose-hstore.yml up -d --wait +``` + +Use the same project name on subsequent `ps`, `stop`, `down` and `up` commands. +Persist `HUGEGRAPH_VERSION=latest-topling` in that deployment's `.env` if preferred. +A published version uses `-topling`; the unsuffixed tag keeps RocksDB. +Topling images support **Linux amd64**. All HugeGraph images in a deployment set +use the same source revision. HStore Server keeps `backend=hstore` and delegates +storage to PD/Store; it does not load a local Topling JNI runtime. Hubble keeps its +independent image tag. + +The images include the verified JNI runtime and set both launcher and business +provider configuration. No JAR download or `LD_PRELOAD` setting is needed at startup. +Existing data is not converted by selecting another image: use new volumes for +ToplingDB, and migrate through the graph APIs after validating your workload. +Returning to RocksDB uses its original deployment and volumes. See +[Topling runtime preparation and validation](../docs/storage-lifecycle.md#storage-provider-selection-and-toplingdb) +for the native configuration and compatibility requirements. + +For a local HStore source build, use the existing development overlay. Supply an +immutable Topling JNI URL and its trusted SHA-256; both PD and Store reuse them: + +```bash +export RUNTIME_VARIANT=topling +export TOPLING_JNI_URL='https://your-artifact-host/immutable/rocksdbjni.jar' +export TOPLING_JNI_SHA256='' +docker compose -p hugegraph-topling-dev \ + -f docker-compose-hstore.yml -f docker-compose.dev.yml up -d --build --wait +``` + +The overlay uses the usual local `:dev` tags. Build it on Linux amd64 and keep its +project volumes separate from a RocksDB source deployment. For direct Docker and +Bake builds, use `--build-arg RUNTIME_VARIANT=topling` or `RUNTIME_VARIANT=topling` +respectively, together with the same JNI URL and checksum build inputs. + +For a verified local JNI JAR, supply a BuildKit secret instead of a download URL: + +```bash +docker buildx build --platform linux/amd64 -f ../hugegraph-server/Dockerfile \ + --build-arg RUNTIME_VARIANT=topling \ + --build-arg TOPLING_JNI_SHA256='' \ + --secret id=topling_jni,src=/absolute/path/rocksdbjni.jar \ + -t hugegraph/hugegraph:local-topling --load .. +``` + +Run this example from the `docker` directory. The checksum and normal native +capability probe still apply. Networks with a private CA can additionally supply +`--secret id=proxy_ca,src=/absolute/path/ca.pem`; the CA is mounted only while +Maven, package installation and artifact download run. + ### Data persistence Each topology creates its own normal Compose network and named volumes. No network or volume needs to be created in advance. diff --git a/docker/bake.hcl b/docker/bake.hcl index 2fc1cf4626..fce84d6c30 100644 --- a/docker/bake.hcl +++ b/docker/bake.hcl @@ -31,6 +31,22 @@ variable "SOURCE_REVISION" { default = "local" } +variable "SOURCE_URL" { + default = "https://github.com/apache/hugegraph" +} + +variable "RUNTIME_VARIANT" { + default = "standard" +} + +variable "TOPLING_JNI_URL" { + default = null +} + +variable "TOPLING_JNI_SHA256" { + default = null +} + variable "IMAGE_TAG" { default = "local" } @@ -50,8 +66,13 @@ target "_common" { MAVEN_PROJECTS = MAVEN_PROJECTS RUNTIME_DEPS_EPOCH = RUNTIME_DEPS_EPOCH SOURCE_REVISION = SOURCE_REVISION + SOURCE_REPOSITORY = SOURCE_URL + RUNTIME_VARIANT = RUNTIME_VARIANT + TOPLING_JNI_URL = TOPLING_JNI_URL + TOPLING_JNI_SHA256 = TOPLING_JNI_SHA256 } - platforms = [ + # The published Topling JNI currently contains Linux x86_64 native code only. + platforms = RUNTIME_VARIANT == "topling" ? ["linux/amd64"] : [ "linux/amd64", "linux/arm64", ] diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 4da9c32e19..6d33219fc1 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -22,6 +22,10 @@ services: build: context: .. dockerfile: hugegraph-pd/Dockerfile + args: + RUNTIME_VARIANT: ${RUNTIME_VARIANT:-standard} + TOPLING_JNI_URL: ${TOPLING_JNI_URL:-} + TOPLING_JNI_SHA256: ${TOPLING_JNI_SHA256:-} healthcheck: start_period: 20s @@ -31,6 +35,10 @@ services: build: context: .. dockerfile: hugegraph-store/Dockerfile + args: + RUNTIME_VARIANT: ${RUNTIME_VARIANT:-standard} + TOPLING_JNI_URL: ${TOPLING_JNI_URL:-} + TOPLING_JNI_SHA256: ${TOPLING_JNI_SHA256:-} healthcheck: start_period: 30s diff --git a/docker/prepare-topling-image.sh b/docker/prepare-topling-image.sh new file mode 100644 index 0000000000..dde641ff21 --- /dev/null +++ b/docker/prepare-topling-image.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Called from the stopped component directory during an image build only. +set -euo pipefail + +if [[ -z "${TOPLING_JNI_URL:-}" || -z "${TOPLING_JNI_SHA256:-}" ]] && + [[ -f /tmp/topling-runtime.env ]]; then + requested_url="${TOPLING_JNI_URL:-}" + requested_sha="${TOPLING_JNI_SHA256:-}" + # Checked-in, reviewed artifact defaults; explicit build inputs win. + source /tmp/topling-runtime.env + TOPLING_JNI_URL="${requested_url:-${TOPLING_JNI_URL:-}}" + TOPLING_JNI_SHA256="${requested_sha:-${TOPLING_JNI_SHA256:-}}" +fi + +[[ "${TOPLING_JNI_SHA256:-}" =~ ^[[:xdigit:]]{64}$ ]] || { + echo 'Error: Topling builds require a trusted TOPLING_JNI_SHA256' >&2 + exit 1 +} +export TOPLING_JNI_SHA256 + +if [[ -r /run/secrets/topling_jni ]]; then + export TOPLING_JNI_JAR=/run/secrets/topling_jni +else + [[ "${TOPLING_JNI_URL:-}" == https://* ]] || { + echo 'Error: Topling builds require an HTTPS TOPLING_JNI_URL or a topling_jni build secret' >&2 + exit 1 + } + download_dir=$(mktemp -d) + trap 'rm -rf "$download_dir"' EXIT + curl --fail --location --retry 3 --proto '=https' --proto-redir '=https' \ + --output "$download_dir/rocksdbjni.jar" "$TOPLING_JNI_URL" + export TOPLING_JNI_JAR="$download_dir/rocksdbjni.jar" +fi +# The shared preparer verifies the hash, ELF/dependencies, and real EasyMigrate +# behavior before installing a component-local runtime and checksum receipt. +bash bin/prepare-topling.sh diff --git a/docker/test-compose.sh b/docker/test-compose.sh index 54418d413a..e3fb826aa3 100644 --- a/docker/test-compose.sh +++ b/docker/test-compose.sh @@ -40,10 +40,13 @@ RENDER_DIR="" # baseline render shows the Compose default whatever the developer exported; # render_with_timeout swaps in a value. Kept as one array rather than a second # copy of the environment below, so a variable added there reaches both renders. -STARTUP_TIMEOUT_ENV=(-u HG_SERVER_STARTUP_TIMEOUT_S) +STARTUP_TIMEOUT_ENV=() +BUILD_RUNTIME_ENV=() compose_auth() { - env "${STARTUP_TIMEOUT_ENV[@]}" \ + env -u HG_SERVER_STARTUP_TIMEOUT_S -u RUNTIME_VARIANT \ + -u TOPLING_JNI_URL -u TOPLING_JNI_SHA256 \ + "${STARTUP_TIMEOUT_ENV[@]}" "${BUILD_RUNTIME_ENV[@]}" \ HUGEGRAPH_VERSION="${VERSION}" \ HUBBLE_IMAGE="${RENDER_HUBBLE_IMAGE}" \ HUGEGRAPH_ADMIN_PASSWORD="${PASSWORD}" \ @@ -385,6 +388,45 @@ hubble_password_helper_check() { "${DOCKER_DIR}/set-hubble-pd-password.sh" hstore "${PD_SECRET}" >/dev/null } +# A published runtime is selected with the existing version parameter. Every +# other setting, including volumes and HStore topology, must render identically. +assert_topling_variants() { + local topology file + for topology in standalone hstore ha; do + case "${topology}" in + standalone) file=docker-compose.yml ;; + hstore) file=docker-compose-hstore.yml ;; + ha) file=docker-compose-3pd-3store-3server.yml ;; + esac + ( + VERSION=ci-version-topling + render "${RENDER_DIR}/${topology}-topling.json" -f "${DOCKER_DIR}/${file}" + ) + jq -e --slurpfile standard "${RENDER_DIR}/${topology}.json" ' + (.services |= with_entries( + if .key != "hubble" then .value.image |= sub("-topling$"; "") + else . end)) | + {services, volumes, networks, name} == + ($standard[0] | {services, volumes, networks, name}) + ' "${RENDER_DIR}/${topology}-topling.json" >/dev/null + done + ( + BUILD_RUNTIME_ENV=(RUNTIME_VARIANT=topling TOPLING_JNI_URL=https://example.invalid/jni.jar + TOPLING_JNI_SHA256=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef) + render "${RENDER_DIR}/dev-topling.json" \ + -f "${DOCKER_DIR}/docker-compose-hstore.yml" \ + -f "${DOCKER_DIR}/docker-compose.dev.yml" + ) + jq -e ' + .services.pd.build.args.RUNTIME_VARIANT == "topling" and + .services.store.build.args.RUNTIME_VARIANT == "topling" and + .services.pd.build.args.TOPLING_JNI_SHA256 == + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" and + .services.server.build.args.RUNTIME_VARIANT == null and + .services.server.environment.HG_SERVER_BACKEND == "hstore" + ' "${RENDER_DIR}/dev-topling.json" >/dev/null +} + run_render() { RENDER_DIR="$(mktemp -d)" trap cleanup EXIT INT TERM @@ -407,6 +449,8 @@ run_render() { assert_dev_override "${RENDER_DIR}/dev.json" \ "${RENDER_DIR}/override.json" + assert_topling_variants + # An absent host value renders the entrypoint default in every topology and # a host value overrides it, so the container budget can be lowered to meet # the healthcheck budget without editing the Compose files. Raising it past diff --git a/docker/test-topling-images.sh b/docker/test-topling-images.sh new file mode 100644 index 0000000000..57dd85c8e9 --- /dev/null +++ b/docker/test-topling-images.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euo pipefail +repo=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +python3 - "$repo" <<'PY' +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + +repo = Path(sys.argv[1]) +required = { + "pd": { + "HG_PD_GRPC_HOST": "pd", + "HG_PD_RAFT_ADDRESS": "pd:8610", + "HG_PD_RAFT_PEERS_LIST": "pd:8610", + "HG_PD_INITIAL_STORE_LIST": "store:8500", + "HG_PD_AUTH_SECRET_KEY": "test-secret", + }, + "store": { + "HG_STORE_PD_ADDRESS": "pd:8686", + "HG_STORE_GRPC_HOST": "store", + "HG_STORE_RAFT_ADDRESS": "store:8510", + }, +} +with tempfile.TemporaryDirectory(prefix="topling-entrypoints-") as temporary: + for component, settings in required.items(): + directory = Path(temporary) / component + (directory / "bin").mkdir(parents=True) + entrypoint = repo / f"hugegraph-{component}/hg-{component}-dist/docker/docker-entrypoint.sh" + shutil.copy(entrypoint, directory / "entrypoint.sh") + launcher = directory / f"bin/start-hugegraph-{component}.sh" + launcher.write_text('#!/bin/bash\nprintf \'%s\' "$SPRING_APPLICATION_JSON" > spring.json\n') + launcher.chmod(0o755) + environment = dict(os.environ, **settings) + environment.pop("TOPLINGDB_ROCKSDB_PROVIDER", None) + for provider in (None, "rocksdb", "topling"): + if provider is not None: + environment["TOPLINGDB_ROCKSDB_PROVIDER"] = provider + result = subprocess.run(["bash", "entrypoint.sh"], cwd=directory, env=environment, + capture_output=True, text=True) + assert result.returncode == 0, result.stderr + document = json.loads((directory / "spring.json").read_text()) + if provider is None: + assert "rocksdb" not in document, "default must preserve mounted provider" + else: + assert document["rocksdb"]["provider"] == provider, document + (directory / "spring.json").unlink() + environment["TOPLINGDB_ROCKSDB_PROVIDER"] = 'topling", "unexpected":true' + result = subprocess.run(["bash", "entrypoint.sh"], cwd=directory, env=environment, + capture_output=True, text=True) + assert result.returncode != 0 and not (directory / "spring.json").exists() + print(f"PASS: {component} provider selection, default preservation and invalid input") + + environment = dict(os.environ) + environment.pop("TOPLING_JNI_SHA256", None) + environment.pop("TOPLING_JNI_URL", None) + helper = repo / "docker/prepare-topling-image.sh" + for values in ({}, {"TOPLING_JNI_SHA256": "not-a-sha"}, + {"TOPLING_JNI_SHA256": "a" * 64, "TOPLING_JNI_URL": "http://example.invalid/jni.jar"}): + result = subprocess.run(["bash", str(helper)], cwd=temporary, env=dict(environment, **values), + capture_output=True, text=True) + assert result.returncode != 0, "unverified artifact accepted" + assert "Error: Topling builds require" in result.stderr, result.stderr + print("PASS: image preparation rejects missing/untrusted artifact inputs") +PY diff --git a/docker/topling-runtime.env b/docker/topling-runtime.env new file mode 100644 index 0000000000..95a00982b7 --- /dev/null +++ b/docker/topling-runtime.env @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Pin the verified immutable fork release here after its native CI probe passes. +# Until then, Topling source builds require explicit URL and SHA-256 build args. diff --git a/docs/storage-lifecycle.md b/docs/storage-lifecycle.md index ddf7731a0a..d79ea5e2d7 100644 --- a/docs/storage-lifecycle.md +++ b/docs/storage-lifecycle.md @@ -495,6 +495,27 @@ For a distributed graph, Server uses `backend=hstore`; PD and Store own its pers storage. A standalone Server keeps `backend=rocksdb`. The launcher selection and the business provider must agree. A mismatch fails before opening that component's database. +### Docker, Compose and Helm + +Prepared image variants reuse the normal deployment mechanisms. Select +`hugegraph/hugegraph:latest-topling` for a standalone Server, or the matching +`hugegraph/pd`, `hugegraph/store` and `hugegraph/server` tags for distributed +HStore. Published versions use `-topling`. Topling images support Linux +amd64 and include the verified JNI runtime; startup does not download dependencies. + +Compose uses its existing `HUGEGRAPH_VERSION=latest-topling` parameter. Helm uses +`--set global.runtimeVariant=topling` to select matching images and amd64 scheduling. +The HStore Server keeps `backend=hstore` and uses the Topling-enabled PD and Store; +it does not load a local Topling runtime. The images configure the business +provider and launcher together. See the [Compose guide](../docker/README.md#toplingdb-images) +and [Helm guide](../helm/hugegraph/README.md#toplingdb-runtime) for complete commands, +source builds, custom registries and immutable image digests. + +Select a new Compose project or Helm release with fresh volumes for validation. +Changing the image does not migrate an existing RocksDB database. Keep the original +deployment's data and use graph APIs for a verified migration; the Helm chart +rejects runtime changes on an initialized persistent release. + ### Standalone Server setup ![Switch RocksDB to ToplingDB](images/switch-rocksdb-to-toplingdb.png) diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 5f12ba6435..38a3814e45 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -97,6 +97,36 @@ Store's wait init container and Helm test reuse the Store and Server images, which already contain `curl`; no separate curl image is needed by default. See the [complete registry example](examples/values-internal-registry.yaml). +### ToplingDB runtime + +Select ToplingDB for a **fresh deployment** with one value: + +```bash +helm install hugegraph-topling ./helm/hugegraph -n hugegraph-topling \ + --create-namespace --set global.runtimeVariant=topling --wait --timeout 15m +``` + +This selects the matching `-topling` image tags for PD, Store and HStore Server, +including the Store wait container and Helm test image. PD and Store own the +ToplingDB databases; Server continues to use `backend=hstore`. Hubble keeps its +independent image. Linux amd64 scheduling is added to these HugeGraph workloads +and the test hook. Contradictory OS/architecture node selectors are rejected; +other scheduling constraints are preserved. + +A component tag such as `1.8.0` becomes `1.8.0-topling`; tags already ending in +`-topling` are kept. Custom repositories and `global.imageRegistry` work as usual. +An explicit `image.digest` remains exact and takes priority over the derived tag; +supply the digest of the matching Topling deployment image. Explicit helper image +overrides also remain exact. + +The default `global.runtimeVariant=standard` keeps RocksDB. Selecting another +runtime does not convert existing data. Use separate empty volumes and migrate +through graph APIs after validating the application. The chart rejects runtime +changes on an existing persistent PD or Store release, including a return from +ToplingDB to RocksDB. Keep the original release and volumes to return to RocksDB. +See the [storage runtime guide](../../docs/storage-lifecycle.md#storage-provider-selection-and-toplingdb) +for native configuration and validation. + ### Storage Use a named dynamic StorageClass when the cluster has no suitable default: @@ -166,7 +196,7 @@ Use `--set mode=strict` or `-f helm/hugegraph/values-strict.yaml`. **Explicit component value > global value > mode default.** An explicit `false` is preserved; null component values inherit. The global settings are -`persistence.enabled` and `global.imageRegistry`. No mode requires HTTPS. +`persistence.enabled`, `global.imageRegistry` and `global.runtimeVariant`. No mode requires HTTPS. | Entry | Access | When to use it | |---|---|---| @@ -204,6 +234,7 @@ with `-f`; see [values.yaml](values.yaml) for all fields and their comments. | `pd.service.type` | PD client entry type | `ClusterIP` | | `*.service.nodePort` | Fixed Server/Hubble port; omitted/0 allocates automatically | unset | | `global.imageRegistry` | Prefix image repositories without an explicit registry | `""` | +| `global.runtimeVariant` | `standard` RocksDB or Linux amd64 `topling` deployment images | `standard` | | `*.image.repository` / `tag` / `digest` | Component image; digest takes priority | see values | | `persistence.enabled` | Global persistent storage | `true` | | `pd.storage.enabled` / `store.storage.enabled` / `hubble.persistence.enabled` | Explicit component storage override | `null` (inherits) | diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index e31813e60e..c2c89de9af 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -45,6 +45,27 @@ {{- $_ := set $component $key $storage -}} {{- if or (not (hasKey $storage "enabled")) (eq (get $storage "enabled" | toString) "") -}}{{- $_ := set $storage "enabled" $persistent -}}{{- end -}} {{- end -}} +{{/* The runtime is an image variant. Keep business/JNI configuration in the image. */}} +{{- $variant := get (.Values.global | default dict) "runtimeVariant" | default "standard" -}} +{{- if not (has $variant (list "standard" "topling")) -}} +{{- fail "global.runtimeVariant must be standard or topling" -}} +{{- end -}} +{{- if eq $variant "topling" -}} +{{- range $comp := list "pd" "store" "server" -}} +{{- $component := get $.Values $comp -}} +{{- $image := $component.image -}} +{{- $tag := $image.tag | default $.Chart.AppVersion -}} +{{- if not (hasSuffix "-topling" $tag) -}}{{- $_ := set $image "tag" (printf "%s-topling" $tag) -}}{{- end -}} +{{- $selector := get $component "nodeSelector" | default dict -}} +{{- range $key, $required := dict "kubernetes.io/arch" "amd64" "kubernetes.io/os" "linux" -}} +{{- if and (hasKey $selector $key) (ne (get $selector $key) $required) -}} +{{- fail (printf "%s.nodeSelector.%s must be %s for global.runtimeVariant=topling" $comp $key $required) -}} +{{- end -}} +{{- $_ := set $selector $key $required -}} +{{- end -}} +{{- $_ := set $component "nodeSelector" $selector -}} +{{- end -}} +{{- end -}} {{- $registry := trimSuffix "/" (get (.Values.global | default dict) "imageRegistry" | default "") -}} {{- if contains "://" $registry -}}{{- fail "global.imageRegistry must be a registry host/path without http:// or https://" -}}{{- end -}} {{- if $registry -}} @@ -793,6 +814,11 @@ and the upgrade passes. {{- fail (printf "%s.storage.enabled cannot change on an initialized release: StatefulSet volumeClaimTemplates are immutable and changing storage can discard data. Use a fresh release and migrate data" $comp) -}} {{- end -}} +{{- $liveRuntime := dig "metadata" "annotations" "hugegraph.apache.org/rocksdb-runtime" "standard" $live -}} +{{- $runtime := get ($.Values.global | default dict) "runtimeVariant" | default "standard" -}} +{{- if and $wasPersistent (ne $liveRuntime $runtime) -}} +{{- fail (printf "global.runtimeVariant cannot change from %s to %s on an initialized persistent %s release. Use a fresh release with empty volumes and migrate data through graph APIs" $liveRuntime $runtime $comp) -}} +{{- end -}} {{- $liveReplicas := int (dig "spec" "replicas" 0 $live) -}} {{- $desired := int (get (get $.Values $comp) "replicas") -}} {{- if and (gt $liveReplicas 0) (lt $desired $liveReplicas) -}} diff --git a/helm/hugegraph/templates/pd-statefulset.yaml b/helm/hugegraph/templates/pd-statefulset.yaml index 0546070ac8..7c5a67f250 100644 --- a/helm/hugegraph/templates/pd-statefulset.yaml +++ b/helm/hugegraph/templates/pd-statefulset.yaml @@ -20,6 +20,8 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: {{ include "hugegraph.pd.name" . }} + annotations: + hugegraph.apache.org/rocksdb-runtime: {{ get (.Values.global | default dict) "runtimeVariant" | default "standard" | quote }} labels: {{- include "hugegraph.labels" . | nindent 4 }} app.kubernetes.io/component: pd diff --git a/helm/hugegraph/templates/store-statefulset.yaml b/helm/hugegraph/templates/store-statefulset.yaml index 81c143ee8a..7cad492516 100644 --- a/helm/hugegraph/templates/store-statefulset.yaml +++ b/helm/hugegraph/templates/store-statefulset.yaml @@ -20,6 +20,8 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: {{ include "hugegraph.store.name" . }} + annotations: + hugegraph.apache.org/rocksdb-runtime: {{ get (.Values.global | default dict) "runtimeVariant" | default "standard" | quote }} labels: {{- include "hugegraph.labels" . | nindent 4 }} app.kubernetes.io/component: store diff --git a/helm/hugegraph/templates/tests/test-connection.yaml b/helm/hugegraph/templates/tests/test-connection.yaml index 020c484f6a..be800d0915 100644 --- a/helm/hugegraph/templates/tests/test-connection.yaml +++ b/helm/hugegraph/templates/tests/test-connection.yaml @@ -45,6 +45,11 @@ metadata: # run replaces it, so both forms exit 0 and a failed Pod keeps its logs. "helm.sh/hook-delete-policy": before-hook-creation spec: + {{- if eq (get (.Values.global | default dict) "runtimeVariant" | default "standard") "topling" }} + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux + {{- end }} {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 4 }} diff --git a/helm/hugegraph/tests/persistence_upgrade_persistent_test.yaml b/helm/hugegraph/tests/persistence_upgrade_persistent_test.yaml index 378ae5705c..914c9878af 100644 --- a/helm/hugegraph/tests/persistence_upgrade_persistent_test.yaml +++ b/helm/hugegraph/tests/persistence_upgrade_persistent_test.yaml @@ -113,3 +113,12 @@ tests: asserts: - isKind: of: Deployment + + - it: rejects reusing persistent RocksDB volumes with Topling + release: + upgrade: true + set: + global.runtimeVariant: topling + asserts: + - failedTemplate: + errorPattern: "global.runtimeVariant cannot change from standard to topling" diff --git a/helm/hugegraph/tests/runtime_upgrade_test.yaml b/helm/hugegraph/tests/runtime_upgrade_test.yaml new file mode 100644 index 0000000000..be59251976 --- /dev/null +++ b/helm/hugegraph/tests/runtime_upgrade_test.yaml @@ -0,0 +1,57 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Topling persistent runtime upgrade guard +templates: + - server-deployment.yaml +kubernetesProvider: + scheme: + "apps/v1/StatefulSet": + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: RELEASE-NAME-hugegraph-pd + namespace: NAMESPACE + annotations: + hugegraph.apache.org/rocksdb-runtime: topling + spec: + replicas: "3" + volumeClaimTemplates: + - metadata: + name: pd-data +tests: + - it: rejects returning Topling volumes to the standard runtime + release: + upgrade: true + asserts: + - failedTemplate: + errorPattern: "global.runtimeVariant cannot change from topling to standard" + + - it: accepts an unchanged persistent Topling upgrade + release: + upgrade: true + set: + global.runtimeVariant: topling + asserts: + - isKind: + of: Deployment diff --git a/helm/hugegraph/tests/runtime_variant_test.yaml b/helm/hugegraph/tests/runtime_variant_test.yaml new file mode 100644 index 0000000000..a71f66c172 --- /dev/null +++ b/helm/hugegraph/tests/runtime_variant_test.yaml @@ -0,0 +1,139 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Topling runtime image selection +templates: + - pd-statefulset.yaml + - store-statefulset.yaml + - server-deployment.yaml + - hubble-deployment.yaml + - tests/test-connection.yaml +tests: + - it: selects the matching PD runtime image and architecture + template: pd-statefulset.yaml + set: + global.runtimeVariant: topling + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: hugegraph/pd:latest-topling + - equal: + path: spec.template.spec.nodeSelector + value: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux + - equal: + path: metadata.annotations["hugegraph.apache.org/rocksdb-runtime"] + value: topling + + - it: reuses the selected Store image for its wait helper + template: store-statefulset.yaml + set: + global.runtimeVariant: topling + store.image.tag: 1.8.0 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: hugegraph/store:1.8.0-topling + - equal: + path: spec.template.spec.initContainers[0].image + value: hugegraph/store:1.8.0-topling + - equal: + path: spec.template.spec.nodeSelector["kubernetes.io/arch"] + value: amd64 + + - it: selects the matching HStore Server image without changing the backend + template: server-deployment.yaml + set: + global.runtimeVariant: topling + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: hugegraph/server:latest-topling + - equal: + path: spec.template.spec.nodeSelector["kubernetes.io/arch"] + value: amd64 + - contains: + path: spec.template.spec.containers[0].env + content: + name: HG_SERVER_BACKEND + value: hstore + + - it: keeps the independent Hubble image + template: hubble-deployment.yaml + set: + global.runtimeVariant: topling + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: hugegraph/hubble:latest + - notExists: + path: spec.template.spec.nodeSelector + + - it: schedules the inherited Helm test image on Linux amd64 + template: tests/test-connection.yaml + set: + global.runtimeVariant: topling + asserts: + - equal: + path: spec.containers[0].image + value: hugegraph/server:latest-topling + - equal: + path: spec.nodeSelector + value: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux + + - it: keeps exact digest references and registry overrides + template: pd-statefulset.yaml + set: + global.runtimeVariant: topling + global.imageRegistry: registry.example.com + pd.image.digest: sha256:43999a5ccda34883a9e4e458e25cb903ce34adbc66782d9f4a5260d68b2b5a82 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: registry.example.com/hugegraph/pd@sha256:43999a5ccda34883a9e4e458e25cb903ce34adbc66782d9f4a5260d68b2b5a82 + + - it: keeps a custom repository and an already suffixed tag + template: store-statefulset.yaml + set: + global.runtimeVariant: topling + global.imageRegistry: ignored.example.com + store.image.repository: custom.example.com/team/store + store.image.tag: custom-topling + store.nodeSelector: + workload: graph + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: custom.example.com/team/store:custom-topling + - equal: + path: spec.template.spec.nodeSelector.workload + value: graph + + - it: keeps standard images and unrestricted architecture by default + template: pd-statefulset.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: hugegraph/pd:latest + - notExists: + path: spec.template.spec.nodeSelector + - equal: + path: metadata.annotations["hugegraph.apache.org/rocksdb-runtime"] + value: standard diff --git a/helm/hugegraph/tests/validate_values_test.yaml b/helm/hugegraph/tests/validate_values_test.yaml index 7f83a4779d..c9db767109 100644 --- a/helm/hugegraph/tests/validate_values_test.yaml +++ b/helm/hugegraph/tests/validate_values_test.yaml @@ -260,3 +260,12 @@ tests: cidr: 10.0.0.0/8 asserts: - notFailedTemplate: {} + + - it: rejects contradictory architecture scheduling + set: + global.runtimeVariant: topling + pd.nodeSelector: + kubernetes.io/arch: arm64 + asserts: + - failedTemplate: + errorPattern: "pd.nodeSelector.kubernetes.io/arch must be amd64" diff --git a/helm/hugegraph/values.schema.json b/helm/hugegraph/values.schema.json index 987e650c16..267f38a558 100644 --- a/helm/hugegraph/values.schema.json +++ b/helm/hugegraph/values.schema.json @@ -34,6 +34,13 @@ "properties": { "imageRegistry": { "type": "string" + }, + "runtimeVariant": { + "type": "string", + "enum": [ + "standard", + "topling" + ] } } }, diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index c81a9ff5c8..2045b84b17 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -23,6 +23,9 @@ mode: easy global: imageRegistry: "" + # standard: normal RocksDB images; topling: matching -topling tags on Linux amd64. + # Select a runtime for a fresh deployment; this does not migrate existing data. + runtimeVariant: standard persistence: enabled: true diff --git a/hugegraph-pd/Dockerfile b/hugegraph-pd/Dockerfile index c7050826ac..ffa32755ba 100644 --- a/hugegraph-pd/Dockerfile +++ b/hugegraph-pd/Dockerfile @@ -17,6 +17,9 @@ # # Dockerfile for HugeGraph PD +# Select the native runtime without changing the source build or deployment layout. +ARG RUNTIME_VARIANT=standard + # 1st stage: build source code FROM --platform=$BUILDPLATFORM maven:3.9.16-eclipse-temurin-17 AS build @@ -29,13 +32,40 @@ ARG MAVEN_PROJECTS="hugegraph-server/hugegraph-dist,hugegraph-pd/hg-pd-dist,huge ARG SOURCE_REVISION=local RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sharing=locked \ + --mount=type=secret,id=proxy_ca,required=false \ + --mount=type=secret,id=maven_settings,required=false \ + set -e; \ + if [ -f /run/secrets/proxy_ca ]; then \ + cp "$JAVA_HOME/lib/security/cacerts" /tmp/hugegraph-build-cacerts; \ + keytool -importcert -noprompt -alias hugegraph-build-proxy \ + -file /run/secrets/proxy_ca -keystore /tmp/hugegraph-build-cacerts -storepass changeit; \ + export MAVEN_OPTS="${MAVEN_OPTS:-} -Djavax.net.ssl.trustStore=/tmp/hugegraph-build-cacerts"; \ + fi; \ + if [ -f /run/secrets/maven_settings ]; then \ + MAVEN_ARGS="-s /run/secrets/maven_settings ${MAVEN_ARGS:-}"; \ + fi; \ mvn install -pl "$MAVEN_PROJECTS" \ - -am $MAVEN_ARGS -e -B -ntp -Dmaven.test.skip=true -Dmaven.javadoc.skip=true \ - && rm ./hugegraph-server/*.tar.gz ./hugegraph-pd/*.tar.gz ./hugegraph-store/*.tar.gz + -am $MAVEN_ARGS -e -B -ntp -Dmaven.test.skip=true -Dmaven.javadoc.skip=true; \ + rm -f /tmp/hugegraph-build-cacerts; \ + rm ./hugegraph-server/*.tar.gz ./hugegraph-pd/*.tar.gz ./hugegraph-store/*.tar.gz # 2nd stage: runtime env # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 -FROM eclipse-temurin:17-jre-jammy +FROM eclipse-temurin:17-jre-jammy AS standard-base + +# The fork's JNI requires glibc 2.38+ and a Java 17 JDK for its capability probe. +FROM eclipse-temurin:17-jdk-noble AS topling-base +ARG TARGETARCH +RUN test "$TARGETARCH" = amd64 || { echo 'Topling images require linux/amd64' >&2; exit 1; } +ENV TOPLINGDB_ROCKSDB_PROVIDER=topling + +FROM ${RUNTIME_VARIANT}-base AS runtime +ARG RUNTIME_VARIANT +ARG SOURCE_REVISION=local +ARG SOURCE_REPOSITORY=https://github.com/apache/hugegraph +LABEL org.opencontainers.image.source="$SOURCE_REPOSITORY" \ + org.opencontainers.image.revision="$SOURCE_REVISION" \ + org.apache.hugegraph.rocksdb-runtime="$RUNTIME_VARIANT" LABEL maintainer="HugeGraph Docker Maintainers " @@ -51,12 +81,19 @@ WORKDIR /hugegraph-pd/ # from hg-pd-dist bin/util.sh. PD runs no port preflight, so unlike the server # images this needs no socket-table tool in its place. ARG RUNTIME_DEPS_EPOCH=1 -RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends --no-install-suggests \ +RUN --mount=type=secret,id=proxy_ca,required=false \ + native_deps="" \ + && if [ "$RUNTIME_VARIANT" = topling ]; then \ + native_deps="libaio1t64 libstdc++6 libsnappy1v5 libbz2-1.0 liblz4-1 libzstd1 zlib1g unzip"; \ + fi \ + && apt_ca=/etc/ssl/certs/ca-certificates.crt \ + && if [ -f /run/secrets/proxy_ca ]; then apt_ca=/run/secrets/proxy_ca; fi \ + && apt-get -q -o Acquire::https::CaInfo="$apt_ca" update \ + && apt-get -q -o Acquire::https::CaInfo="$apt_ca" install -y --no-install-recommends --no-install-suggests \ dumb-init \ procps \ curl \ - vim \ + vim ${native_deps} \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -66,6 +103,19 @@ COPY --from=build /pkg/hugegraph-pd/apache-hugegraph-pd-*/ /hugegraph-pd/ COPY hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh . RUN chmod 755 ./docker-entrypoint.sh +# Download only during an explicit Topling build, verify and prepare with the +# distribution's normal capability probe. No network access is needed at startup. +ARG TOPLING_JNI_URL +ARG TOPLING_JNI_SHA256 +COPY docker/prepare-topling-image.sh docker/topling-runtime.env /tmp/ +RUN --mount=type=secret,id=proxy_ca,required=false \ + --mount=type=secret,id=topling_jni,required=false \ + if [ "$RUNTIME_VARIANT" = topling ]; then \ + if [ -f /run/secrets/proxy_ca ]; then export CURL_CA_BUNDLE=/run/secrets/proxy_ca; fi; \ + bash /tmp/prepare-topling-image.sh; \ + fi \ + && rm /tmp/prepare-topling-image.sh /tmp/topling-runtime.env + EXPOSE 8620 VOLUME /hugegraph-pd diff --git a/hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh b/hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh index f3bf1437e1..19953217f2 100755 --- a/hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh +++ b/hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh @@ -19,6 +19,12 @@ set -euo pipefail log() { echo "[hugegraph-pd-entrypoint] $*"; } +ROCKSDB_PROVIDER="${TOPLINGDB_ROCKSDB_PROVIDER:-rocksdb}" +case "${ROCKSDB_PROVIDER}" in + rocksdb | topling) ;; + *) log "ERROR: TOPLINGDB_ROCKSDB_PROVIDER must be rocksdb or topling"; exit 2 ;; +esac + require_env() { local name="$1" if [[ -z "${!name:-}" ]]; then @@ -125,10 +131,18 @@ AUTH_JSON="\"auth\": { \"secret-key\": \"$(json_escape "${HG_PD_AUTH_SECRET_KEY} # only way to expose another endpoint from this image. MANAGEMENT_JSON="\"management\": { \"endpoints\": { \"web\": { \"exposure\": { \"include\": \"$(json_escape "${HG_PD_ACTUATOR_EXPOSURE}")\" } } } }," +# Preserve mounted business configuration in the ordinary image. An explicitly +# selected runtime (including the Topling image's default) configures both sides. +ROCKSDB_JSON="" +if [[ -n "${TOPLINGDB_ROCKSDB_PROVIDER:-}" ]]; then + ROCKSDB_JSON="\"rocksdb\": { \"provider\": \"${ROCKSDB_PROVIDER}\" }," +fi + SPRING_APPLICATION_JSON="$(cat < 13 -FROM eclipse-temurin:17-jdk-jammy +FROM eclipse-temurin:17-jdk-jammy AS standard-base + +# The fork's JNI requires glibc 2.38+ and a Java 17 JDK for its capability probe. +FROM eclipse-temurin:17-jdk-noble AS topling-base +ARG TARGETARCH +RUN test "$TARGETARCH" = amd64 || { echo 'Topling images require linux/amd64' >&2; exit 1; } +ENV TOPLINGDB_ROCKSDB_PROVIDER=topling + +FROM ${RUNTIME_VARIANT}-base AS runtime +ARG RUNTIME_VARIANT +ARG SOURCE_REVISION=local +ARG SOURCE_REPOSITORY=https://github.com/apache/hugegraph +LABEL org.opencontainers.image.source="$SOURCE_REPOSITORY" \ + org.opencontainers.image.revision="$SOURCE_REVISION" \ + org.apache.hugegraph.rocksdb-runtime="$RUNTIME_VARIANT" # Local Arthas attach requires the JDK attach and compiler modules. LABEL maintainer="HugeGraph Docker Maintainers " @@ -53,13 +83,20 @@ WORKDIR /hugegraph-server/ # permanently inconclusive and a duplicate start is no longer caught. It # replaces lsof, which no shipped script calls any more. ARG RUNTIME_DEPS_EPOCH=1 -RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends --no-install-suggests \ +RUN --mount=type=secret,id=proxy_ca,required=false \ + native_deps="" \ + && if [ "$RUNTIME_VARIANT" = topling ]; then \ + native_deps="libaio1t64 libstdc++6 libsnappy1v5 libbz2-1.0 liblz4-1 libzstd1 zlib1g unzip"; \ + fi \ + && apt_ca=/etc/ssl/certs/ca-certificates.crt \ + && if [ -f /run/secrets/proxy_ca ]; then apt_ca=/run/secrets/proxy_ca; fi \ + && apt-get -q -o Acquire::https::CaInfo="$apt_ca" update \ + && apt-get -q -o Acquire::https::CaInfo="$apt_ca" install -y --no-install-recommends --no-install-suggests \ dumb-init \ procps \ curl \ iproute2 \ - vim \ + vim ${native_deps} \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -72,6 +109,19 @@ COPY hugegraph-server/hugegraph-dist/docker/scripts/detect-storage.groovy ./scri COPY hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh . RUN chmod 755 ./docker-entrypoint.sh +# Download only during an explicit Topling build, verify and prepare with the +# distribution's normal capability probe. No network access is needed at startup. +ARG TOPLING_JNI_URL +ARG TOPLING_JNI_SHA256 +COPY docker/prepare-topling-image.sh docker/topling-runtime.env /tmp/ +RUN --mount=type=secret,id=proxy_ca,required=false \ + --mount=type=secret,id=topling_jni,required=false \ + if [ "$RUNTIME_VARIANT" = topling ]; then \ + if [ -f /run/secrets/proxy_ca ]; then export CURL_CA_BUNDLE=/run/secrets/proxy_ca; fi; \ + bash /tmp/prepare-topling-image.sh; \ + fi \ + && rm /tmp/prepare-topling-image.sh /tmp/topling-runtime.env + EXPOSE 8080 VOLUME /hugegraph-server diff --git a/hugegraph-server/Dockerfile-hstore b/hugegraph-server/Dockerfile-hstore index 351b116d6b..5ef3b210eb 100644 --- a/hugegraph-server/Dockerfile-hstore +++ b/hugegraph-server/Dockerfile-hstore @@ -29,15 +29,34 @@ ARG MAVEN_PROJECTS="hugegraph-server/hugegraph-dist,hugegraph-pd/hg-pd-dist,huge ARG SOURCE_REVISION=local RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sharing=locked \ + --mount=type=secret,id=proxy_ca,required=false \ + --mount=type=secret,id=maven_settings,required=false \ + set -e; \ + if [ -f /run/secrets/proxy_ca ]; then \ + cp "$JAVA_HOME/lib/security/cacerts" /tmp/hugegraph-build-cacerts; \ + keytool -importcert -noprompt -alias hugegraph-build-proxy \ + -file /run/secrets/proxy_ca -keystore /tmp/hugegraph-build-cacerts -storepass changeit; \ + export MAVEN_OPTS="${MAVEN_OPTS:-} -Djavax.net.ssl.trustStore=/tmp/hugegraph-build-cacerts"; \ + fi; \ + if [ -f /run/secrets/maven_settings ]; then \ + MAVEN_ARGS="-s /run/secrets/maven_settings ${MAVEN_ARGS:-}"; \ + fi; \ mvn install -pl "$MAVEN_PROJECTS" \ - -am $MAVEN_ARGS -e -B -ntp -Dmaven.test.skip=true -Dmaven.javadoc.skip=true \ - && rm ./hugegraph-server/*.tar.gz ./hugegraph-pd/*.tar.gz ./hugegraph-store/*.tar.gz + -am $MAVEN_ARGS -e -B -ntp -Dmaven.test.skip=true -Dmaven.javadoc.skip=true; \ + rm -f /tmp/hugegraph-build-cacerts; \ + rm ./hugegraph-server/*.tar.gz ./hugegraph-pd/*.tar.gz ./hugegraph-store/*.tar.gz # 2nd stage: runtime env # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 FROM eclipse-temurin:17-jdk-jammy # Local Arthas attach requires the JDK attach and compiler modules. +ARG SOURCE_REVISION=local +ARG SOURCE_REPOSITORY=https://github.com/apache/hugegraph +LABEL org.opencontainers.image.source="$SOURCE_REPOSITORY" \ + org.opencontainers.image.revision="$SOURCE_REVISION" \ + org.apache.hugegraph.rocksdb-runtime="hstore" + LABEL maintainer="HugeGraph Docker Maintainers " # TODO: use g1gc or zgc as default @@ -53,8 +72,13 @@ WORKDIR /hugegraph-server/ # permanently inconclusive and a duplicate start is no longer caught. It # replaces lsof, which no shipped script calls any more. ARG RUNTIME_DEPS_EPOCH=1 -RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends --no-install-suggests \ +RUN --mount=type=secret,id=proxy_ca,required=false \ + --mount=type=secret,id=maven_settings,required=false \ + set -e; \ + apt_ca=/etc/ssl/certs/ca-certificates.crt; \ + if [ -f /run/secrets/proxy_ca ]; then apt_ca=/run/secrets/proxy_ca; fi; \ + apt-get -q -o Acquire::https::CaInfo="$apt_ca" update \ + && apt-get -q -o Acquire::https::CaInfo="$apt_ca" install -y --no-install-recommends --no-install-suggests \ dumb-init \ procps \ curl \ diff --git a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh index fa4f9bcfbd..acaf8bb046 100755 --- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh +++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh @@ -561,5 +561,33 @@ for invalid_k8s_flag in 1 0 enabled "t rue" "tr ue"; do "${TEST_HOME}/conf/graphs/hugegraph.properties" done +# A Topling image configures every local graph while retaining remote graphs. +printf 'backend=rocksdb\n' > "${TEST_HOME}/conf/graphs/analytics.properties" +printf 'backend=hstore\nrocksdb.provider=rocksdb\n' > "${TEST_HOME}/conf/graphs/remote.properties" +( + cd "${TEST_HOME}" + HG_SERVER_BACKEND=rocksdb TOPLINGDB_ROCKSDB_PROVIDER=topling bash ./docker-entrypoint.sh +) +for graph in hugegraph analytics; do + grep -qx 'rocksdb.provider=topling' "${TEST_HOME}/conf/graphs/${graph}.properties" +done +grep -qx 'rocksdb.provider=rocksdb' "${TEST_HOME}/conf/graphs/remote.properties" +# Default RocksDB startup retains mounted settings so business/launcher mismatch +# checks can still reject an incorrectly selected runtime instead of hiding it. +( + cd "${TEST_HOME}" + env -u TOPLINGDB_ROCKSDB_PROVIDER bash ./docker-entrypoint.sh +) +grep -qx 'rocksdb.provider=topling' "${TEST_HOME}/conf/graphs/analytics.properties" +cp "${TEST_HOME}/conf/graphs/analytics.properties" "${TEST_HOME}/analytics.before-invalid-provider" +if ( + cd "${TEST_HOME}" + TOPLINGDB_ROCKSDB_PROVIDER=invalid bash ./docker-entrypoint.sh +); then + echo "invalid TOPLINGDB_ROCKSDB_PROVIDER unexpectedly succeeded" >&2 + exit 1 +fi +cmp "${TEST_HOME}/analytics.before-invalid-provider" "${TEST_HOME}/conf/graphs/analytics.properties" + echo "PASS: Docker entrypoint configures HStore discovery and authentication" bash "${SCRIPT_DIR}/test/test-docker-entrypoint-process.sh" diff --git a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh index 4835f45e5e..eec6f53be6 100755 --- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh +++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh @@ -26,6 +26,12 @@ mkdir -p "${DOCKER_FOLDER}" log() { echo "[hugegraph-server-entrypoint] $*"; } +ROCKSDB_PROVIDER="${TOPLINGDB_ROCKSDB_PROVIDER:-rocksdb}" +case "${ROCKSDB_PROVIDER}" in + rocksdb | topling) ;; + *) log "ERROR: TOPLINGDB_ROCKSDB_PROVIDER must be rocksdb or topling"; exit 2 ;; +esac + source ./bin/util.sh record_jvm_startup_diagnostics run_startup_step() { @@ -228,6 +234,17 @@ esac # filesystem also needs the conf files on a writable path, since set_prop in this script edits them # in place; the chart refuses readOnlyRootFilesystem until then. [[ -n "${HG_SERVER_BACKEND:-}" ]] && set_prop "backend" "${HG_SERVER_BACKEND}" "${GRAPH_CONF}" +# Apply the image's selected JNI provider to every embedded graph in the normal +# container graphs directory, including mounted additional graph definitions. +# Distributed graphs keep backend=hstore and do not select a local provider. +if [[ -n "${TOPLINGDB_ROCKSDB_PROVIDER:-}" ]]; then + for graph_conf in ./conf/graphs/*.properties; do + [[ -f "${graph_conf}" ]] || continue + if [[ "$(get_prop_encoded backend "${graph_conf}")" == rocksdb ]]; then + set_prop "rocksdb.provider" "${ROCKSDB_PROVIDER}" "${graph_conf}" + fi + done +fi [[ -n "${HG_SERVER_PD_PEERS:-}" ]] && set_prop "pd.peers" "${HG_SERVER_PD_PEERS}" "${GRAPH_CONF}" [[ -n "${HG_SERVER_USE_PD:-}" ]] && \ set_prop "usePD" "${HG_SERVER_USE_PD}" "${REST_SERVER_CONF}" diff --git a/hugegraph-store/Dockerfile b/hugegraph-store/Dockerfile index 9e6fbdf183..61167679e6 100644 --- a/hugegraph-store/Dockerfile +++ b/hugegraph-store/Dockerfile @@ -17,6 +17,9 @@ # # Dockerfile for HugeGraph Store +# Select the native runtime without changing the source build or deployment layout. +ARG RUNTIME_VARIANT=standard + # 1st stage: build source code FROM --platform=$BUILDPLATFORM maven:3.9.16-eclipse-temurin-17 AS build @@ -29,13 +32,40 @@ ARG MAVEN_PROJECTS="hugegraph-server/hugegraph-dist,hugegraph-pd/hg-pd-dist,huge ARG SOURCE_REVISION=local RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sharing=locked \ + --mount=type=secret,id=proxy_ca,required=false \ + --mount=type=secret,id=maven_settings,required=false \ + set -e; \ + if [ -f /run/secrets/proxy_ca ]; then \ + cp "$JAVA_HOME/lib/security/cacerts" /tmp/hugegraph-build-cacerts; \ + keytool -importcert -noprompt -alias hugegraph-build-proxy \ + -file /run/secrets/proxy_ca -keystore /tmp/hugegraph-build-cacerts -storepass changeit; \ + export MAVEN_OPTS="${MAVEN_OPTS:-} -Djavax.net.ssl.trustStore=/tmp/hugegraph-build-cacerts"; \ + fi; \ + if [ -f /run/secrets/maven_settings ]; then \ + MAVEN_ARGS="-s /run/secrets/maven_settings ${MAVEN_ARGS:-}"; \ + fi; \ mvn install -pl "$MAVEN_PROJECTS" \ - -am $MAVEN_ARGS -e -B -ntp -Dmaven.test.skip=true -Dmaven.javadoc.skip=true \ - && rm ./hugegraph-server/*.tar.gz ./hugegraph-pd/*.tar.gz ./hugegraph-store/*.tar.gz + -am $MAVEN_ARGS -e -B -ntp -Dmaven.test.skip=true -Dmaven.javadoc.skip=true; \ + rm -f /tmp/hugegraph-build-cacerts; \ + rm ./hugegraph-server/*.tar.gz ./hugegraph-pd/*.tar.gz ./hugegraph-store/*.tar.gz # 2nd stage: runtime env # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 -FROM eclipse-temurin:17-jdk-jammy +FROM eclipse-temurin:17-jdk-jammy AS standard-base + +# The fork's JNI requires glibc 2.38+ and a Java 17 JDK for its capability probe. +FROM eclipse-temurin:17-jdk-noble AS topling-base +ARG TARGETARCH +RUN test "$TARGETARCH" = amd64 || { echo 'Topling images require linux/amd64' >&2; exit 1; } +ENV TOPLINGDB_ROCKSDB_PROVIDER=topling + +FROM ${RUNTIME_VARIANT}-base AS runtime +ARG RUNTIME_VARIANT +ARG SOURCE_REVISION=local +ARG SOURCE_REPOSITORY=https://github.com/apache/hugegraph +LABEL org.opencontainers.image.source="$SOURCE_REPOSITORY" \ + org.opencontainers.image.revision="$SOURCE_REVISION" \ + org.apache.hugegraph.rocksdb-runtime="$RUNTIME_VARIANT" # Local Arthas attach requires the JDK attach and compiler modules. LABEL maintainer="HugeGraph Docker Maintainers " @@ -52,12 +82,19 @@ WORKDIR /hugegraph-store/ # from hg-store-dist bin/util.sh. Store runs no port preflight, so unlike the # server images this needs no socket-table tool in its place. ARG RUNTIME_DEPS_EPOCH=1 -RUN apt-get -q update \ - && apt-get -q install -y --no-install-recommends --no-install-suggests \ +RUN --mount=type=secret,id=proxy_ca,required=false \ + native_deps="" \ + && if [ "$RUNTIME_VARIANT" = topling ]; then \ + native_deps="libaio1t64 libstdc++6 libsnappy1v5 libbz2-1.0 liblz4-1 libzstd1 zlib1g unzip"; \ + fi \ + && apt_ca=/etc/ssl/certs/ca-certificates.crt \ + && if [ -f /run/secrets/proxy_ca ]; then apt_ca=/run/secrets/proxy_ca; fi \ + && apt-get -q -o Acquire::https::CaInfo="$apt_ca" update \ + && apt-get -q -o Acquire::https::CaInfo="$apt_ca" install -y --no-install-recommends --no-install-suggests \ dumb-init \ procps \ curl \ - vim \ + vim ${native_deps} \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -67,6 +104,19 @@ COPY --from=build /pkg/hugegraph-store/apache-hugegraph-store-*/ /hugegraph-stor COPY hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh . RUN chmod 755 ./docker-entrypoint.sh +# Download only during an explicit Topling build, verify and prepare with the +# distribution's normal capability probe. No network access is needed at startup. +ARG TOPLING_JNI_URL +ARG TOPLING_JNI_SHA256 +COPY docker/prepare-topling-image.sh docker/topling-runtime.env /tmp/ +RUN --mount=type=secret,id=proxy_ca,required=false \ + --mount=type=secret,id=topling_jni,required=false \ + if [ "$RUNTIME_VARIANT" = topling ]; then \ + if [ -f /run/secrets/proxy_ca ]; then export CURL_CA_BUNDLE=/run/secrets/proxy_ca; fi; \ + bash /tmp/prepare-topling-image.sh; \ + fi \ + && rm /tmp/prepare-topling-image.sh /tmp/topling-runtime.env + EXPOSE 8520 VOLUME /hugegraph-store diff --git a/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh b/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh index 8ea9022a33..c841c29768 100755 --- a/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh +++ b/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh @@ -19,6 +19,12 @@ set -euo pipefail log() { echo "[hugegraph-store-entrypoint] $*"; } +ROCKSDB_PROVIDER="${TOPLINGDB_ROCKSDB_PROVIDER:-rocksdb}" +case "${ROCKSDB_PROVIDER}" in + rocksdb | topling) ;; + *) log "ERROR: TOPLINGDB_ROCKSDB_PROVIDER must be rocksdb or topling"; exit 2 ;; +esac + require_env() { local name="$1" if [[ -z "${!name:-}" ]]; then @@ -55,9 +61,17 @@ require_env "HG_STORE_RAFT_ADDRESS" : "${HG_STORE_REST_PORT:=8520}" : "${HG_STORE_DATA_PATH:=/hugegraph-store/storage}" +# Preserve mounted business configuration in the ordinary image. An explicitly +# selected runtime (including the Topling image's default) configures both sides. +ROCKSDB_JSON="" +if [[ -n "${TOPLINGDB_ROCKSDB_PROVIDER:-}" ]]; then + ROCKSDB_JSON="\"rocksdb\": { \"provider\": \"${ROCKSDB_PROVIDER}\" }," +fi + # ── Build SPRING_APPLICATION_JSON ───────────────────────────────────── SPRING_APPLICATION_JSON="$(cat < Date: Fri, 23 Oct 2026 14:23:31 +0000 Subject: [PATCH 02/29] fix(docker): default Topling images to public community components --- docker/README.md | 8 + docker/prepare-topling-image.sh | 4 + docker/test-topling-images.sh | 47 ++ docker/toplingdb.yaml | 53 ++ hugegraph-pd/Dockerfile | 4 +- hugegraph-server/Dockerfile | 4 +- hugegraph-store/Dockerfile | 4 +- install-dist/release-docs/LICENSE | 20 + install-dist/release-docs/NOTICE | 19 + .../licenses/LICENSE-toplingdb-jni-8.10.2.txt | 586 ++++++++++++++++++ .../optional-image-dependencies.txt | 17 + 11 files changed, 760 insertions(+), 6 deletions(-) create mode 100644 docker/toplingdb.yaml create mode 100644 install-dist/release-docs/licenses/LICENSE-toplingdb-jni-8.10.2.txt create mode 100644 install-dist/scripts/dependency/optional-image-dependencies.txt diff --git a/docker/README.md b/docker/README.md index d04b11775b..5c8d4083e4 100644 --- a/docker/README.md +++ b/docker/README.md @@ -298,6 +298,14 @@ independent image tag. The images include the verified JNI runtime and set both launcher and business provider configuration. No JAR download or `LD_PRELOAD` setting is needed at startup. +The default image uses the community JNI with SkipList memtables and BlockBasedTable +SSTs; it includes no proprietary trial modules. Its native profile starts with a +64 MiB block cache and three 16 MiB write buffers per column family. Native memory +still scales with databases and column families; these settings do not cap total +process memory. Mount a tuned `conf/toplingdb.yaml` or select an external file with +`TOPLINGDB_EASY_MIGRATE_CONF` when sizing a production deployment. CSPP and other +optional modules require your own compatible JNI artifact and native profile. +The unpacked distribution's full profile is unchanged. Existing data is not converted by selecting another image: use new volumes for ToplingDB, and migrate through the graph APIs after validating your workload. Returning to RocksDB uses its original deployment and volumes. See diff --git a/docker/prepare-topling-image.sh b/docker/prepare-topling-image.sh index dde641ff21..4a02fb6ce7 100644 --- a/docker/prepare-topling-image.sh +++ b/docker/prepare-topling-image.sh @@ -50,3 +50,7 @@ fi # The shared preparer verifies the hash, ELF/dependencies, and real EasyMigrate # behavior before installing a component-local runtime and checksum receipt. bash bin/prepare-topling.sh + +# Community images need no proprietary factory registration. Keep the full +# distribution profile unchanged and install the image's smaller native profile. +cp "$(dirname "${BASH_SOURCE[0]}")/toplingdb.yaml" conf/toplingdb.yaml diff --git a/docker/test-topling-images.sh b/docker/test-topling-images.sh index 57dd85c8e9..f7178534c5 100644 --- a/docker/test-topling-images.sh +++ b/docker/test-topling-images.sh @@ -22,6 +22,7 @@ import json import os from pathlib import Path import shutil +import re import subprocess import sys import tempfile @@ -81,4 +82,50 @@ with tempfile.TemporaryDirectory(prefix="topling-entrypoints-") as temporary: assert result.returncode != 0, "unverified artifact accepted" assert "Error: Topling builds require" in result.stderr, result.stderr print("PASS: image preparation rejects missing/untrusted artifact inputs") + + # Exercise the real image helper through preparation and profile installation, + # with only download/native preparation stubbed (native JNI is tested separately). + distribution = Path(temporary) / "distribution" + (distribution / "bin").mkdir(parents=True) + (distribution / "conf").mkdir() + (distribution / "tools").mkdir() + original_profile = repo / "hugegraph-server/hugegraph-dist/src/assembly/static/conf/toplingdb.yaml" + original_bytes = original_profile.read_bytes() + shutil.copy(original_profile, distribution / "conf/toplingdb.yaml") + preparer = distribution / "bin/prepare-topling.sh" + preparer.write_text("#!/bin/bash\ntest -r \"$TOPLING_JNI_JAR\" && test \"${#TOPLING_JNI_SHA256}\" -eq 64\n") + downloader = distribution / "tools/curl" + downloader.write_text("#!/bin/bash\nwhile (( $# )); do if [[ $1 == --output ]]; then printf stub > \"$2\"; exit 0; fi; shift; done\nexit 1\n") + downloader.chmod(0o755) + environment.update(TOPLING_JNI_SHA256="a" * 64, TOPLING_JNI_URL="https://example.invalid/immutable.jar") + environment["PATH"] = str(distribution / "tools") + os.pathsep + environment["PATH"] + result = subprocess.run(["bash", str(helper)], cwd=distribution, env=environment, + capture_output=True, text=True) + assert result.returncode == 0, result.stderr + community_profile = repo / "docker/toplingdb.yaml" + assert (distribution / "conf/toplingdb.yaml").read_bytes() == community_profile.read_bytes() + assert original_profile.read_bytes() == original_bytes, "distribution source profile changed" + profile = community_profile.read_text() + # This profile contains only indentation-based mappings. Reject duplicate + # paths instead of silently letting the YAML reader discard an earlier key. + seen = set() + parents = [] + for line in profile.splitlines(): + match = re.match(r"^( *)([A-Za-z_][\w]*):", line) + if match is None: + continue + indent, key = len(match[1]), match[2] + while parents and parents[-1][0] >= indent: + parents.pop() + path = tuple(parent[1] for parent in parents) + (key,) + assert path not in seen, "duplicate YAML key: " + ".".join(path) + seen.add(path) + parents.append((indent, key)) + classes = set(re.findall(r"^\s+class:\s*(\S+)$", profile, re.MULTILINE)) + assert classes == {"LRUCache", "SkipList", "BlockBasedTable"}, classes + assert "CSPP" not in profile and "ToplingZip" not in profile + for setting in ("capacity: 64M", "write_buffer_size: 16M", "max_write_buffer_number: 3", + "memtable_as_log_index: false", "auto_start_http: false"): + assert setting in profile, setting + print("PASS: community image installs small native profile and preserves full distribution profile") PY diff --git a/docker/toplingdb.yaml b/docker/toplingdb.yaml new file mode 100644 index 0000000000..5c0ee4226e --- /dev/null +++ b/docker/toplingdb.yaml @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Community image defaults. Memory grows with databases and column families; +# these are individual native budgets, not a process memory limit. +http: + auto_start_http: false +Cache: + lru_cache: + class: LRUCache + params: + capacity: 64M +MemTableRepFactory: + skiplist: + class: SkipList + params: + lookahead: 0 +TableFactory: + block_based: + class: BlockBasedTable + params: + block_cache: "${lru_cache}" +CFOptions: + default: + memtable_factory: "${skiplist}" + table_factory: block_based + write_buffer_size: 16M + max_write_buffer_number: 3 + merge_operator: uint64add +DBOptions: + log: + create_if_missing: true + create_missing_column_families: true + default: + create_if_missing: true + create_missing_column_families: false + max_background_jobs: 2 + # Java WriteBatch does not carry Topling's mmap WAL references. + memtable_as_log_index: false diff --git a/hugegraph-pd/Dockerfile b/hugegraph-pd/Dockerfile index ffa32755ba..f6c4f89e6e 100644 --- a/hugegraph-pd/Dockerfile +++ b/hugegraph-pd/Dockerfile @@ -107,14 +107,14 @@ RUN chmod 755 ./docker-entrypoint.sh # distribution's normal capability probe. No network access is needed at startup. ARG TOPLING_JNI_URL ARG TOPLING_JNI_SHA256 -COPY docker/prepare-topling-image.sh docker/topling-runtime.env /tmp/ +COPY docker/prepare-topling-image.sh docker/topling-runtime.env docker/toplingdb.yaml /tmp/ RUN --mount=type=secret,id=proxy_ca,required=false \ --mount=type=secret,id=topling_jni,required=false \ if [ "$RUNTIME_VARIANT" = topling ]; then \ if [ -f /run/secrets/proxy_ca ]; then export CURL_CA_BUNDLE=/run/secrets/proxy_ca; fi; \ bash /tmp/prepare-topling-image.sh; \ fi \ - && rm /tmp/prepare-topling-image.sh /tmp/topling-runtime.env + && rm /tmp/prepare-topling-image.sh /tmp/topling-runtime.env /tmp/toplingdb.yaml EXPOSE 8620 VOLUME /hugegraph-pd diff --git a/hugegraph-server/Dockerfile b/hugegraph-server/Dockerfile index 8a22a92e2b..d33e6660fc 100644 --- a/hugegraph-server/Dockerfile +++ b/hugegraph-server/Dockerfile @@ -113,14 +113,14 @@ RUN chmod 755 ./docker-entrypoint.sh # distribution's normal capability probe. No network access is needed at startup. ARG TOPLING_JNI_URL ARG TOPLING_JNI_SHA256 -COPY docker/prepare-topling-image.sh docker/topling-runtime.env /tmp/ +COPY docker/prepare-topling-image.sh docker/topling-runtime.env docker/toplingdb.yaml /tmp/ RUN --mount=type=secret,id=proxy_ca,required=false \ --mount=type=secret,id=topling_jni,required=false \ if [ "$RUNTIME_VARIANT" = topling ]; then \ if [ -f /run/secrets/proxy_ca ]; then export CURL_CA_BUNDLE=/run/secrets/proxy_ca; fi; \ bash /tmp/prepare-topling-image.sh; \ fi \ - && rm /tmp/prepare-topling-image.sh /tmp/topling-runtime.env + && rm /tmp/prepare-topling-image.sh /tmp/topling-runtime.env /tmp/toplingdb.yaml EXPOSE 8080 VOLUME /hugegraph-server diff --git a/hugegraph-store/Dockerfile b/hugegraph-store/Dockerfile index 61167679e6..33b018225d 100644 --- a/hugegraph-store/Dockerfile +++ b/hugegraph-store/Dockerfile @@ -108,14 +108,14 @@ RUN chmod 755 ./docker-entrypoint.sh # distribution's normal capability probe. No network access is needed at startup. ARG TOPLING_JNI_URL ARG TOPLING_JNI_SHA256 -COPY docker/prepare-topling-image.sh docker/topling-runtime.env /tmp/ +COPY docker/prepare-topling-image.sh docker/topling-runtime.env docker/toplingdb.yaml /tmp/ RUN --mount=type=secret,id=proxy_ca,required=false \ --mount=type=secret,id=topling_jni,required=false \ if [ "$RUNTIME_VARIANT" = topling ]; then \ if [ -f /run/secrets/proxy_ca ]; then export CURL_CA_BUNDLE=/run/secrets/proxy_ca; fi; \ bash /tmp/prepare-topling-image.sh; \ fi \ - && rm /tmp/prepare-topling-image.sh /tmp/topling-runtime.env + && rm /tmp/prepare-topling-image.sh /tmp/topling-runtime.env /tmp/toplingdb.yaml EXPOSE 8520 VOLUME /hugegraph-store diff --git a/install-dist/release-docs/LICENSE b/install-dist/release-docs/LICENSE index 6e27cb1e2e..c92dea893d 100644 --- a/install-dist/release-docs/LICENSE +++ b/install-dist/release-docs/LICENSE @@ -907,3 +907,23 @@ The text of each license is also included in licenses/LICENSE-[project].txt. # See licenses/LICENSE-jansi-2.4.0.txt for license details (Apache License 2.0 or LGPL 3.0+). Ivy 2.6.0 includes third-party BSD and MIT licensed code; see licenses/LICENSE-ivy-2.6.0.txt. + +======================================================================== +Optional ToplingDB JNI runtime for container images +======================================================================== +The explicit Topling container variant may additionally include the community +JNI runtime from https://github.com/hugegraph/toplingdb. It is installed as +topling/rocksdbjni.jar, outside the standard runtime lib directory. Standard +RocksDB distributions and images do not include this optional artifact. + +The RocksDB core uses its Apache 2.0 licensing option. Public Topling components +and their bundled dependencies retain their Apache 2.0, BSD, MIT and Boost +licenses. See licenses/LICENSE-toplingdb-jni-8.10.2.txt and the original license +texts under META-INF/licenses/ in the JNI JAR. Its embedded +META-INF/topling-build-info.txt identifies the source revisions and enabled +components. The community profile disables the private ToplingZipTable builder, +CSPP MemTable and distributed compaction components. + +The exact artifact URL and checksum are recorded in docker/topling-runtime.env +in the source repository. The optional image dependency inventory is +install-dist/scripts/dependency/optional-image-dependencies.txt. diff --git a/install-dist/release-docs/NOTICE b/install-dist/release-docs/NOTICE index 2f6fa03e5e..31abcef127 100644 --- a/install-dist/release-docs/NOTICE +++ b/install-dist/release-docs/NOTICE @@ -3328,3 +3328,22 @@ Arthas 4.1.5 embedded Logback 1.2.13 NOTICE Logback: the reliable, generic, fast and flexible logging framework. Copyright (C) 1999-2015, QOS.ch. All rights reserved. + +======================================================================== + +ToplingDB JNI community runtime (optional container images) + +======================================================================== + +Topling container images additionally include software developed by Topling, +Inc., with code from RocksDB, LevelDB and the bundled public dependencies. + +Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +Copyright (c) 2011 The LevelDB Authors. All rights reserved. +Copyright (c) 2021, Topling Inc. All rights reserved. +Copyright (c) 2020, Bytedance Inc. All rights reserved. + +The original component copyright notices and license texts are retained in +META-INF/licenses/ inside topling/rocksdbjni.jar. Source revisions and the +selected community components are recorded in META-INF/topling-build-info.txt. +See licenses/LICENSE-toplingdb-jni-8.10.2.txt for the licensing summary and texts. diff --git a/install-dist/release-docs/licenses/LICENSE-toplingdb-jni-8.10.2.txt b/install-dist/release-docs/licenses/LICENSE-toplingdb-jni-8.10.2.txt new file mode 100644 index 0000000000..98af2a2844 --- /dev/null +++ b/install-dist/release-docs/licenses/LICENSE-toplingdb-jni-8.10.2.txt @@ -0,0 +1,586 @@ +Optional ToplingDB JNI 8.10.2 community runtime +============================================= + +Source: https://github.com/hugegraph/toplingdb + +This component is included only in explicitly selected Topling container images. +The downloaded rocksdbjni-8.10.2-linux64.jar is prepared as +/topling/rocksdbjni.jar, outside the standard runtime lib directory. +The standard RocksDB images and source distributions do not bundle this JNI. + +The community build disables WITH_TOPLING_ROCKS, WITH_CSPP_MEMTABLE and +WITH_TOPLING_DCOMPACT. Its public SidePlugin components include rockside, +cspp-wbwi, topling-sst and topling-zip_table_reader under Apache License 2.0, +and topling-zip under its BSD license. The RocksDB core's Apache License 2.0 +option is selected from its upstream GPLv2 / Apache 2.0 dual license. + +The JNI JAR retains the original source license files and copyright notices +under META-INF/licenses/, with checksums in META-INF/licenses/manifest.sha256. +META-INF/topling-build-info.txt records the exact source/plugin revisions and +build configuration. These per-build records accompany the immutable artifact; +see docker/topling-runtime.env in the source repository for its URL and SHA-256. +The optional artifact and installed aliases are recorded separately in +install-dist/scripts/dependency/optional-image-dependencies.txt. + +The following texts preserve the principal upstream licenses and embedded +third-party notices. The complete per-build collection remains in the JNI JAR. + +======================================================================== +RocksDB core and Apache-licensed public SidePlugin components +Source file: LICENSE.Apache +======================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +======================================================================== +LevelDB-derived code +Source file: LICENSE.leveldb +======================================================================== + +This contains code that is from LevelDB, and that code is under the following license: + +Copyright (c) 2011 The LevelDB Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +======================================================================== +Topling-zip +Source file: sideplugin/topling-zip/LICENSE +======================================================================== + +Copyright (c) 2021, Topling Inc. +All rights reserved. + +Copyright (c) 2020, Bytedance Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +======================================================================== +rapidyaml and c4core +Source file: sideplugin/rockside/3rdparty/rapidyaml/LICENSE.txt +======================================================================== + +Copyright (c) 2018, Joao Paulo Magalhaes + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +======================================================================== +CivetWeb +Source file: sideplugin/rockside/src/topling/web/civetweb.c +======================================================================== + +/* Copyright (c) 2013-2021 the Civetweb developers + * Copyright (c) 2004-2013 Sergey Lyubka + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +======================================================================== +JSON for Modern C++ +Source file: sideplugin/topling-zip/src/terark/json.hpp +======================================================================== + +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ +| | |__ | | | | | | version 3.8.0 +|_____|_____|_____|_|___| https://github.com/topling/json + +Licensed under the MIT License . +SPDX-License-Identifier: MIT +Copyright (c) 2013-2019 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +======================================================================== +libbase64 +Source file: sideplugin/topling-zip/3rdparty/base64/LICENSE +======================================================================== + +Copyright (c) 2005-2007, Nick Galbreath +Copyright (c) 2013-2017, Alfred Klomp +Copyright (c) 2015-2017, Wojciech Mula +Copyright (c) 2016-2017, Matthieu Darbois +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +- Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +======================================================================== +libdivsufsort-lite (vendored by Zstandard) +Source file: sideplugin/topling-zip/3rdparty/zstd/zstd/dictBuilder/divsufsort.c +======================================================================== + +/* + * divsufsort.c for libdivsufsort-lite + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +======================================================================== +debugbreak (c4core) +Source file: sideplugin/rockside/3rdparty/rapidyaml/ext/c4core/src/c4/ext/debugbreak/COPYING +======================================================================== + +Copyright (c) 2011-2016, Scott Tsai + +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +======================================================================== +Boost +Source file: sideplugin/topling-zip/boost-include/LICENSE_1_0.txt +======================================================================== + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +======================================================================== +Zstandard 1.4.5 (BSD licensing option) +Source file: .github/licenses/zstd-1.4.5-LICENSE +======================================================================== + +BSD License + +For Zstandard software + +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +======================================================================== +xxHash (vendored by legacy Zstandard) +Source file: sideplugin/topling-zip/3rdparty/zstd/zstd/legacy/zstd_v01.c +======================================================================== + +/* + zstd - standard compression library + Copyright (C) 2014-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - zstd source repository : https://github.com/Cyan4973/zstd + - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c +*/ diff --git a/install-dist/scripts/dependency/optional-image-dependencies.txt b/install-dist/scripts/dependency/optional-image-dependencies.txt new file mode 100644 index 0000000000..ccba3ece81 --- /dev/null +++ b/install-dist/scripts/dependency/optional-image-dependencies.txt @@ -0,0 +1,17 @@ +# Optional image-only dependency; excluded from the standard Maven/lib inventory. +# The API version does not identify immutable native contents: use the URL and +# SHA-256 pin in docker/topling-runtime.env and the embedded source provenance. + +runtime=topling-community +version=8.10.2 +source_repository=https://github.com/hugegraph/toplingdb +artifact_filename=rocksdbjni-8.10.2-linux64.jar +artifact_pin=docker/topling-runtime.env:TOPLING_JNI_URL,TOPLING_JNI_SHA256 +installed_server=/hugegraph-server/topling/rocksdbjni.jar +installed_pd=/hugegraph-pd/topling/rocksdbjni.jar +installed_store=/hugegraph-store/topling/rocksdbjni.jar +provenance=META-INF/topling-build-info.txt +licenses=META-INF/licenses/ +license_checksums=META-INF/licenses/manifest.sha256 +build_flags=WITH_TOPLING_ROCKS=0 WITH_CSPP_MEMTABLE=0 WITH_TOPLING_DCOMPACT=0 +license_summary=install-dist/release-docs/licenses/LICENSE-toplingdb-jni-8.10.2.txt From 446ec33219ac51ab574b9ee5a48d89f3cf0c20b4 Mon Sep 17 00:00:00 2001 From: orion Date: Fri, 23 Oct 2026 14:25:57 +0000 Subject: [PATCH 03/29] fix(docker): mount local JNI with a named build context --- docker/README.md | 10 ++++++---- docker/prepare-topling-image.sh | 6 +++--- hugegraph-pd/Dockerfile | 6 +++++- hugegraph-server/Dockerfile | 6 +++++- hugegraph-store/Dockerfile | 6 +++++- 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/docker/README.md b/docker/README.md index 5c8d4083e4..3a8ac2c44d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -328,18 +328,20 @@ project volumes separate from a RocksDB source deployment. For direct Docker and Bake builds, use `--build-arg RUNTIME_VARIANT=topling` or `RUNTIME_VARIANT=topling` respectively, together with the same JNI URL and checksum build inputs. -For a verified local JNI JAR, supply a BuildKit secret instead of a download URL: +For a verified local JNI JAR, put it in a directory as `rocksdbjni.jar` and supply +that directory as a named build context instead of a download URL: ```bash docker buildx build --platform linux/amd64 -f ../hugegraph-server/Dockerfile \ --build-arg RUNTIME_VARIANT=topling \ --build-arg TOPLING_JNI_SHA256='' \ - --secret id=topling_jni,src=/absolute/path/rocksdbjni.jar \ + --build-context topling-jni=/absolute/path/to/jni-directory \ -t hugegraph/hugegraph:local-topling --load .. ``` -Run this example from the `docker` directory. The checksum and normal native -capability probe still apply. Networks with a private CA can additionally supply +Run this example from the `docker` directory. Bake accepts the same directory +with `--set '*.contexts.topling-jni=/absolute/path/to/jni-directory'`. The checksum +and normal native capability probe still apply. Networks with a private CA can additionally supply `--secret id=proxy_ca,src=/absolute/path/ca.pem`; the CA is mounted only while Maven, package installation and artifact download run. diff --git a/docker/prepare-topling-image.sh b/docker/prepare-topling-image.sh index 4a02fb6ce7..ff2289e6a2 100644 --- a/docker/prepare-topling-image.sh +++ b/docker/prepare-topling-image.sh @@ -34,11 +34,11 @@ fi } export TOPLING_JNI_SHA256 -if [[ -r /run/secrets/topling_jni ]]; then - export TOPLING_JNI_JAR=/run/secrets/topling_jni +if [[ -r /run/topling-jni/rocksdbjni.jar ]]; then + export TOPLING_JNI_JAR=/run/topling-jni/rocksdbjni.jar else [[ "${TOPLING_JNI_URL:-}" == https://* ]] || { - echo 'Error: Topling builds require an HTTPS TOPLING_JNI_URL or a topling_jni build secret' >&2 + echo 'Error: Topling builds require an HTTPS TOPLING_JNI_URL or a topling-jni build context containing rocksdbjni.jar' >&2 exit 1 } download_dir=$(mktemp -d) diff --git a/hugegraph-pd/Dockerfile b/hugegraph-pd/Dockerfile index f6c4f89e6e..a96107b224 100644 --- a/hugegraph-pd/Dockerfile +++ b/hugegraph-pd/Dockerfile @@ -49,6 +49,10 @@ RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sh rm -f /tmp/hugegraph-build-cacerts; \ rm ./hugegraph-server/*.tar.gz ./hugegraph-pd/*.tar.gz ./hugegraph-store/*.tar.gz +# Optional local JNI directory: --build-context topling-jni=/path/to/directory. +# An empty default stage keeps ordinary and URL-based builds self-contained. +FROM scratch AS topling-jni + # 2nd stage: runtime env # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 FROM eclipse-temurin:17-jre-jammy AS standard-base @@ -109,7 +113,7 @@ ARG TOPLING_JNI_URL ARG TOPLING_JNI_SHA256 COPY docker/prepare-topling-image.sh docker/topling-runtime.env docker/toplingdb.yaml /tmp/ RUN --mount=type=secret,id=proxy_ca,required=false \ - --mount=type=secret,id=topling_jni,required=false \ + --mount=type=bind,from=topling-jni,target=/run/topling-jni \ if [ "$RUNTIME_VARIANT" = topling ]; then \ if [ -f /run/secrets/proxy_ca ]; then export CURL_CA_BUNDLE=/run/secrets/proxy_ca; fi; \ bash /tmp/prepare-topling-image.sh; \ diff --git a/hugegraph-server/Dockerfile b/hugegraph-server/Dockerfile index d33e6660fc..37b2ed70d4 100644 --- a/hugegraph-server/Dockerfile +++ b/hugegraph-server/Dockerfile @@ -49,6 +49,10 @@ RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sh rm -f /tmp/hugegraph-build-cacerts; \ rm ./hugegraph-server/*.tar.gz ./hugegraph-pd/*.tar.gz ./hugegraph-store/*.tar.gz +# Optional local JNI directory: --build-context topling-jni=/path/to/directory. +# An empty default stage keeps ordinary and URL-based builds self-contained. +FROM scratch AS topling-jni + # 2nd stage: runtime env # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 FROM eclipse-temurin:17-jdk-jammy AS standard-base @@ -115,7 +119,7 @@ ARG TOPLING_JNI_URL ARG TOPLING_JNI_SHA256 COPY docker/prepare-topling-image.sh docker/topling-runtime.env docker/toplingdb.yaml /tmp/ RUN --mount=type=secret,id=proxy_ca,required=false \ - --mount=type=secret,id=topling_jni,required=false \ + --mount=type=bind,from=topling-jni,target=/run/topling-jni \ if [ "$RUNTIME_VARIANT" = topling ]; then \ if [ -f /run/secrets/proxy_ca ]; then export CURL_CA_BUNDLE=/run/secrets/proxy_ca; fi; \ bash /tmp/prepare-topling-image.sh; \ diff --git a/hugegraph-store/Dockerfile b/hugegraph-store/Dockerfile index 33b018225d..1adfea2f2b 100644 --- a/hugegraph-store/Dockerfile +++ b/hugegraph-store/Dockerfile @@ -49,6 +49,10 @@ RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sh rm -f /tmp/hugegraph-build-cacerts; \ rm ./hugegraph-server/*.tar.gz ./hugegraph-pd/*.tar.gz ./hugegraph-store/*.tar.gz +# Optional local JNI directory: --build-context topling-jni=/path/to/directory. +# An empty default stage keeps ordinary and URL-based builds self-contained. +FROM scratch AS topling-jni + # 2nd stage: runtime env # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 FROM eclipse-temurin:17-jdk-jammy AS standard-base @@ -110,7 +114,7 @@ ARG TOPLING_JNI_URL ARG TOPLING_JNI_SHA256 COPY docker/prepare-topling-image.sh docker/topling-runtime.env docker/toplingdb.yaml /tmp/ RUN --mount=type=secret,id=proxy_ca,required=false \ - --mount=type=secret,id=topling_jni,required=false \ + --mount=type=bind,from=topling-jni,target=/run/topling-jni \ if [ "$RUNTIME_VARIANT" = topling ]; then \ if [ -f /run/secrets/proxy_ca ]; then export CURL_CA_BUNDLE=/run/secrets/proxy_ca; fi; \ bash /tmp/prepare-topling-image.sh; \ From 9dab027cc45472cf604bb33732171a3f67757227 Mon Sep 17 00:00:00 2001 From: orion Date: Fri, 23 Oct 2026 14:35:05 +0000 Subject: [PATCH 04/29] docs(topling): describe the native CPU requirement --- docker/README.md | 3 ++- helm/hugegraph/README.md | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 3a8ac2c44d..281290f36d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -291,7 +291,8 @@ docker compose -p hugegraph-topling-hstore -f docker-compose-hstore.yml up -d -- Use the same project name on subsequent `ps`, `stop`, `down` and `up` commands. Persist `HUGEGRAPH_VERSION=latest-topling` in that deployment's `.env` if preferred. A published version uses `-topling`; the unsuffixed tag keeps RocksDB. -Topling images support **Linux amd64**. All HugeGraph images in a deployment set +Topling images support **Linux amd64 on a Haswell-compatible CPU**, including +AVX2 and BMI2, as required by the current JNI build. All HugeGraph images in a deployment set use the same source revision. HStore Server keeps `backend=hstore` and delegates storage to PD/Store; it does not load a local Topling JNI runtime. Hubble keeps its independent image tag. diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 38a3814e45..bbfe03965b 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -113,6 +113,9 @@ independent image. Linux amd64 scheduling is added to these HugeGraph workloads and the test hook. Contradictory OS/architecture node selectors are rejected; other scheduling constraints are preserved. +The current JNI requires a Haswell-compatible x86-64 CPU, including AVX2 and BMI2. +The architecture selector does not inspect CPU extensions; use a compatible node pool. + A component tag such as `1.8.0` becomes `1.8.0-topling`; tags already ending in `-topling` are kept. Custom repositories and `global.imageRegistry` work as usual. An explicit `image.digest` remains exact and takes priority over the derived tag; From cf042274056a5f7690c75c835b63ee8c1ce25b9f Mon Sep 17 00:00:00 2001 From: orion Date: Fri, 23 Oct 2026 14:48:02 +0000 Subject: [PATCH 05/29] fix(topling): pin the verified community JNI release --- docker/topling-runtime.env | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/topling-runtime.env b/docker/topling-runtime.env index 95a00982b7..fab1bcaff3 100644 --- a/docker/topling-runtime.env +++ b/docker/topling-runtime.env @@ -15,5 +15,7 @@ # specific language governing permissions and limitations # under the License. -# Pin the verified immutable fork release here after its native CI probe passes. -# Until then, Topling source builds require explicit URL and SHA-256 build args. +# Community JNI: no private ToplingZip builder, CSPP MemTable or dcompact. +# Native probe and provenance: hugegraph/toplingdb Actions run 39969281764. +TOPLING_JNI_URL=https://github.com/hugegraph/toplingdb/releases/download/jni-2916f6895c62e535a03985d8e9b0879e64304715/rocksdbjni-8.10.2-linux64.jar +TOPLING_JNI_SHA256=c9066873d5eb38585517c0c4065565897aab96629585731b6fb36d64b81efef2 From 61645b11ba8a277be5bf157a9f43b164fb437082 Mon Sep 17 00:00:00 2001 From: orion Date: Fri, 23 Oct 2026 14:50:01 +0000 Subject: [PATCH 06/29] docs(topling): show the source-pinned default build --- docker/README.md | 19 ++++++++++++++----- docs/storage-lifecycle.md | 7 ++++++- helm/hugegraph/README.md | 5 ++++- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/docker/README.md b/docker/README.md index 281290f36d..a330026b61 100644 --- a/docker/README.md +++ b/docker/README.md @@ -313,13 +313,13 @@ Returning to RocksDB uses its original deployment and volumes. See [Topling runtime preparation and validation](../docs/storage-lifecycle.md#storage-provider-selection-and-toplingdb) for the native configuration and compatibility requirements. -For a local HStore source build, use the existing development overlay. Supply an -immutable Topling JNI URL and its trusted SHA-256; both PD and Store reuse them: +Source builds use the community JNI release and trusted SHA-256 pinned in +[`topling-runtime.env`](topling-runtime.env). The image build downloads and verifies +that artifact once; selecting the variant is the only required build input. +For a local HStore source build, use the existing development overlay: ```bash export RUNTIME_VARIANT=topling -export TOPLING_JNI_URL='https://your-artifact-host/immutable/rocksdbjni.jar' -export TOPLING_JNI_SHA256='' docker compose -p hugegraph-topling-dev \ -f docker-compose-hstore.yml -f docker-compose.dev.yml up -d --build --wait ``` @@ -327,7 +327,16 @@ docker compose -p hugegraph-topling-dev \ The overlay uses the usual local `:dev` tags. Build it on Linux amd64 and keep its project volumes separate from a RocksDB source deployment. For direct Docker and Bake builds, use `--build-arg RUNTIME_VARIANT=topling` or `RUNTIME_VARIANT=topling` -respectively, together with the same JNI URL and checksum build inputs. +respectively; the same pinned JNI defaults apply. + +To use a different compatible JNI release, optionally override both +`TOPLING_JNI_URL` and `TOPLING_JNI_SHA256`. Compose and Bake accept them as environment +variables; direct Docker builds accept the same names as build arguments: + +```bash +export TOPLING_JNI_URL='https://your-artifact-host/immutable/rocksdbjni.jar' +export TOPLING_JNI_SHA256='' +``` For a verified local JNI JAR, put it in a directory as `rocksdbjni.jar` and supply that directory as a named build context instead of a download URL: diff --git a/docs/storage-lifecycle.md b/docs/storage-lifecycle.md index d79ea5e2d7..791708e92c 100644 --- a/docs/storage-lifecycle.md +++ b/docs/storage-lifecycle.md @@ -501,7 +501,12 @@ Prepared image variants reuse the normal deployment mechanisms. Select `hugegraph/hugegraph:latest-topling` for a standalone Server, or the matching `hugegraph/pd`, `hugegraph/store` and `hugegraph/server` tags for distributed HStore. Published versions use `-topling`. Topling images support Linux -amd64 and include the verified JNI runtime; startup does not download dependencies. +amd64 and include the verified community JNI runtime; startup does not download +dependencies. Source image builds select it with `RUNTIME_VARIANT=topling` and use +the immutable release URL and trusted SHA-256 in +[`docker/topling-runtime.env`](../docker/topling-runtime.env). Custom JNI URL/checksum +build inputs are optional overrides. The unpacked-distribution preparation below +still requires an explicit JAR and its trusted checksum. Compose uses its existing `HUGEGRAPH_VERSION=latest-topling` parameter. Helm uses `--set global.runtimeVariant=topling` to select matching images and amd64 scheduling. diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index bbfe03965b..04364c7908 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -111,7 +111,10 @@ including the Store wait container and Helm test image. PD and Store own the ToplingDB databases; Server continues to use `backend=hstore`. Hubble keeps its independent image. Linux amd64 scheduling is added to these HugeGraph workloads and the test hook. Contradictory OS/architecture node selectors are rejected; -other scheduling constraints are preserved. +other scheduling constraints are preserved. The deployment images already include +the verified community JNI; no artifact URL or checksum is needed in Helm values. +For source image builds, `RUNTIME_VARIANT=topling` uses the release pinned in +[`docker/topling-runtime.env`](../../docker/topling-runtime.env). The current JNI requires a Haswell-compatible x86-64 CPU, including AVX2 and BMI2. The architecture selector does not inspect CPU extensions; use a compatible node pool. From 0accc3b3714672b5b305264478232cd74622ef3c Mon Sep 17 00:00:00 2001 From: imbajin Date: Sat, 24 Oct 2026 00:16:18 +0800 Subject: [PATCH 07/29] fix(docker): align shared Maven stage boundary --- hugegraph-server/Dockerfile-hstore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hugegraph-server/Dockerfile-hstore b/hugegraph-server/Dockerfile-hstore index 5ef3b210eb..3ca1c33854 100644 --- a/hugegraph-server/Dockerfile-hstore +++ b/hugegraph-server/Dockerfile-hstore @@ -46,6 +46,11 @@ RUN --mount=type=cache,id=hugegraph-maven-${SOURCE_REVISION},target=/root/.m2,sh rm -f /tmp/hugegraph-build-cacerts; \ rm ./hugegraph-server/*.tar.gz ./hugegraph-pd/*.tar.gz ./hugegraph-store/*.tar.gz +# Optional local JNI directory: --build-context topling-jni=/path/to/directory. +# An empty default stage keeps ordinary and URL-based builds self-contained. +FROM scratch AS topling-jni +# HStore intentionally does not consume this stage and keeps its standard runtime. + # 2nd stage: runtime env # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 FROM eclipse-temurin:17-jdk-jammy From 58e974591c3fc105215390f79182d9cd552b036b Mon Sep 17 00:00:00 2001 From: orion Date: Fri, 23 Oct 2026 16:19:49 +0000 Subject: [PATCH 08/29] fix(docker): keep replacement runtimes outside persistent volumes --- docker/README.md | 16 +++++++++++++++- hugegraph-pd/Dockerfile | 3 ++- hugegraph-server/Dockerfile | 3 ++- hugegraph-server/Dockerfile-hstore | 1 - hugegraph-store/Dockerfile | 3 ++- 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/docker/README.md b/docker/README.md index a330026b61..2eed932a13 100644 --- a/docker/README.md +++ b/docker/README.md @@ -435,7 +435,21 @@ Both Server images (`hugegraph/hugegraph` and `hugegraph/server`) write the Huge `/hugegraph-server/logs` holds `hugegraph-server.log`, JVM crash logs (`hs_err_pid__.log`) and out-of-memory heap dumps (`heapdump__/java_pid.hprof`), where `` is the container's local time as `YYYYMMDD-HHMMSS`. Each start gets its own `heapdump_*` directory, so the Server and the JVMs it starts, such as computer jobs, write separate dumps. HotSpot names each dump and crash log after the JVM's PID, so a JVM that reuses the PID of an earlier one from the same start cannot write its dump, and its crash log replaces the earlier one's. The host name (on Kubernetes, the pod name, or `spec.hostname` when that is set) and a `-1`, `-2`, ... counter keep names from clashing; the counter also separates pods that share a host name. If the directory cannot be created, for example on a full volume, the Server still starts and dumps go to `logs/` itself as `java_pid.hprof`. That is best effort: a full volume may have no room for the dump either, and a restarted container that reuses a PID cannot write over an earlier dump with the same name. If the logs path contains a double quote or `%`, the launcher warns and heap dumps go to `logs/` itself, with no per-launch directory and no crash log default; the `JAVA_TOOL_OPTIONS` opt-out below still applies. A path that contains both quote characters gets no heap dump default at all. -Both images declare `VOLUME /hugegraph-server`, so `docker restart` and a Compose recreate keep the files in that anonymous volume. `docker compose down` leaves the volume behind but the next `up` does not attach it, so the files become hard to reach and keep using disk. A later `docker compose down -v` does not remove that orphaned volume either, since it only removes anonymous volumes of the containers it removes; note the volume name with `docker inspect` before `down` and delete it with `docker volume rm `, or stop with `docker compose down -v` in the first place. Avoid `docker volume prune` for this: it deletes every unused anonymous volume on the Docker host, including other projects' data. Mount a named volume or a bind mount at `/hugegraph-server/logs` to keep the files reachable. Kubernetes starts a new container on every restart and does not turn the image's `VOLUME` into a pod volume, so mount an `emptyDir` or a PersistentVolumeClaim at `/hugegraph-server/logs`. Give each pod its own volume, or a per-pod directory on a shared PVC: crash files get distinct names, but `hugegraph-server.log`, the audit log and the slow-query log have fixed names, and pods sharing them would roll them over for each other. For a per-pod directory, pass the pod name in through the Downward API and use it in `subPathExpr` (the Helm chart sets neither): +The images persist database directories rather than the application installation. A +`docker restart` keeps logs in the existing container; replacing the container requires +a named volume or bind mount at `/hugegraph-server/logs` to retain them. Older images +declared `VOLUME /hugegraph-server`, which could also retain old binaries and JNI during +an image change. Do not mount an old installation directory over a replacement image; +keep database and any custom configuration mounts separate. An old anonymous root volume +can remain after its container is removed. Inspect its contents before removing that +specific volume; it may contain logs or data from a deployment without explicit data +mounts. Kubernetes starts a new container on every restart, so mount an `emptyDir` or a +PersistentVolumeClaim at `/hugegraph-server/logs` to preserve logs across container +restarts. Give each pod its own volume, or a per-pod directory on a shared PVC: crash +files get distinct names, but `hugegraph-server.log`, the audit log and the slow-query +log have fixed names, and pods sharing them would roll them over for each other. For a +per-pod directory, pass the pod name in through the Downward API and use it in +`subPathExpr` (the Helm chart sets neither): ```yaml env: diff --git a/hugegraph-pd/Dockerfile b/hugegraph-pd/Dockerfile index a96107b224..3c69d793c1 100644 --- a/hugegraph-pd/Dockerfile +++ b/hugegraph-pd/Dockerfile @@ -121,7 +121,8 @@ RUN --mount=type=secret,id=proxy_ca,required=false \ && rm /tmp/prepare-topling-image.sh /tmp/topling-runtime.env /tmp/toplingdb.yaml EXPOSE 8620 -VOLUME /hugegraph-pd +# Persist data without hiding a replacement image's binaries and native runtime. +VOLUME /hugegraph-pd/pd_data HEALTHCHECK --interval=15s --timeout=10s --start-period=90s --retries=3 \ CMD curl -fsS http://localhost:8620/v1/health >/dev/null diff --git a/hugegraph-server/Dockerfile b/hugegraph-server/Dockerfile index 37b2ed70d4..cefff239dd 100644 --- a/hugegraph-server/Dockerfile +++ b/hugegraph-server/Dockerfile @@ -127,7 +127,8 @@ RUN --mount=type=secret,id=proxy_ca,required=false \ && rm /tmp/prepare-topling-image.sh /tmp/topling-runtime.env /tmp/toplingdb.yaml EXPOSE 8080 -VOLUME /hugegraph-server +# The default data and WAL share this directory. Keep lib/bin/conf image-owned. +VOLUME /hugegraph-server/rocksdb-data HEALTHCHECK --interval=15s --timeout=10s --start-period=90s --retries=3 \ CMD curl -fsS http://localhost:8080/versions >/dev/null diff --git a/hugegraph-server/Dockerfile-hstore b/hugegraph-server/Dockerfile-hstore index 3ca1c33854..92abac2798 100644 --- a/hugegraph-server/Dockerfile-hstore +++ b/hugegraph-server/Dockerfile-hstore @@ -106,7 +106,6 @@ COPY hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh . RUN chmod 755 ./docker-entrypoint.sh EXPOSE 8080 -VOLUME /hugegraph-server HEALTHCHECK --interval=15s --timeout=10s --start-period=90s --retries=3 \ CMD curl -fsS http://localhost:8080/versions >/dev/null diff --git a/hugegraph-store/Dockerfile b/hugegraph-store/Dockerfile index 1adfea2f2b..0daf7af844 100644 --- a/hugegraph-store/Dockerfile +++ b/hugegraph-store/Dockerfile @@ -122,7 +122,8 @@ RUN --mount=type=secret,id=proxy_ca,required=false \ && rm /tmp/prepare-topling-image.sh /tmp/topling-runtime.env /tmp/toplingdb.yaml EXPOSE 8520 -VOLUME /hugegraph-store +# Persist data and Raft state without retaining the previous image's runtime. +VOLUME /hugegraph-store/storage HEALTHCHECK --interval=15s --timeout=10s --start-period=90s --retries=3 \ CMD curl -fsS http://localhost:8520/v1/health >/dev/null From 40804cf74612c12c8d556f95da5a352a7ff73dac Mon Sep 17 00:00:00 2001 From: imbajin Date: Sat, 24 Oct 2026 00:23:26 +0800 Subject: [PATCH 09/29] test(docker): lock image persistence boundaries --- .github/scripts/check-docker-images.sh | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/scripts/check-docker-images.sh b/.github/scripts/check-docker-images.sh index 41c8911246..f1265f18ce 100644 --- a/.github/scripts/check-docker-images.sh +++ b/.github/scripts/check-docker-images.sh @@ -6,7 +6,7 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -# ++# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software @@ -30,6 +30,28 @@ for pair in \ HC=$(docker image inspect --format '{{json .Config.Healthcheck}}' "$IMAGE_ID") [[ "$HC" != null ]] || { echo "Missing HEALTHCHECK: $dockerfile"; exit 1; } + case "$dockerfile" in + hugegraph-pd/Dockerfile) + EXPECTED_VOLUME=/hugegraph-pd/pd_data + ;; + hugegraph-store/Dockerfile) + EXPECTED_VOLUME=/hugegraph-store/storage + ;; + hugegraph-server/Dockerfile) + EXPECTED_VOLUME=/hugegraph-server/rocksdb-data + ;; + hugegraph-server/Dockerfile-hstore) + EXPECTED_VOLUME= + ;; + esac + ACTUAL_VOLUMES=$(docker image inspect --format \ + '{{range $path, $_ := .Config.Volumes}}{{$path}}{{"\n"}}{{end}}' \ + "$IMAGE_ID" | sed '/^$/d' | sort) + [[ "$ACTUAL_VOLUMES" == "$EXPECTED_VOLUME" ]] || { + echo "ERROR: $dockerfile declares volumes [$ACTUAL_VOLUMES]; expected [$EXPECTED_VOLUME]" + exit 1 + } + # Helm reuses these images for tool containers instead of pulling curlimages/curl. # Check the same non-root/read-only tool environment as the Server test hook. if [[ "$dockerfile" == hugegraph-store/* || "$dockerfile" == hugegraph-server/* ]]; then From ea8a49809ffe29c6055e08fcab213eee5c056dee Mon Sep 17 00:00:00 2001 From: orion Date: Fri, 23 Oct 2026 16:33:49 +0000 Subject: [PATCH 10/29] feat(helm): preserve qualified storage across RocksDB runtime switches --- docker/README.md | 41 +++++-- docs/storage-lifecycle.md | 34 ++++-- helm/hugegraph/README.md | 31 +++-- helm/hugegraph/docs/operations.md | 29 +++++ helm/hugegraph/templates/_helpers.tpl | 5 - .../persistence_upgrade_persistent_test.yaml | 6 +- .../tests/runtime_upgrade_standard_test.yaml | 109 ++++++++++++++++++ .../hugegraph/tests/runtime_upgrade_test.yaml | 76 +++++++++++- helm/hugegraph/values.yaml | 2 +- 9 files changed, 295 insertions(+), 38 deletions(-) create mode 100644 helm/hugegraph/tests/runtime_upgrade_standard_test.yaml diff --git a/docker/README.md b/docker/README.md index 2eed932a13..6b004b1ba7 100644 --- a/docker/README.md +++ b/docker/README.md @@ -280,7 +280,7 @@ Compose files, credentials, ports and lifecycle commands apply: ```bash export HUGEGRAPH_VERSION=latest-topling # Standalone Server with embedded ToplingDB -# A distinct project creates fresh volumes for this runtime. +# First deployment docker compose -p hugegraph-topling-standalone -f docker-compose.yml up -d --wait # Or PD + Store using ToplingDB, with an HStore Server @@ -306,12 +306,38 @@ still scales with databases and column families; these settings do not cap total process memory. Mount a tuned `conf/toplingdb.yaml` or select an external file with `TOPLINGDB_EASY_MIGRATE_CONF` when sizing a production deployment. CSPP and other optional modules require your own compatible JNI artifact and native profile. -The unpacked distribution's full profile is unchanged. -Existing data is not converted by selecting another image: use new volumes for -ToplingDB, and migrate through the graph APIs after validating your workload. -Returning to RocksDB uses its original deployment and volumes. See +The unpacked distribution's full profile is unchanged. The default community +profile uses standard BlockBasedTable SSTs and supports opening compatible RocksDB +data after a normal shutdown. The publication check seeds data using standard +images from the same source commit, switches to Topling on the same volumes, and +checks schema, indexes, graph operations and restart persistence. See [Topling runtime preparation and validation](../docs/storage-lifecycle.md#storage-provider-selection-and-toplingdb) -for the native configuration and compatibility requirements. +for the native configuration and tested compatibility boundary. + +For an existing deployment, stop incoming work, stop the services normally, and +select the matching Topling images. Keep the same Compose project, credentials and +data mounts. For standalone, from this directory: + +```bash +docker compose -f docker-compose.yml stop +HUGEGRAPH_VERSION='-topling' \ + docker compose -f docker-compose.yml up -d --wait +``` + +Use `docker-compose-hstore.yml` for the corresponding distributed deployment. +The named data volumes are reattached automatically; no export/import or new +project is needed. Return to standard RocksDB with the same stop/up sequence and +its matching unsuffixed tag. Prefer versioned tags or verified digests: Topling +`latest-topling` is published on demand and can lag standard `latest`. Changing +storage providers does not bypass HugeGraph or RocksDB version compatibility. +Back up the data and qualify representative existing data before production use. + +The images declare only database directories as Docker volumes, so replacing an +image also replaces its binaries, configuration defaults and JNI. The supplied +Compose files already mount these data directories separately. If an older +custom deployment kept data only in an anonymous whole-application volume, first +identify and preserve that volume's database directories; they are not copied +automatically into a new data volume. Custom data/WAL paths must remain mounted. Source builds use the community JNI release and trusted SHA-256 pinned in [`topling-runtime.env`](topling-runtime.env). The image build downloads and verifies @@ -324,8 +350,7 @@ docker compose -p hugegraph-topling-dev \ -f docker-compose-hstore.yml -f docker-compose.dev.yml up -d --build --wait ``` -The overlay uses the usual local `:dev` tags. Build it on Linux amd64 and keep its -project volumes separate from a RocksDB source deployment. For direct Docker and +The overlay uses the usual local `:dev` tags. Build it on Linux amd64. For direct Docker and Bake builds, use `--build-arg RUNTIME_VARIANT=topling` or `RUNTIME_VARIANT=topling` respectively; the same pinned JNI defaults apply. diff --git a/docs/storage-lifecycle.md b/docs/storage-lifecycle.md index 791708e92c..3d5b99389d 100644 --- a/docs/storage-lifecycle.md +++ b/docs/storage-lifecycle.md @@ -516,10 +516,27 @@ provider and launcher together. See the [Compose guide](../docker/README.md#topl and [Helm guide](../helm/hugegraph/README.md#toplingdb-runtime) for complete commands, source builds, custom registries and immutable image digests. -Select a new Compose project or Helm release with fresh volumes for validation. -Changing the image does not migrate an existing RocksDB database. Keep the original -deployment's data and use graph APIs for a verified migration; the Helm chart -rejects runtime changes on an initialized persistent release. +Validate a new deployment first, then qualify a normal stop and image switch using +existing RocksDB data. With the default community profile, Topling uses standard +BlockBasedTable SSTs; a compatible existing database can be opened directly, without +graph API export/import. Preserve the Compose project or Helm release, credentials, +data/WAL paths and named volumes/PVCs. The chart permits a runtime change while +retaining its storage and Raft membership protections. The images keep binaries and +JNI outside Docker data volumes so a Compose image change loads the selected runtime. + +The image publication gate checks standard-to-Topling compatibility with standard +images built from the same HugeGraph source and the pinned community JNI (currently +RocksDB API 8.10.2). It checks real native-library selection, persisted data, schema, +indexes, continued writes and normal restart. This does not establish compatibility +between arbitrary HugeGraph/RocksDB releases or custom native table formats. Use a +matching version pair, retain a backup and qualify representative data. Topling +Latest is manual and may lag standard Latest; formal Release publishes both variants +from one Apache HugeGraph commit. + +For an older custom Docker deployment with data stored only in an anonymous volume +covering the entire installation, preserve its database directories explicitly before +changing to data-only mounts. The supplied Compose data mounts already keep these +directories separate. See the linked deployment guides for the stop/switch/up commands. ### Standalone Server setup @@ -682,9 +699,12 @@ Stop incoming work and use the component's normal stop script. PD drains its sch metadata work, joins Raft and closes the metadata database and native options. Do not use a forced kill as evidence of normal resource cleanup. -To return to standard RocksDB, stop the component, unset `TOPLINGDB_ROCKSDB_PROVIDER` -and set its business provider back to `rocksdb`. Use the corresponding standard-provider -data directory; do not test a provider switch by reusing the other provider's database. +To return to standard RocksDB, stop the component normally, unset +`TOPLINGDB_ROCKSDB_PROVIDER` and set its business provider back to `rocksdb`. Keep +the same qualified data and WAL paths when switching the default BlockBasedTable +profile. Native profiles that produce custom SST formats need separate compatibility +validation before returning to a standard runtime. For containers, select the +matching unsuffixed image instead; it configures the launcher and provider together. ## Implementation navigation diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 04364c7908..31ffe9579a 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -99,7 +99,18 @@ See the [complete registry example](examples/values-internal-registry.yaml). ### ToplingDB runtime -Select ToplingDB for a **fresh deployment** with one value: +Switch an existing release to ToplingDB with one value while keeping its values +and persistent volumes: + +```bash +helm upgrade hugegraph ./helm/hugegraph -n hugegraph --reuse-values \ + --set global.runtimeVariant=topling --wait --timeout 15m +``` + +Use matching standard/Topling images of the same HugeGraph version. Development +`latest` and `latest-topling` tags are published independently and may refer to +different source revisions; select a matching build before changing the runtime. +For a new installation, use the same runtime parameter: ```bash helm install hugegraph-topling ./helm/hugegraph -n hugegraph-topling \ @@ -125,13 +136,17 @@ An explicit `image.digest` remains exact and takes priority over the derived tag supply the digest of the matching Topling deployment image. Explicit helper image overrides also remain exact. -The default `global.runtimeVariant=standard` keeps RocksDB. Selecting another -runtime does not convert existing data. Use separate empty volumes and migrate -through graph APIs after validating the application. The chart rejects runtime -changes on an existing persistent PD or Store release, including a return from -ToplingDB to RocksDB. Keep the original release and volumes to return to RocksDB. -See the [storage runtime guide](../../docs/storage-lifecycle.md#storage-provider-selection-and-toplingdb) -for native configuration and validation. +The default `global.runtimeVariant=standard` keeps RocksDB. The verified stock +RocksDB 8.10.2 and pinned community ToplingDB 8.10.2 use compatible data: the +upgrade retains PD/Store data and Raft volumes, schema, indexes and graph records. +Keep the release name, namespace, data paths, storage settings and credentials; +no graph export or new release is needed. Allow normal Pod shutdown during the +maintenance window. The same verified pair also supports returning to RocksDB: +repeat the upgrade with `global.runtimeVariant=standard`, keeping the same PVCs. +Use base image tags without the `-topling` suffix, or update explicit digests to +the matching RocksDB images. See [storage runtime upgrades](docs/operations.md#storage-runtime-upgrades) +for rollout behavior and the [storage runtime guide](../../docs/storage-lifecycle.md#storage-provider-selection-and-toplingdb) +for the verified native versions, profile and compatibility coverage. ### Storage diff --git a/helm/hugegraph/docs/operations.md b/helm/hugegraph/docs/operations.md index 4c90e1e6ba..a16e2422fa 100644 --- a/helm/hugegraph/docs/operations.md +++ b/helm/hugegraph/docs/operations.md @@ -193,6 +193,35 @@ Server replicas can be changed routinely; CPU-utilization HPA requires positive CPU requests. Store growth still needs volume and shard planning. Use chart updates for Store scale changes so Hubble's endpoint list is updated too. +## Storage runtime upgrades + +For the verified stock RocksDB 8.10.2 and pinned community ToplingDB 8.10.2, +change `global.runtimeVariant` with the ordinary +[Helm upgrade command](../README.md#toplingdb-runtime). The existing PD and Store +StatefulSet names, data paths and PVCs stay the same, including Store Raft data. +The verified pair supports both `standard` to `topling` and a return to `standard` +after Topling writes, using the same volumes. +Use matching HugeGraph image versions and retain the installed credentials and +storage values; runtime selection does not require graph export or initialization. +Development `latest` tags may have different source revisions because they are +published independently. + +The default `RollingUpdate` strategy terminates each Pod normally before its +replacement starts with the same ordinal and PVC. `podManagementPolicy: Parallel` +controls creation and scaling, not parallel rolling replacement within a +StatefulSet. PD and Store have separate controllers, so plan the change in a +maintenance window; this does not promise uninterrupted requests. Keep enough +termination grace for database shutdown and do not force-delete the old Pods. +The existing [Store rollout and recovery checks](#rolling-store-updates-and-recovery) +still apply. + +The compatibility scope is the pinned community profile, which uses SkipList and +BlockBasedTable. Other JNI versions and custom SST formats need their own +compatibility evidence. Exact image digest overrides must identify the target +runtime, since a digest takes precedence over a derived tag. See the +[storage runtime guide](../../../docs/storage-lifecycle.md#storage-provider-selection-and-toplingdb) +for the tested native versions and data coverage. + ## Rolling Store updates and recovery Use `store.updateStrategy.type=OnDelete` for a controlled rollout. Replace one diff --git a/helm/hugegraph/templates/_helpers.tpl b/helm/hugegraph/templates/_helpers.tpl index c2c89de9af..685e39d837 100644 --- a/helm/hugegraph/templates/_helpers.tpl +++ b/helm/hugegraph/templates/_helpers.tpl @@ -814,11 +814,6 @@ and the upgrade passes. {{- fail (printf "%s.storage.enabled cannot change on an initialized release: StatefulSet volumeClaimTemplates are immutable and changing storage can discard data. Use a fresh release and migrate data" $comp) -}} {{- end -}} -{{- $liveRuntime := dig "metadata" "annotations" "hugegraph.apache.org/rocksdb-runtime" "standard" $live -}} -{{- $runtime := get ($.Values.global | default dict) "runtimeVariant" | default "standard" -}} -{{- if and $wasPersistent (ne $liveRuntime $runtime) -}} -{{- fail (printf "global.runtimeVariant cannot change from %s to %s on an initialized persistent %s release. Use a fresh release with empty volumes and migrate data through graph APIs" $liveRuntime $runtime $comp) -}} -{{- end -}} {{- $liveReplicas := int (dig "spec" "replicas" 0 $live) -}} {{- $desired := int (get (get $.Values $comp) "replicas") -}} {{- if and (gt $liveReplicas 0) (lt $desired $liveReplicas) -}} diff --git a/helm/hugegraph/tests/persistence_upgrade_persistent_test.yaml b/helm/hugegraph/tests/persistence_upgrade_persistent_test.yaml index 914c9878af..3698581755 100644 --- a/helm/hugegraph/tests/persistence_upgrade_persistent_test.yaml +++ b/helm/hugegraph/tests/persistence_upgrade_persistent_test.yaml @@ -114,11 +114,11 @@ tests: - isKind: of: Deployment - - it: rejects reusing persistent RocksDB volumes with Topling + - it: accepts changing runtime while keeping persistent storage release: upgrade: true set: global.runtimeVariant: topling asserts: - - failedTemplate: - errorPattern: "global.runtimeVariant cannot change from standard to topling" + - isKind: + of: Deployment diff --git a/helm/hugegraph/tests/runtime_upgrade_standard_test.yaml b/helm/hugegraph/tests/runtime_upgrade_standard_test.yaml new file mode 100644 index 0000000000..02491e1189 --- /dev/null +++ b/helm/hugegraph/tests/runtime_upgrade_standard_test.yaml @@ -0,0 +1,109 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Retained RocksDB volumes on Topling upgrades +templates: + - pd-statefulset.yaml + - store-statefulset.yaml +kubernetesProvider: + scheme: + "apps/v1/StatefulSet": + gvr: + group: apps + version: v1 + resource: statefulsets + namespaced: true + objects: + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: RELEASE-NAME-hugegraph-pd + namespace: NAMESPACE + spec: + replicas: "3" + volumeClaimTemplates: + - metadata: + name: pd-data + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: RELEASE-NAME-hugegraph-store + namespace: NAMESPACE + spec: + replicas: "3" + volumeClaimTemplates: + - metadata: + name: store-data +tests: + - it: changes PD to Topling while retaining its existing claim and data mount + template: pd-statefulset.yaml + release: + upgrade: true + set: + global.runtimeVariant: topling + asserts: + - equal: + path: metadata.name + value: RELEASE-NAME-hugegraph-pd + - equal: + path: spec.template.spec.containers[0].image + value: hugegraph/pd:latest-topling + - equal: + path: spec.volumeClaimTemplates[0].metadata.name + value: pd-data + - equal: + path: spec.template.spec.containers[0].volumeMounts[0] + value: + name: pd-data + mountPath: /hugegraph-pd/pd_data + - equal: + path: spec.replicas + value: 3 + - equal: + path: spec.updateStrategy.type + value: RollingUpdate + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 300 + + - it: changes Store to Topling while retaining its existing claim and data mount + template: store-statefulset.yaml + release: + upgrade: true + set: + global.runtimeVariant: topling + asserts: + - equal: + path: metadata.name + value: RELEASE-NAME-hugegraph-store + - equal: + path: spec.template.spec.containers[0].image + value: hugegraph/store:latest-topling + - equal: + path: spec.volumeClaimTemplates[0].metadata.name + value: store-data + - equal: + path: spec.template.spec.containers[0].volumeMounts[0] + value: + name: store-data + mountPath: /hugegraph-store/storage + - equal: + path: spec.replicas + value: 3 + - equal: + path: spec.updateStrategy.type + value: RollingUpdate diff --git a/helm/hugegraph/tests/runtime_upgrade_test.yaml b/helm/hugegraph/tests/runtime_upgrade_test.yaml index be59251976..e92e9efdda 100644 --- a/helm/hugegraph/tests/runtime_upgrade_test.yaml +++ b/helm/hugegraph/tests/runtime_upgrade_test.yaml @@ -15,9 +15,10 @@ # limitations under the License. # -suite: Topling persistent runtime upgrade guard +suite: Retained Topling volumes on runtime upgrades templates: - - server-deployment.yaml + - pd-statefulset.yaml + - store-statefulset.yaml kubernetesProvider: scheme: "apps/v1/StatefulSet": @@ -39,13 +40,73 @@ kubernetesProvider: volumeClaimTemplates: - metadata: name: pd-data + - apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: RELEASE-NAME-hugegraph-store + namespace: NAMESPACE + annotations: + hugegraph.apache.org/rocksdb-runtime: topling + spec: + replicas: "3" + volumeClaimTemplates: + - metadata: + name: store-data tests: - - it: rejects returning Topling volumes to the standard runtime + - it: returns PD to RocksDB with the same persistent volume and rollout settings + template: pd-statefulset.yaml + release: + upgrade: true + asserts: + - equal: + path: metadata.name + value: RELEASE-NAME-hugegraph-pd + - equal: + path: spec.template.spec.containers[0].image + value: hugegraph/pd:latest + - equal: + path: spec.volumeClaimTemplates[0].metadata.name + value: pd-data + - equal: + path: spec.template.spec.containers[0].volumeMounts[0] + value: + name: pd-data + mountPath: /hugegraph-pd/pd_data + - equal: + path: spec.replicas + value: 3 + - equal: + path: spec.updateStrategy.type + value: RollingUpdate + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 300 + + - it: returns Store to RocksDB with the same persistent volume + template: store-statefulset.yaml release: upgrade: true asserts: - - failedTemplate: - errorPattern: "global.runtimeVariant cannot change from topling to standard" + - equal: + path: metadata.name + value: RELEASE-NAME-hugegraph-store + - equal: + path: spec.template.spec.containers[0].image + value: hugegraph/store:latest + - equal: + path: spec.volumeClaimTemplates[0].metadata.name + value: store-data + - equal: + path: spec.template.spec.containers[0].volumeMounts[0] + value: + name: store-data + mountPath: /hugegraph-store/storage + - equal: + path: spec.replicas + value: 3 + - equal: + path: spec.updateStrategy.type + value: RollingUpdate - it: accepts an unchanged persistent Topling upgrade release: @@ -54,4 +115,7 @@ tests: global.runtimeVariant: topling asserts: - isKind: - of: Deployment + of: StatefulSet + - equal: + path: metadata.annotations["hugegraph.apache.org/rocksdb-runtime"] + value: topling diff --git a/helm/hugegraph/values.yaml b/helm/hugegraph/values.yaml index 2045b84b17..39c55edb03 100644 --- a/helm/hugegraph/values.yaml +++ b/helm/hugegraph/values.yaml @@ -24,7 +24,7 @@ mode: easy global: imageRegistry: "" # standard: normal RocksDB images; topling: matching -topling tags on Linux amd64. - # Select a runtime for a fresh deployment; this does not migrate existing data. + # Matching compatible images reuse existing data/PVCs after normal shutdown. runtimeVariant: standard persistence: enabled: true From b3a560471e503165932b6a1f4fc58f0712fc4130 Mon Sep 17 00:00:00 2001 From: orion Date: Fri, 23 Oct 2026 16:34:34 +0000 Subject: [PATCH 11/29] fix(ci): keep the image check license header executable --- .github/scripts/check-docker-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/check-docker-images.sh b/.github/scripts/check-docker-images.sh index f1265f18ce..c135e7cf60 100644 --- a/.github/scripts/check-docker-images.sh +++ b/.github/scripts/check-docker-images.sh @@ -6,7 +6,7 @@ # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at -+# +# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software From b2dfc49474a804c4a78792a3c4ec8c9ea07e3d2c Mon Sep 17 00:00:00 2001 From: imbajin Date: Sat, 24 Oct 2026 01:37:08 +0800 Subject: [PATCH 12/29] fix(helm): reject digest that bypasses Topling selection --- .../templates/runtime-variant-validation.yaml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 helm/hugegraph/templates/runtime-variant-validation.yaml diff --git a/helm/hugegraph/templates/runtime-variant-validation.yaml b/helm/hugegraph/templates/runtime-variant-validation.yaml new file mode 100644 index 0000000000..ca88ce4325 --- /dev/null +++ b/helm/hugegraph/templates/runtime-variant-validation.yaml @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +{{- include "hugegraph.resolveValues" . -}} +{{- if eq (get (.Values.global | default dict) "runtimeVariant" | default "standard") "topling" -}} +{{- range $component := list "pd" "store" "server" -}} +{{- $image := get (get $.Values $component) "image" | default dict -}} +{{- if ne (trim (get $image "digest" | default "")) "" -}} +{{- fail (printf "%s.image.digest cannot be used with global.runtimeVariant=topling: digest overrides the selected -topling tag. Clear the digest and use a matching version tag; variant-specific immutable digests require an explicit chart model" $component) -}} +{{- end -}} +{{- end -}} +{{- end -}} From f9388918d0a44e5928b1534640af5e75172896fa Mon Sep 17 00:00:00 2001 From: imbajin Date: Sat, 24 Oct 2026 01:37:21 +0800 Subject: [PATCH 13/29] test(helm): cover Topling digest overrides --- .../tests/runtime_variant_digest_test.yaml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 helm/hugegraph/tests/runtime_variant_digest_test.yaml diff --git a/helm/hugegraph/tests/runtime_variant_digest_test.yaml b/helm/hugegraph/tests/runtime_variant_digest_test.yaml new file mode 100644 index 0000000000..9909f32116 --- /dev/null +++ b/helm/hugegraph/tests/runtime_variant_digest_test.yaml @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +suite: Topling runtime digest selection +templates: + - runtime-variant-validation.yaml +tests: + - it: rejects a digest that would override the selected PD Topling tag + set: + global.runtimeVariant: topling + pd.image.digest: sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + asserts: + - failedTemplate: + errorPattern: "pd.image.digest cannot be used with global.runtimeVariant=topling" + + - it: rejects a digest that would override the selected Store Topling tag + set: + global.runtimeVariant: topling + store.image.digest: sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + asserts: + - failedTemplate: + errorPattern: "store.image.digest cannot be used with global.runtimeVariant=topling" + + - it: rejects a digest that would override the selected Server companion tag + set: + global.runtimeVariant: topling + server.image.digest: sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc + asserts: + - failedTemplate: + errorPattern: "server.image.digest cannot be used with global.runtimeVariant=topling" From 055fa436250587bb0f0895da78e1dec2a73a79a4 Mon Sep 17 00:00:00 2001 From: orion Date: Sun, 25 Oct 2026 06:37:25 +0000 Subject: [PATCH 14/29] fix(topling): honor selected storage provider - Keep explicit image provider above retained PD configuration. - Default runtime-created local RocksDB graphs to that provider. - Route the image contract to PD CI and cover its selector. - Clarify Helm digests and manual graph-config upgrade safeguards. --- .github/scripts/ci-policy.py | 1 + .github/scripts/test_ci_policy.py | 8 ++ docker/README.md | 51 ++++++++++-- docs/storage-lifecycle.md | 22 ++++- helm/hugegraph/README.md | 9 +- helm/hugegraph/docs/operations.md | 6 +- .../apache/hugegraph/core/GraphManager.java | 12 +++ .../unit/core/GraphManagerConfigTest.java | 46 ++++++++++ .../node/listener/PdConfigureListener.java | 17 ++++ .../listener/PdConfigureListenerYamlTest.java | 83 +++++++++++++++++++ 10 files changed, 241 insertions(+), 14 deletions(-) diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index d3e75dad2e..cc7cb024dc 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -101,6 +101,7 @@ TRAVIS = "hugegraph-server/hugegraph-dist/src/assembly/travis/" # Contract tests follow their executing suites, without runtime consumer fan-out. CONTRACT_SUITES = { + "docker/test-topling-images.sh": {"pd"}, TRAVIS + "test-check-java-runtime-logs.sh": {"struct"}, TRAVIS + "check-java-runtime-logs.sh": {"server", "commons", "struct", "pd", "store", "hstore", "cluster", "macos", "macos_full", "hbase", "riscv64"}, diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index 5c1d84f2a7..77c11e3e67 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -406,6 +406,14 @@ def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self): condition = re.search(r"^ if: (.+)$", step, re.M).group(1) self.assertTrue(eval(condition.replace("inputs.backend", repr(backend)), {"__builtins__": {}})) + + def test_topling_image_contract_selects_its_pd_executor(self): + plan = self.plan_for(["docker/test-topling-images.sh"]) + self.assertEqual({"pd"}, set(plan["selected"])) + self.assertEqual({"pd_store"}, self.coordinator_jobs(plan)) + workflow = (Path(__file__).resolve().parents[1] / "workflows/pd-store-ci.yml").read_text() + pd = workflow.split("\n pd:\n", 1)[1].split("\n store:\n", 1)[0] + self.assertIn("bash docker/test-topling-images.sh", pd) mixed = self.plan_for([policy.TRAVIS + "test-stop-hugegraph-store.sh", "hugegraph-pd/src/A.java"]) self.assertTrue({"struct", "pd", "store", "hstore", "cluster"}.issubset(mixed["selected"])) checker = self.plan_for([policy.TRAVIS + "check-java-runtime-logs.sh"]) diff --git a/docker/README.md b/docker/README.md index 6b004b1ba7..23d0273e38 100644 --- a/docker/README.md +++ b/docker/README.md @@ -314,27 +314,63 @@ checks schema, indexes, graph operations and restart persistence. See [Topling runtime preparation and validation](../docs/storage-lifecycle.md#storage-provider-selection-and-toplingdb) for the native configuration and tested compatibility boundary. -For an existing deployment, stop incoming work, stop the services normally, and +For an existing standalone deployment, first preserve **graph definitions as well +as database files**. API-created graphs write `conf/graphs/*.properties`; the +supplied named `server-data` volume does not persist that directory. Before the +first image replacement, stop incoming work and the old Server, then copy its +definitions out while the old container still exists: + +```bash +docker compose -f docker-compose.yml stop server +docker compose -f docker-compose.yml cp server:/hugegraph-server/conf/graphs ./graphs-backup +cp -a ./graphs-backup ./graph-config +``` + +Keep `graphs-backup` untouched. Save a writable configuration mount for the working +copy as `docker-compose.graph-config.yml`, and retain it on every subsequent start: + +```yaml +services: + server: + volumes: + - ./graph-config:/hugegraph-server/conf/graphs +``` + +Apply this override alongside the normal Compose file when starting the new +image. Verify all expected definitions and their data/WAL paths before replacement. +Custom `graphs` directories need equivalent backup and mounts. Database directories +alone cannot rediscover missing graph definitions. This is a manual prerequisite; +the image does not migrate old anonymous volumes or automatically persist future +API-created graph definitions outside the mounted configuration directory. + +Then stop incoming work, stop the services normally, and select the matching Topling images. Keep the same Compose project, credentials and data mounts. For standalone, from this directory: ```bash docker compose -f docker-compose.yml stop HUGEGRAPH_VERSION='-topling' \ - docker compose -f docker-compose.yml up -d --wait + docker compose -f docker-compose.yml -f docker-compose.graph-config.yml up -d --wait ``` Use `docker-compose-hstore.yml` for the corresponding distributed deployment. The named data volumes are reattached automatically; no export/import or new -project is needed. Return to standard RocksDB with the same stop/up sequence and -its matching unsuffixed tag. Prefer versioned tags or verified digests: Topling +project is needed for the compatible database files. When returning to standard +RocksDB with mounted graph definitions, first set `rocksdb.provider=rocksdb` in +each `backend=rocksdb` working configuration while the Server is stopped. The +Topling entrypoint writes `rocksdb.provider=topling` into these writable files; +the standard image intentionally preserves mounted values and does not reset +them. Do not edit `backend=hstore` definitions. Then use the same stop/up sequence +and matching unsuffixed tag, retaining the configuration override above. +Prefer versioned tags or verified digests: Topling `latest-topling` is published on demand and can lag standard `latest`. Changing storage providers does not bypass HugeGraph or RocksDB version compatibility. Back up the data and qualify representative existing data before production use. The images declare only database directories as Docker volumes, so replacing an image also replaces its binaries, configuration defaults and JNI. The supplied -Compose files already mount these data directories separately. If an older +Compose files mount these database directories separately, but graph definitions +still require the configuration backup and mount described above. If an older custom deployment kept data only in an anonymous whole-application volume, first identify and preserve that volume's database directories; they are not copied automatically into a new data volume. Custom data/WAL paths must remain mounted. @@ -389,6 +425,11 @@ Each topology creates its own normal Compose network and named volumes. No netwo Standalone stores RocksDB data at `/hugegraph-server/rocksdb-data`. The HStore topologies keep PD and Store data in topology-local volumes. Hubble uses `jdbc:h2:file:/hubble/data/hubble;DB_CLOSE_ON_EXIT=FALSE` and stores uploaded files under `/hubble/data/upload-files`. +Standalone API-created graph definitions are separate files under `conf/graphs`. +Before any first replacement of an older root-volume image, including a standard +to standard upgrade, follow the [configuration backup and mount steps](#toplingdb-images). +The data volume alone does not preserve or reload those definitions. + `docker compose down` keeps named-volume data. `docker compose down -v` intentionally deletes it. diff --git a/docs/storage-lifecycle.md b/docs/storage-lifecycle.md index 3d5b99389d..b168fdbb16 100644 --- a/docs/storage-lifecycle.md +++ b/docs/storage-lifecycle.md @@ -511,8 +511,11 @@ still requires an explicit JAR and its trusted checksum. Compose uses its existing `HUGEGRAPH_VERSION=latest-topling` parameter. Helm uses `--set global.runtimeVariant=topling` to select matching images and amd64 scheduling. The HStore Server keeps `backend=hstore` and uses the Topling-enabled PD and Store; -it does not load a local Topling runtime. The images configure the business -provider and launcher together. See the [Compose guide](../docker/README.md#toplingdb-images) +it does not load a local Topling runtime. Explicit `TOPLINGDB_ROCKSDB_PROVIDER` +selection stays above retained PD-managed Store configuration without rewriting +that PD configuration. Newly created local RocksDB graphs inherit the selected +image provider only when the request omits `rocksdb.provider`; explicit graph +values and HStore backends remain unchanged. See the [Compose guide](../docker/README.md#toplingdb-images) and [Helm guide](../helm/hugegraph/README.md#toplingdb-runtime) for complete commands, source builds, custom registries and immutable image digests. @@ -535,8 +538,19 @@ from one Apache HugeGraph commit. For an older custom Docker deployment with data stored only in an anonymous volume covering the entire installation, preserve its database directories explicitly before -changing to data-only mounts. The supplied Compose data mounts already keep these -directories separate. See the linked deployment guides for the stop/switch/up commands. +changing to data-only mounts. The supplied Compose data mounts keep database +directories separate, but they do not persist `conf/graphs/*.properties`, including +API-created graph definitions. Before the first image replacement, copy those +definitions from the stopped old container and keep a separate backup plus a +writable working configuration mount on subsequent starts. Do not assume data +files alone can restore missing definitions. No automatic configuration migration +or new persistence model is implemented here. + +Topling startup also rewrites the provider in mounted local RocksDB graph files. +Before returning to a standard image, stop the Server and manually restore +`rocksdb.provider=rocksdb` in those working files. Standard images preserve mounted +provider values; selecting an unsuffixed tag alone does not undo the earlier write. +See the linked Compose guide for the complete backup/mount and switch prerequisites. ### Standalone Server setup diff --git a/helm/hugegraph/README.md b/helm/hugegraph/README.md index 31ffe9579a..72e5e6723f 100644 --- a/helm/hugegraph/README.md +++ b/helm/hugegraph/README.md @@ -132,9 +132,12 @@ The architecture selector does not inspect CPU extensions; use a compatible node A component tag such as `1.8.0` becomes `1.8.0-topling`; tags already ending in `-topling` are kept. Custom repositories and `global.imageRegistry` work as usual. -An explicit `image.digest` remains exact and takes priority over the derived tag; -supply the digest of the matching Topling deployment image. Explicit helper image -overrides also remain exact. +In `global.runtimeVariant=topling`, clear `pd.image.digest`, `store.image.digest` +and `server.image.digest`; the chart rejects these overrides because a digest +would bypass the derived `-topling` tag. Use matching version tags for this runtime. +Variant-specific immutable digest fields are not currently supported. In standard +mode an explicit digest still takes precedence over the tag. Explicit helper image +overrides remain exact. The default `global.runtimeVariant=standard` keeps RocksDB. The verified stock RocksDB 8.10.2 and pinned community ToplingDB 8.10.2 use compatible data: the diff --git a/helm/hugegraph/docs/operations.md b/helm/hugegraph/docs/operations.md index a16e2422fa..811a643916 100644 --- a/helm/hugegraph/docs/operations.md +++ b/helm/hugegraph/docs/operations.md @@ -217,8 +217,10 @@ still apply. The compatibility scope is the pinned community profile, which uses SkipList and BlockBasedTable. Other JNI versions and custom SST formats need their own -compatibility evidence. Exact image digest overrides must identify the target -runtime, since a digest takes precedence over a derived tag. See the +compatibility evidence. For `global.runtimeVariant=topling`, clear the PD, Store +and Server `image.digest` values and use matching `-topling` version tags; the +chart rejects those digest overrides instead of silently running the wrong +runtime. Standard mode retains exact digest support. See the [storage runtime guide](../../../docs/storage-lifecycle.md#storage-provider-selection-and-toplingdb) for the tested native versions and data coverage. diff --git a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java index d1669fb0a4..096eb21f17 100644 --- a/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java +++ b/hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java @@ -1454,11 +1454,23 @@ public HugeGraph createGraphLocal(String name, String configText) { PropertiesConfiguration propConfig = ConfigUtil.buildConfig(configText); HugeConfig config = new HugeConfig(propConfig); + applyImageProvider(config, System.getenv("TOPLINGDB_ROCKSDB_PROVIDER")); this.checkOptions(config); return this.createGraphLocal(config, name); } + private static void applyImageProvider(HugeConfig config, String provider) { + if (StringUtils.isEmpty(provider) || + !"rocksdb".equals(config.get(CoreOptions.BACKEND)) || + config.containsKey("rocksdb.provider")) { + return; + } + E.checkArgument("rocksdb".equals(provider) || "topling".equals(provider), + "Invalid TOPLINGDB_ROCKSDB_PROVIDER: %s", provider); + config.setProperty("rocksdb.provider", provider); + } + private HugeGraph createGraphLocal(HugeConfig config, String name) { HugeGraph graph = null; try { diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/GraphManagerConfigTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/GraphManagerConfigTest.java index 51628c0d98..e06aeda891 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/GraphManagerConfigTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/GraphManagerConfigTest.java @@ -110,6 +110,52 @@ public class GraphManagerConfigTest { + @Test + public void testDynamicRocksGraphInheritsImageProvider() { + OptionSpace.register("core", CoreOptions.instance()); + OptionSpace.register("rocksdb", RocksDBOptions.instance()); + for (String provider : new String[]{"rocksdb", "topling"}) { + PropertiesConfiguration properties = new PropertiesConfiguration(); + properties.setProperty("backend", "rocksdb"); + HugeConfig config = new HugeConfig(properties); + applyImageProvider(config, provider); + Assert.assertEquals(provider, config.get(RocksDBOptions.PROVIDER)); + } + } + + @Test + public void testDynamicProviderPreservesExplicitAndDistributedConfiguration() { + PropertiesConfiguration properties = new PropertiesConfiguration(); + properties.setProperty("backend", "rocksdb"); + properties.setProperty("rocksdb.provider", "rocksdb"); + HugeConfig explicit = new HugeConfig(properties); + applyImageProvider(explicit, "topling"); + Assert.assertEquals("rocksdb", explicit.getProperty("rocksdb.provider")); + properties.clearProperty("rocksdb.provider"); + properties.setProperty("backend", "hstore"); + HugeConfig distributed = new HugeConfig(properties); + applyImageProvider(distributed, "topling"); + Assert.assertFalse(distributed.containsKey("rocksdb.provider")); + } + + @Test + public void testDynamicProviderKeepsDefaultWithoutOptInAndRejectsInvalidSelection() { + PropertiesConfiguration properties = new PropertiesConfiguration(); + properties.setProperty("backend", "rocksdb"); + HugeConfig config = new HugeConfig(properties); + applyImageProvider(config, null); + applyImageProvider(config, ""); + Assert.assertFalse(config.containsKey("rocksdb.provider")); + Assert.assertThrows(IllegalArgumentException.class, + () -> applyImageProvider(config, "invalid")); + Assert.assertFalse(config.containsKey("rocksdb.provider")); + } + + private static void applyImageProvider(HugeConfig config, String provider) { + Whitebox.invoke(GraphManager.class, new Class[]{HugeConfig.class, String.class}, + "applyImageProvider", null, config, provider); + } + private static ExecutorService previousEventExecutor; private static ExecutorService previousTaskDbExecutor; private static ExecutorService taskDbExecutor; diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/PdConfigureListener.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/PdConfigureListener.java index 070d3c15ee..c9b7a734b8 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/PdConfigureListener.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/PdConfigureListener.java @@ -63,11 +63,14 @@ public class PdConfigureListener implements + "restart-hugegraph-store.sh"; private ConfigurableApplicationContext context; private File pdConfFile; + private String selectedProvider; // private String restartPath = workDir + fileSeparator + "lib" + fileSeparator; @Override public void onApplicationEvent(ApplicationEnvironmentPreparedEvent event) { MutablePropertySources sources = event.getEnvironment().getPropertySources(); + this.selectedProvider = event.getEnvironment().getProperty("TOPLINGDB_ROCKSDB_PROVIDER"); + this.prioritizeSelectedProvider(sources); String pdAddress = event.getEnvironment().getProperty("pdserver.address"); pdConfFile = new File(configFilePath); @@ -123,9 +126,23 @@ private void updatePdConfig(MutablePropertySources sources, KvClient client, writeYml(pdConfig); PropertiesPropertySource source = new PropertiesPropertySource("pd-config", configs); sources.addFirst(source); + this.prioritizeSelectedProvider(sources); } } + private void prioritizeSelectedProvider(MutablePropertySources sources) { + if (StringUtils.isEmpty(this.selectedProvider)) { + return; + } + if (!"rocksdb".equals(this.selectedProvider) && !"topling".equals(this.selectedProvider)) { + throw new IllegalArgumentException("Invalid TOPLINGDB_ROCKSDB_PROVIDER: " + + this.selectedProvider); + } + Properties override = new Properties(); + override.setProperty("rocksdb.provider", this.selectedProvider); + sources.addFirst(new PropertiesPropertySource("rocksdb-runtime-provider", override)); + } + private Properties getYmlConfig(String yml) { Yaml yaml = new Yaml(new SafeConstructor(new LoaderOptions())); Iterable load = yaml.loadAll(yml); diff --git a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/listener/PdConfigureListenerYamlTest.java b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/listener/PdConfigureListenerYamlTest.java index b54ce9efc0..755f5cf37b 100644 --- a/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/listener/PdConfigureListenerYamlTest.java +++ b/hugegraph-store/hg-store-node/src/test/java/org/apache/hugegraph/store/node/listener/PdConfigureListenerYamlTest.java @@ -19,24 +19,107 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.lang.reflect.Field; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.List; import java.util.Properties; +import javax.tools.ToolProvider; + +import org.apache.hugegraph.pd.client.KvClient; +import org.apache.hugegraph.pd.grpc.kv.KResponse; +import org.apache.hugegraph.util.RocksDBRuntime; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; import org.springframework.boot.SpringApplication; import org.springframework.boot.WebApplicationType; import org.springframework.boot.env.YamlPropertySourceLoader; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Configuration; import org.springframework.core.env.PropertySource; +import org.springframework.core.env.PropertiesPropertySource; +import org.springframework.core.env.StandardEnvironment; import org.springframework.core.io.ByteArrayResource; import org.springframework.core.io.FileSystemResource; import org.yaml.snakeyaml.error.YAMLException; public class PdConfigureListenerYamlTest { + @TempDir + public Path temporary; + + @Test + public void testSelectedRuntimeSurvivesRetainedPdConfiguration() throws Exception { + for (String provider : new String[]{"rocksdb", "topling"}) { + StandardEnvironment environment = new StandardEnvironment(); + Properties image = new Properties(); + image.setProperty("rocksdb.provider", provider); + environment.getPropertySources().addFirst(new PropertiesPropertySource("spring.application.json", image)); + String retained = "rocksdb:\n provider: " + + ("topling".equals(provider) ? "rocksdb" : "topling") + + "\n max_background_jobs: 8\n"; + Path file = this.temporary.resolve(provider + "-application-pd.yml"); + Files.writeString(file, retained); + PdConfigureListener listener = new PdConfigureListener(); + set(listener, "pdConfFile", file.toFile()); + set(listener, "selectedProvider", provider); + KvClient client = Mockito.mock(KvClient.class); + Mockito.when(client.get("S:Timestamp")).thenReturn( + KResponse.newBuilder().setValue(Long.toString(System.currentTimeMillis() + 60000)).build()); + Method update = PdConfigureListener.class.getDeclaredMethod( + "updatePdConfig", org.springframework.core.env.MutablePropertySources.class, + KvClient.class, String.class); + update.setAccessible(true); + update.invoke(listener, environment.getPropertySources(), client, retained); + Assertions.assertEquals(provider, environment.getProperty("rocksdb.provider")); + Assertions.assertEquals("8", environment.getProperty("rocksdb.max_background_jobs")); + Assertions.assertEquals(retained, Files.readString(file)); + verifyRuntime(provider, environment.getProperty("rocksdb.provider")); + } + } + + @Test + public void testNoRuntimeSelectionDoesNotOverrideManagedProvider() throws Exception { + StandardEnvironment environment = new StandardEnvironment(); + Properties configured = new Properties(); + configured.setProperty("rocksdb.provider", "topling"); + environment.getPropertySources().addFirst(new PropertiesPropertySource("pd-config", configured)); + Method prioritize = PdConfigureListener.class.getDeclaredMethod( + "prioritizeSelectedProvider", org.springframework.core.env.MutablePropertySources.class); + prioritize.setAccessible(true); + prioritize.invoke(new PdConfigureListener(), environment.getPropertySources()); + Assertions.assertEquals("topling", environment.getProperty("rocksdb.provider")); + Assertions.assertFalse(environment.getPropertySources().contains("rocksdb-runtime-provider")); + } + + private static void set(PdConfigureListener listener, String name, Object value) throws Exception { + Field field = PdConfigureListener.class.getDeclaredField(name); + field.setAccessible(true); + field.set(listener, value); + } + + private void verifyRuntime(String runtime, String configured) throws Exception { + Path classes = Files.createDirectory(this.temporary.resolve(runtime)); + Path source = classes.resolve("RocksDB.java"); + Files.writeString(source, "package org.rocksdb; public class RocksDB {}" + + ("topling".equals(runtime) ? " class SidePluginRepo {}" : "")); + Assertions.assertEquals(0, ToolProvider.getSystemJavaCompiler().run( + null, null, null, "-d", classes.toString(), source.toString())); + ClassLoader previous = Thread.currentThread().getContextClassLoader(); + try (URLClassLoader loader = new URLClassLoader(new URL[]{classes.toUri().toURL()}, null)) { + Thread.currentThread().setContextClassLoader(loader); + RocksDBRuntime.verify(configured); + } finally { + Thread.currentThread().setContextClassLoader(previous); + } + } + @Test public void testFlattenMapsAndMultipleDocuments() throws Exception { Properties properties = parse("rocksdb:\n max_background_jobs: 8\n" + From 4c2a5834a0b09add73e0d92645dbe257bf42d150 Mon Sep 17 00:00:00 2001 From: imbajin Date: Sun, 25 Oct 2026 15:48:47 +0800 Subject: [PATCH 15/29] fix(docker): report preserved PD provider accurately --- hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh b/hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh index 19953217f2..e8df173a78 100755 --- a/hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh +++ b/hugegraph-pd/hg-pd-dist/docker/docker-entrypoint.sh @@ -5,7 +5,7 @@ # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # @@ -157,7 +157,11 @@ JSON export SPRING_APPLICATION_JSON log "effective config:" -log " rocksdb.provider=${ROCKSDB_PROVIDER}" +if [[ -n "${TOPLINGDB_ROCKSDB_PROVIDER:-}" ]]; then + log " rocksdb.provider=${ROCKSDB_PROVIDER}" +else + log " rocksdb.provider=" +fi log " grpc.host=${HG_PD_GRPC_HOST}" log " grpc.port=${HG_PD_GRPC_PORT}" log " server.port=${HG_PD_REST_PORT}" From 8d7576c2e2d1bdbc0ac4b6ee66ff7d1925eb0fc7 Mon Sep 17 00:00:00 2001 From: imbajin Date: Sun, 25 Oct 2026 15:49:04 +0800 Subject: [PATCH 16/29] fix(docker): report preserved Store provider accurately --- hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh b/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh index c841c29768..db1f4f6ebf 100755 --- a/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh +++ b/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh @@ -5,7 +5,7 @@ # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # @@ -84,7 +84,11 @@ JSON export SPRING_APPLICATION_JSON log "effective config:" -log " rocksdb.provider=${ROCKSDB_PROVIDER}" +if [[ -n "${TOPLINGDB_ROCKSDB_PROVIDER:-}" ]]; then + log " rocksdb.provider=${ROCKSDB_PROVIDER}" +else + log " rocksdb.provider=" +fi log " pdserver.address=${HG_STORE_PD_ADDRESS}" log " grpc.host=${HG_STORE_GRPC_HOST}" log " grpc.port=${HG_STORE_GRPC_PORT}" From 4e4de30fa2e19148ae090ec1d87855ade58f2e5f Mon Sep 17 00:00:00 2001 From: imbajin Date: Sun, 25 Oct 2026 15:49:30 +0800 Subject: [PATCH 17/29] test(docker): lock provider log semantics --- docker/test-topling-images.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/test-topling-images.sh b/docker/test-topling-images.sh index f7178534c5..cbefe4c8f0 100644 --- a/docker/test-topling-images.sh +++ b/docker/test-topling-images.sh @@ -5,9 +5,9 @@ # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -62,8 +62,11 @@ with tempfile.TemporaryDirectory(prefix="topling-entrypoints-") as temporary: document = json.loads((directory / "spring.json").read_text()) if provider is None: assert "rocksdb" not in document, "default must preserve mounted provider" + assert "rocksdb.provider=" in result.stdout + assert "rocksdb.provider=rocksdb" not in result.stdout else: assert document["rocksdb"]["provider"] == provider, document + assert f"rocksdb.provider={provider}" in result.stdout (directory / "spring.json").unlink() environment["TOPLINGDB_ROCKSDB_PROVIDER"] = 'topling", "unexpected":true' result = subprocess.run(["bash", "entrypoint.sh"], cwd=directory, env=environment, From c888d1c953cbf3e18b1cb8cdb5b2ca484752fbfe Mon Sep 17 00:00:00 2001 From: imbajin Date: Sun, 25 Oct 2026 16:22:18 +0800 Subject: [PATCH 18/29] fix(docker): preserve graph definitions and ship runtime notices --- docker/bake.hcl | 5 + docker/prepare-topling-image.sh | 46 ++ docker/test-topling-images.sh | 73 ++- hugegraph-server/Dockerfile | 8 +- install-dist/release-docs/LICENSE | 20 - install-dist/release-docs/NOTICE | 19 - .../licenses/LICENSE-toplingdb-jni-8.10.2.txt | 586 ------------------ .../optional-image-dependencies.txt | 8 +- 8 files changed, 128 insertions(+), 637 deletions(-) mode change 100644 => 100755 docker/prepare-topling-image.sh mode change 100644 => 100755 docker/test-topling-images.sh delete mode 100644 install-dist/release-docs/licenses/LICENSE-toplingdb-jni-8.10.2.txt diff --git a/docker/bake.hcl b/docker/bake.hcl index fce84d6c30..346f2fd323 100644 --- a/docker/bake.hcl +++ b/docker/bake.hcl @@ -37,6 +37,11 @@ variable "SOURCE_URL" { variable "RUNTIME_VARIANT" { default = "standard" + + validation { + condition = RUNTIME_VARIANT == "standard" || RUNTIME_VARIANT == "topling" + error_message = "RUNTIME_VARIANT must be standard or topling" + } } variable "TOPLING_JNI_URL" { diff --git a/docker/prepare-topling-image.sh b/docker/prepare-topling-image.sh old mode 100644 new mode 100755 index ff2289e6a2..f3bf83fe61 --- a/docker/prepare-topling-image.sh +++ b/docker/prepare-topling-image.sh @@ -51,6 +51,52 @@ fi # behavior before installing a component-local runtime and checksum receipt. bash bin/prepare-topling.sh +# Put the optional runtime's notices next to the installed JNI instead of in +# the generic Maven/source distribution, which does not contain this artifact. +license_stage=topling/.license-stage +rm -rf "$license_stage" topling/licenses +mkdir -p "$license_stage" +unzip -qq "$TOPLING_JNI_JAR" 'META-INF/licenses/*' -d "$license_stage" +[[ -d "$license_stage/META-INF/licenses" && + -s "$license_stage/META-INF/licenses/manifest.sha256" ]] || { + echo 'Error: Topling JNI is missing embedded license materials' >&2 + exit 1 +} +mv "$license_stage/META-INF/licenses" topling/licenses +rm -rf "$license_stage" +cat > topling/LICENSE <<'EOF' +ToplingDB JNI community runtime license materials +================================================= + +The complete license texts distributed with this optional JNI runtime are in +this directory's licenses/ subdirectory. The immutable rocksdbjni.jar retains +the same META-INF/licenses/ collection and checksum manifest. + +RocksDB core uses its Apache License 2.0 option. Public Topling components and +bundled dependencies retain their respective Apache-2.0, BSD, MIT and Boost +licenses. See NOTICE for principal attributions and +META-INF/topling-build-info.txt inside rocksdbjni.jar for source revisions and +build configuration. +EOF +cat > topling/NOTICE <<'EOF' +ToplingDB JNI community runtime + +This optional runtime includes software developed by Topling, Inc., together +with code from RocksDB, LevelDB and bundled public dependencies. + +Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +Copyright (c) 2011 The LevelDB Authors. All rights reserved. +Copyright (c) 2021, Topling Inc. All rights reserved. +Copyright (c) 2020, Bytedance Inc. All rights reserved. + +Original component notices and license texts are retained in licenses/ beside +this file and in META-INF/licenses/ inside rocksdbjni.jar. Source revisions and +the selected community components are recorded in +META-INF/topling-build-info.txt inside rocksdbjni.jar. +EOF +chmod 0644 topling/LICENSE topling/NOTICE +find topling/licenses -type f -exec chmod 0644 {} + + # Community images need no proprietary factory registration. Keep the full # distribution profile unchanged and install the image's smaller native profile. cp "$(dirname "${BASH_SOURCE[0]}")/toplingdb.yaml" conf/toplingdb.yaml diff --git a/docker/test-topling-images.sh b/docker/test-topling-images.sh old mode 100644 new mode 100755 index cbefe4c8f0..c2a9a7d446 --- a/docker/test-topling-images.sh +++ b/docker/test-topling-images.sh @@ -21,13 +21,48 @@ python3 - "$repo" <<'PY' import json import os from pathlib import Path -import shutil import re +import shutil import subprocess import sys import tempfile +import zipfile repo = Path(sys.argv[1]) + +# Lock the source/build contracts that do not require a Docker daemon. +bake = (repo / "docker/bake.hcl").read_text() +assert 'condition = RUNTIME_VARIANT == "standard" || RUNTIME_VARIANT == "topling"' in bake +assert 'error_message = "RUNTIME_VARIANT must be standard or topling"' in bake +standalone = (repo / "hugegraph-server/Dockerfile").read_text() +assert "mv ./conf/graphs ./rocksdb-data/.hugegraph-config/graphs" in standalone +assert "ln -s ../rocksdb-data/.hugegraph-config/graphs ./conf/graphs" in standalone +assert "VOLUME /hugegraph-server/rocksdb-data" in standalone +for generic in (repo / "install-dist/release-docs/LICENSE", + repo / "install-dist/release-docs/NOTICE"): + assert "ToplingDB JNI" not in generic.read_text(), generic +assert not (repo / "install-dist/release-docs/licenses/LICENSE-toplingdb-jni-8.10.2.txt").exists() +inventory = (repo / "install-dist/scripts/dependency/optional-image-dependencies.txt").read_text() +for entry in ("installed_license_directory=topling/licenses/", + "installed_license_summary=topling/LICENSE", + "installed_notice=topling/NOTICE"): + assert entry in inventory, entry + +docker = shutil.which("docker") +if docker is not None: + buildx = subprocess.run([docker, "buildx", "version"], capture_output=True, text=True) + if buildx.returncode == 0: + invalid = subprocess.run( + [docker, "buildx", "bake", "-f", str(repo / "docker/bake.hcl"), "--print", "pd"], + env={**os.environ, "RUNTIME_VARIANT": "toplingdb"}, + capture_output=True, text=True) + assert invalid.returncode != 0, "Bake accepted an invalid runtime variant" + assert "RUNTIME_VARIANT must be standard or topling" in invalid.stderr + invalid.stdout + else: + print("SKIP: Docker buildx is unavailable; Bake validation checked statically") +else: + print("SKIP: Docker CLI is unavailable; Bake validation checked statically") + required = { "pd": { "HG_PD_GRPC_HOST": "pd", @@ -86,8 +121,15 @@ with tempfile.TemporaryDirectory(prefix="topling-entrypoints-") as temporary: assert "Error: Topling builds require" in result.stderr, result.stderr print("PASS: image preparation rejects missing/untrusted artifact inputs") - # Exercise the real image helper through preparation and profile installation, - # with only download/native preparation stubbed (native JNI is tested separately). + # Exercise the real image helper through preparation, notice extraction and + # profile installation, with only native preparation stubbed. + fixture_jar = Path(temporary) / "rocksdbjni.jar" + with zipfile.ZipFile(fixture_jar, "w") as jar: + jar.writestr("META-INF/licenses/LICENSE.Apache", "Apache fixture\n") + jar.writestr("META-INF/licenses/LICENSE.leveldb", "BSD fixture\n") + jar.writestr("META-INF/licenses/manifest.sha256", "fixture manifest\n") + jar.writestr("META-INF/topling-build-info.txt", "fixture build\n") + distribution = Path(temporary) / "distribution" (distribution / "bin").mkdir(parents=True) (distribution / "conf").mkdir() @@ -96,11 +138,21 @@ with tempfile.TemporaryDirectory(prefix="topling-entrypoints-") as temporary: original_bytes = original_profile.read_bytes() shutil.copy(original_profile, distribution / "conf/toplingdb.yaml") preparer = distribution / "bin/prepare-topling.sh" - preparer.write_text("#!/bin/bash\ntest -r \"$TOPLING_JNI_JAR\" && test \"${#TOPLING_JNI_SHA256}\" -eq 64\n") + preparer.write_text("#!/bin/bash\n" + "test -r \"$TOPLING_JNI_JAR\" && test \"${#TOPLING_JNI_SHA256}\" -eq 64\n" + "mkdir -p topling\n" + "cp \"$TOPLING_JNI_JAR\" topling/rocksdbjni.jar\n") downloader = distribution / "tools/curl" - downloader.write_text("#!/bin/bash\nwhile (( $# )); do if [[ $1 == --output ]]; then printf stub > \"$2\"; exit 0; fi; shift; done\nexit 1\n") + downloader.write_text("#!/bin/bash\n" + "while (( $# )); do\n" + " if [[ $1 == --output ]]; then cp \"$FIXTURE_JAR\" \"$2\"; exit 0; fi\n" + " shift\n" + "done\n" + "exit 1\n") downloader.chmod(0o755) - environment.update(TOPLING_JNI_SHA256="a" * 64, TOPLING_JNI_URL="https://example.invalid/immutable.jar") + environment.update(TOPLING_JNI_SHA256="a" * 64, + TOPLING_JNI_URL="https://example.invalid/immutable.jar", + FIXTURE_JAR=str(fixture_jar)) environment["PATH"] = str(distribution / "tools") + os.pathsep + environment["PATH"] result = subprocess.run(["bash", str(helper)], cwd=distribution, env=environment, capture_output=True, text=True) @@ -108,6 +160,13 @@ with tempfile.TemporaryDirectory(prefix="topling-entrypoints-") as temporary: community_profile = repo / "docker/toplingdb.yaml" assert (distribution / "conf/toplingdb.yaml").read_bytes() == community_profile.read_bytes() assert original_profile.read_bytes() == original_bytes, "distribution source profile changed" + assert (distribution / "topling/LICENSE").is_file() + assert (distribution / "topling/NOTICE").is_file() + assert (distribution / "topling/licenses/LICENSE.Apache").read_text() == "Apache fixture\n" + assert (distribution / "topling/licenses/LICENSE.leveldb").read_text() == "BSD fixture\n" + assert (distribution / "topling/licenses/manifest.sha256").is_file() + assert "ToplingDB JNI community runtime" in (distribution / "topling/NOTICE").read_text() + profile = community_profile.read_text() # This profile contains only indentation-based mappings. Reject duplicate # paths instead of silently letting the YAML reader discard an earlier key. @@ -130,5 +189,5 @@ with tempfile.TemporaryDirectory(prefix="topling-entrypoints-") as temporary: for setting in ("capacity: 64M", "write_buffer_size: 16M", "max_write_buffer_number: 3", "memtable_as_log_index: false", "auto_start_http: false"): assert setting in profile, setting - print("PASS: community image installs small native profile and preserves full distribution profile") + print("PASS: community image installs notices, small native profile and preserves full distribution profile") PY diff --git a/hugegraph-server/Dockerfile b/hugegraph-server/Dockerfile index cefff239dd..0fff523a6a 100644 --- a/hugegraph-server/Dockerfile +++ b/hugegraph-server/Dockerfile @@ -105,7 +105,10 @@ RUN --mount=type=secret,id=proxy_ca,required=false \ && rm -rf /var/lib/apt/lists/* COPY --from=build /pkg/hugegraph-server/apache-hugegraph-server-*/ /hugegraph-server/ -RUN sed -i "s/^restserver.url.*$/restserver.url=http:\/\/0.0.0.0:8080/g" ./conf/rest-server.properties +RUN sed -i "s/^restserver.url.*$/restserver.url=http:\/\/0.0.0.0:8080/g" ./conf/rest-server.properties \ + && mkdir -p ./rocksdb-data/.hugegraph-config \ + && mv ./conf/graphs ./rocksdb-data/.hugegraph-config/graphs \ + && ln -s ../rocksdb-data/.hugegraph-config/graphs ./conf/graphs # 2. Init docker script COPY hugegraph-server/hugegraph-dist/docker/scripts/remote-connect.groovy ./scripts @@ -127,7 +130,8 @@ RUN --mount=type=secret,id=proxy_ca,required=false \ && rm /tmp/prepare-topling-image.sh /tmp/topling-runtime.env /tmp/toplingdb.yaml EXPOSE 8080 -# The default data and WAL share this directory. Keep lib/bin/conf image-owned. +# The named data volume seeds and retains graph definitions through conf/graphs, +# while lib/bin and the rest of conf stay image-owned across replacement. VOLUME /hugegraph-server/rocksdb-data HEALTHCHECK --interval=15s --timeout=10s --start-period=90s --retries=3 \ diff --git a/install-dist/release-docs/LICENSE b/install-dist/release-docs/LICENSE index c92dea893d..6e27cb1e2e 100644 --- a/install-dist/release-docs/LICENSE +++ b/install-dist/release-docs/LICENSE @@ -907,23 +907,3 @@ The text of each license is also included in licenses/LICENSE-[project].txt. # See licenses/LICENSE-jansi-2.4.0.txt for license details (Apache License 2.0 or LGPL 3.0+). Ivy 2.6.0 includes third-party BSD and MIT licensed code; see licenses/LICENSE-ivy-2.6.0.txt. - -======================================================================== -Optional ToplingDB JNI runtime for container images -======================================================================== -The explicit Topling container variant may additionally include the community -JNI runtime from https://github.com/hugegraph/toplingdb. It is installed as -topling/rocksdbjni.jar, outside the standard runtime lib directory. Standard -RocksDB distributions and images do not include this optional artifact. - -The RocksDB core uses its Apache 2.0 licensing option. Public Topling components -and their bundled dependencies retain their Apache 2.0, BSD, MIT and Boost -licenses. See licenses/LICENSE-toplingdb-jni-8.10.2.txt and the original license -texts under META-INF/licenses/ in the JNI JAR. Its embedded -META-INF/topling-build-info.txt identifies the source revisions and enabled -components. The community profile disables the private ToplingZipTable builder, -CSPP MemTable and distributed compaction components. - -The exact artifact URL and checksum are recorded in docker/topling-runtime.env -in the source repository. The optional image dependency inventory is -install-dist/scripts/dependency/optional-image-dependencies.txt. diff --git a/install-dist/release-docs/NOTICE b/install-dist/release-docs/NOTICE index 31abcef127..2f6fa03e5e 100644 --- a/install-dist/release-docs/NOTICE +++ b/install-dist/release-docs/NOTICE @@ -3328,22 +3328,3 @@ Arthas 4.1.5 embedded Logback 1.2.13 NOTICE Logback: the reliable, generic, fast and flexible logging framework. Copyright (C) 1999-2015, QOS.ch. All rights reserved. - -======================================================================== - -ToplingDB JNI community runtime (optional container images) - -======================================================================== - -Topling container images additionally include software developed by Topling, -Inc., with code from RocksDB, LevelDB and the bundled public dependencies. - -Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -Copyright (c) 2011 The LevelDB Authors. All rights reserved. -Copyright (c) 2021, Topling Inc. All rights reserved. -Copyright (c) 2020, Bytedance Inc. All rights reserved. - -The original component copyright notices and license texts are retained in -META-INF/licenses/ inside topling/rocksdbjni.jar. Source revisions and the -selected community components are recorded in META-INF/topling-build-info.txt. -See licenses/LICENSE-toplingdb-jni-8.10.2.txt for the licensing summary and texts. diff --git a/install-dist/release-docs/licenses/LICENSE-toplingdb-jni-8.10.2.txt b/install-dist/release-docs/licenses/LICENSE-toplingdb-jni-8.10.2.txt deleted file mode 100644 index 98af2a2844..0000000000 --- a/install-dist/release-docs/licenses/LICENSE-toplingdb-jni-8.10.2.txt +++ /dev/null @@ -1,586 +0,0 @@ -Optional ToplingDB JNI 8.10.2 community runtime -============================================= - -Source: https://github.com/hugegraph/toplingdb - -This component is included only in explicitly selected Topling container images. -The downloaded rocksdbjni-8.10.2-linux64.jar is prepared as -/topling/rocksdbjni.jar, outside the standard runtime lib directory. -The standard RocksDB images and source distributions do not bundle this JNI. - -The community build disables WITH_TOPLING_ROCKS, WITH_CSPP_MEMTABLE and -WITH_TOPLING_DCOMPACT. Its public SidePlugin components include rockside, -cspp-wbwi, topling-sst and topling-zip_table_reader under Apache License 2.0, -and topling-zip under its BSD license. The RocksDB core's Apache License 2.0 -option is selected from its upstream GPLv2 / Apache 2.0 dual license. - -The JNI JAR retains the original source license files and copyright notices -under META-INF/licenses/, with checksums in META-INF/licenses/manifest.sha256. -META-INF/topling-build-info.txt records the exact source/plugin revisions and -build configuration. These per-build records accompany the immutable artifact; -see docker/topling-runtime.env in the source repository for its URL and SHA-256. -The optional artifact and installed aliases are recorded separately in -install-dist/scripts/dependency/optional-image-dependencies.txt. - -The following texts preserve the principal upstream licenses and embedded -third-party notices. The complete per-build collection remains in the JNI JAR. - -======================================================================== -RocksDB core and Apache-licensed public SidePlugin components -Source file: LICENSE.Apache -======================================================================== - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -======================================================================== -LevelDB-derived code -Source file: LICENSE.leveldb -======================================================================== - -This contains code that is from LevelDB, and that code is under the following license: - -Copyright (c) 2011 The LevelDB Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -======================================================================== -Topling-zip -Source file: sideplugin/topling-zip/LICENSE -======================================================================== - -Copyright (c) 2021, Topling Inc. -All rights reserved. - -Copyright (c) 2020, Bytedance Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -======================================================================== -rapidyaml and c4core -Source file: sideplugin/rockside/3rdparty/rapidyaml/LICENSE.txt -======================================================================== - -Copyright (c) 2018, Joao Paulo Magalhaes - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the "Software"), -to deal in the Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, sublicense, -and/or sell copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included -in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -======================================================================== -CivetWeb -Source file: sideplugin/rockside/src/topling/web/civetweb.c -======================================================================== - -/* Copyright (c) 2013-2021 the Civetweb developers - * Copyright (c) 2004-2013 Sergey Lyubka - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -======================================================================== -JSON for Modern C++ -Source file: sideplugin/topling-zip/src/terark/json.hpp -======================================================================== - -/* - __ _____ _____ _____ - __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 3.8.0 -|_____|_____|_____|_|___| https://github.com/topling/json - -Licensed under the MIT License . -SPDX-License-Identifier: MIT -Copyright (c) 2013-2019 Niels Lohmann . - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -======================================================================== -libbase64 -Source file: sideplugin/topling-zip/3rdparty/base64/LICENSE -======================================================================== - -Copyright (c) 2005-2007, Nick Galbreath -Copyright (c) 2013-2017, Alfred Klomp -Copyright (c) 2015-2017, Wojciech Mula -Copyright (c) 2016-2017, Matthieu Darbois -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -- Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - -- Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS -IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED -TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -======================================================================== -libdivsufsort-lite (vendored by Zstandard) -Source file: sideplugin/topling-zip/3rdparty/zstd/zstd/dictBuilder/divsufsort.c -======================================================================== - -/* - * divsufsort.c for libdivsufsort-lite - * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -======================================================================== -debugbreak (c4core) -Source file: sideplugin/rockside/3rdparty/rapidyaml/ext/c4core/src/c4/ext/debugbreak/COPYING -======================================================================== - -Copyright (c) 2011-2016, Scott Tsai - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -======================================================================== -Boost -Source file: sideplugin/topling-zip/boost-include/LICENSE_1_0.txt -======================================================================== - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - -======================================================================== -Zstandard 1.4.5 (BSD licensing option) -Source file: .github/licenses/zstd-1.4.5-LICENSE -======================================================================== - -BSD License - -For Zstandard software - -Copyright (c) 2016-present, Facebook, Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * Neither the name Facebook nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -======================================================================== -xxHash (vendored by legacy Zstandard) -Source file: sideplugin/topling-zip/3rdparty/zstd/zstd/legacy/zstd_v01.c -======================================================================== - -/* - zstd - standard compression library - Copyright (C) 2014-2015, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd - - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c -*/ diff --git a/install-dist/scripts/dependency/optional-image-dependencies.txt b/install-dist/scripts/dependency/optional-image-dependencies.txt index ccba3ece81..5ccf58cb8b 100644 --- a/install-dist/scripts/dependency/optional-image-dependencies.txt +++ b/install-dist/scripts/dependency/optional-image-dependencies.txt @@ -11,7 +11,9 @@ installed_server=/hugegraph-server/topling/rocksdbjni.jar installed_pd=/hugegraph-pd/topling/rocksdbjni.jar installed_store=/hugegraph-store/topling/rocksdbjni.jar provenance=META-INF/topling-build-info.txt -licenses=META-INF/licenses/ -license_checksums=META-INF/licenses/manifest.sha256 +embedded_licenses=META-INF/licenses/ +embedded_license_checksums=META-INF/licenses/manifest.sha256 +installed_license_directory=topling/licenses/ +installed_license_summary=topling/LICENSE +installed_notice=topling/NOTICE build_flags=WITH_TOPLING_ROCKS=0 WITH_CSPP_MEMTABLE=0 WITH_TOPLING_DCOMPACT=0 -license_summary=install-dist/release-docs/licenses/LICENSE-toplingdb-jni-8.10.2.txt From 9e75b16d1b6e1f6ec2e7df7bf0ebda8aa7f8dd98 Mon Sep 17 00:00:00 2001 From: imbajin Date: Sun, 25 Oct 2026 16:29:47 +0800 Subject: [PATCH 19/29] test(docker): verify graph definitions survive recreate --- .github/scripts/docker-deployment.py | 29 +++++++++++++++ .github/scripts/test_docker_deployment.py | 43 +++++++++++++++++++++++ 2 files changed, 72 insertions(+) mode change 100644 => 100755 .github/scripts/docker-deployment.py mode change 100644 => 100755 .github/scripts/test_docker_deployment.py diff --git a/.github/scripts/docker-deployment.py b/.github/scripts/docker-deployment.py old mode 100644 new mode 100755 index 12b253ec2f..da68b726a8 --- a/.github/scripts/docker-deployment.py +++ b/.github/scripts/docker-deployment.py @@ -29,6 +29,8 @@ ADMIN_PASSWORD = "ci-compose-password" PD_PASSWORD = "ci-compose-pd-secret" +GRAPH_CONFIG_LINK = "../rocksdb-data/.hugegraph-config/graphs" +GRAPH_CONFIG_PROBE = "/hugegraph-server/conf/graphs/.compose-recreate-probe" def command(*args): @@ -124,6 +126,28 @@ def verify_graph(run_id): check=True, timeout=960, env=environment) +def verify_standalone_recreate(compose, container, image, expected): + # GraphManager writes dynamic definitions through conf/graphs. The image + # redirects that path into the existing data volume; prove a real Compose + # replacement keeps the path and does not merely pass a static Dockerfile check. + script = (f'test "$(readlink /hugegraph-server/conf/graphs)" = "{GRAPH_CONFIG_LINK}"\n' + f'printf persisted > {GRAPH_CONFIG_PROBE}') + command("docker", "exec", container, "sh", "-ceu", script) + subprocess.run(compose + ["up", "-d", "--pull", "never", "--force-recreate", + "--no-deps", "--wait", "--wait-timeout", "600", "server"], + check=True, timeout=660) + replacement = command(*compose, "ps", "-q", "server") + if not replacement or "\n" in replacement or replacement == container: + raise RuntimeError("Compose did not replace the standalone Server container") + verify_image(replacement, image, expected) + health = command("docker", "inspect", "--format", "{{.State.Health.Status}}", replacement) + if health != "healthy": + raise RuntimeError(f"Replacement Server is {health}") + if command("docker", "exec", replacement, "cat", GRAPH_CONFIG_PROBE) != "persisted": + raise RuntimeError("conf/graphs did not survive the standalone container replacement") + verify_server() + + def smoke(tag, topology, images): root = Path(__file__).resolve().parents[2] project = f"hg-pr-{tag}-{Path(topology).stem}" @@ -132,6 +156,7 @@ def smoke(tag, topology, images): expected = {service: command("docker", "image", "inspect", "--format", "{{.Id}}", f"{repository}:{tag}") for service, repository in images.items()} failed = False + containers = {} try: subprocess.run(compose + ["up", "-d", "--pull", "never", "--wait", "--wait-timeout", "600"] + list(images), check=True, timeout=660) @@ -139,6 +164,7 @@ def smoke(tag, topology, images): container = command(*compose, "ps", "-q", service) if not container or "\n" in container: raise RuntimeError(f"Expected exactly one {service} container") + containers[service] = container verify_image(container, f"{repository}:{tag}", expected[service]) health = command("docker", "inspect", "--format", "{{.State.Health.Status}}", container) if health != "healthy": @@ -147,6 +173,9 @@ def smoke(tag, topology, images): if "pd" in images: verify_storage() verify_graph(project.replace("-", "_")) + if topology == "docker-compose.yml": + verify_standalone_recreate(compose, containers["server"], + f"{images['server']}:{tag}", expected["server"]) except BaseException: failed = True for args in (["ps"], ["logs", "--no-color", "--tail", "200"]): diff --git a/.github/scripts/test_docker_deployment.py b/.github/scripts/test_docker_deployment.py old mode 100644 new mode 100755 index 7eeca7ff8e..b23d093d50 --- a/.github/scripts/test_docker_deployment.py +++ b/.github/scripts/test_docker_deployment.py @@ -68,6 +68,22 @@ def test_startup_timeout_preserves_failure_and_cleans_compose(self): deployment.smoke("ci-1-1", "docker-compose.yml", {"server": "hugegraph/hugegraph"}) self.assertEqual(run.call_args_list[-1].args[0][-3:], ["down", "-v", "--remove-orphans"]) + def test_standalone_recreates_after_graph_checks(self): + with patch.object(deployment, "command", + side_effect=["sha256:server", "server", "sha256:server", "healthy"]), \ + patch.object(deployment.subprocess, "run") as run, \ + patch.object(deployment, "verify_server") as server, \ + patch.object(deployment, "verify_graph") as graph, \ + patch.object(deployment, "verify_standalone_recreate") as recreate: + deployment.smoke("ci-1-1", "docker-compose.yml", {"server": "hugegraph/hugegraph"}) + args = recreate.call_args.args + self.assertEqual(args[1:], ("server", "hugegraph/hugegraph:ci-1-1", "sha256:server")) + self.assertEqual(args[0][:4], ["docker", "compose", "-p", + "hg-pr-ci-1-1-docker-compose"]) + server.assert_called_once_with() + graph.assert_called_once_with("hg_pr_ci_1_1_docker_compose") + self.assertEqual(run.call_args_list[-1].args[0][-3:], ["down", "-v", "--remove-orphans"]) + def test_hstore_starts_only_pr_services_and_cleans_after_checks(self): images = {"pd": "hugegraph/pd", "store": "hugegraph/store", "server": "hugegraph/server"} identities = ["sha256:pd", "sha256:store", "sha256:server"] @@ -87,6 +103,33 @@ def test_hstore_starts_only_pr_services_and_cleans_after_checks(self): self.assertEqual(run.call_args_list[-1].args[0][-3:], ["down", "-v", "--remove-orphans"]) +class GraphDefinitionPersistenceTest(unittest.TestCase): + def test_force_recreate_keeps_graph_config_in_data_volume(self): + compose = ["docker", "compose", "-p", "fixture", "-f", "/repo/docker/docker-compose.yml"] + with patch.object(deployment, "command", + side_effect=["", "replacement", "healthy", "persisted"]) as command, \ + patch.object(deployment.subprocess, "run") as run, \ + patch.object(deployment, "verify_image") as image, \ + patch.object(deployment, "verify_server") as server: + deployment.verify_standalone_recreate( + compose, "original", "hugegraph/hugegraph:ci-1-1", "sha256:pr") + + recreate = run.call_args.args[0] + self.assertEqual(recreate[:len(compose)], compose) + for option in ("--force-recreate", "--no-deps", "--wait"): + self.assertIn(option, recreate) + self.assertEqual(recreate[-1], "server") + image.assert_called_once_with("replacement", "hugegraph/hugegraph:ci-1-1", "sha256:pr") + server.assert_called_once_with() + write = command.call_args_list[0].args + self.assertEqual(write[:4], ("docker", "exec", "original", "sh")) + self.assertIn(deployment.GRAPH_CONFIG_LINK, write[-1]) + self.assertIn(deployment.GRAPH_CONFIG_PROBE, write[-1]) + self.assertEqual(command.call_args_list[-1].args, + ("docker", "exec", "replacement", "cat", + deployment.GRAPH_CONFIG_PROBE)) + + class GraphSmokeTest(unittest.TestCase): def test_reuses_graph_write_read_and_gremlin_checks_with_auth(self): with patch.dict(os.environ, {"PATH": "/existing/tools"}), \ From 4931d8a00cc3351b4484deacab968fbd4d09668a Mon Sep 17 00:00:00 2001 From: orion Date: Sun, 25 Oct 2026 10:04:05 +0000 Subject: [PATCH 20/29] fix(docker): preserve graph configuration across runtime upgrades --- .github/scripts/check-docker-images.sh | 6 +++ .github/scripts/docker-deployment.py | 54 +++++++++++++++---- .github/scripts/test_docker_deployment.py | 28 ++++++---- .github/workflows/docker-build-ci.yml | 12 +++-- docker/README.md | 44 ++++++++------- docker/test-topling-images.sh | 1 + docs/storage-lifecycle.md | 28 +++++----- hugegraph-pd/Dockerfile | 1 + hugegraph-server/Dockerfile | 2 + .../docker/docker-entrypoint-test.sh | 50 ++++++++++++++++- .../docker/docker-entrypoint.sh | 26 +++++++++ hugegraph-store/Dockerfile | 1 + 12 files changed, 198 insertions(+), 55 deletions(-) diff --git a/.github/scripts/check-docker-images.sh b/.github/scripts/check-docker-images.sh index c135e7cf60..446ba99d45 100644 --- a/.github/scripts/check-docker-images.sh +++ b/.github/scripts/check-docker-images.sh @@ -51,6 +51,12 @@ for pair in \ echo "ERROR: $dockerfile declares volumes [$ACTUAL_VOLUMES]; expected [$EXPECTED_VOLUME]" exit 1 } + if [[ "$dockerfile" != hugegraph-server/Dockerfile-hstore ]]; then + expected_provider=rocksdb + if [[ "${RUNTIME_VARIANT:-standard}" == topling ]]; then expected_provider=topling; fi + docker image inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$IMAGE_ID" | + grep -qx "TOPLINGDB_ROCKSDB_PROVIDER=$expected_provider" + fi # Helm reuses these images for tool containers instead of pulling curlimages/curl. # Check the same non-root/read-only tool environment as the Server test hook. diff --git a/.github/scripts/docker-deployment.py b/.github/scripts/docker-deployment.py index da68b726a8..15e87ea171 100755 --- a/.github/scripts/docker-deployment.py +++ b/.github/scripts/docker-deployment.py @@ -23,6 +23,7 @@ import subprocess import sys import urllib.error +import urllib.parse import urllib.request import xml.etree.ElementTree as ET @@ -30,7 +31,8 @@ ADMIN_PASSWORD = "ci-compose-password" PD_PASSWORD = "ci-compose-pd-secret" GRAPH_CONFIG_LINK = "../rocksdb-data/.hugegraph-config/graphs" -GRAPH_CONFIG_PROBE = "/hugegraph-server/conf/graphs/.compose-recreate-probe" +PERSISTED_GRAPH = "ci_persisted" +PERSISTED_VERTEX = "retained-vertex" def command(*args): @@ -43,8 +45,11 @@ def verify_image(container, image, expected): raise RuntimeError(f"Container {container} uses {actual}; expected {image} ({expected})") -def response(url, credentials=None): - request = urllib.request.Request(url) +def response(url, credentials=None, *, method=None, payload=None): + data = json.dumps(payload).encode() if payload is not None else None + request = urllib.request.Request(url, data=data, method=method) + if data is not None: + request.add_header("Content-Type", "application/json") if credentials: encoded = base64.b64encode(credentials.encode()).decode() request.add_header("Authorization", "Basic " + encoded) @@ -55,8 +60,8 @@ def response(url, credentials=None): return error.code, error.read().decode() -def expect_response(url, expected=200, credentials=None): - actual, body = response(url, credentials) +def expect_response(url, expected=200, credentials=None, **request_options): + actual, body = response(url, credentials, **request_options) if actual != expected: raise RuntimeError(f"{url}: HTTP {actual}, expected {expected}") return body @@ -126,13 +131,43 @@ def verify_graph(run_id): check=True, timeout=960, env=environment) +def create_persisted_graph(): + url = f"http://localhost:8080/graphspaces/DEFAULT/graphs/{PERSISTED_GRAPH}" + credentials = f"admin:{ADMIN_PASSWORD}" + expect_response(url, 201, credentials, method="POST", payload={ + "backend": "rocksdb", + "rocksdb.data_path": "/hugegraph-server/rocksdb-data/ci-persisted/data", + "rocksdb.wal_path": "/hugegraph-server/rocksdb-data/ci-persisted/wal", + }) + expect_response(url + "/schema/propertykeys", 202, credentials, method="POST", + payload={"name": "name", "data_type": "TEXT", "cardinality": "SINGLE"}) + expect_response(url + "/schema/vertexlabels", 201, credentials, method="POST", + payload={"name": "retained", "id_strategy": "CUSTOMIZE_STRING", + "properties": ["name"]}) + expect_response(url + "/graph/vertices", 201, credentials, method="POST", + payload={"id": PERSISTED_VERTEX, "label": "retained", + "properties": {"name": "survives-recreate"}}) + verify_persisted_graph() + + +def verify_persisted_graph(): + url = f"http://localhost:8080/graphspaces/DEFAULT/graphs/{PERSISTED_GRAPH}" + credentials = f"admin:{ADMIN_PASSWORD}" + vertex_id = urllib.parse.quote(json.dumps(PERSISTED_VERTEX), safe="") + vertex = json.loads(expect_response(url + "/graph/vertices/" + vertex_id, + credentials=credentials)) + if vertex.get("properties", {}).get("name") != "survives-recreate": + raise RuntimeError("API-created graph data did not survive container replacement") + + def verify_standalone_recreate(compose, container, image, expected): # GraphManager writes dynamic definitions through conf/graphs. The image # redirects that path into the existing data volume; prove a real Compose # replacement keeps the path and does not merely pass a static Dockerfile check. - script = (f'test "$(readlink /hugegraph-server/conf/graphs)" = "{GRAPH_CONFIG_LINK}"\n' - f'printf persisted > {GRAPH_CONFIG_PROBE}') - command("docker", "exec", container, "sh", "-ceu", script) + link = command("docker", "exec", container, "readlink", "/hugegraph-server/conf/graphs") + if link != GRAPH_CONFIG_LINK: + raise RuntimeError(f"Graph configuration does not use the data volume: {link}") + create_persisted_graph() subprocess.run(compose + ["up", "-d", "--pull", "never", "--force-recreate", "--no-deps", "--wait", "--wait-timeout", "600", "server"], check=True, timeout=660) @@ -143,9 +178,8 @@ def verify_standalone_recreate(compose, container, image, expected): health = command("docker", "inspect", "--format", "{{.State.Health.Status}}", replacement) if health != "healthy": raise RuntimeError(f"Replacement Server is {health}") - if command("docker", "exec", replacement, "cat", GRAPH_CONFIG_PROBE) != "persisted": - raise RuntimeError("conf/graphs did not survive the standalone container replacement") verify_server() + verify_persisted_graph() def smoke(tag, topology, images): diff --git a/.github/scripts/test_docker_deployment.py b/.github/scripts/test_docker_deployment.py index b23d093d50..33926041f7 100755 --- a/.github/scripts/test_docker_deployment.py +++ b/.github/scripts/test_docker_deployment.py @@ -107,10 +107,12 @@ class GraphDefinitionPersistenceTest(unittest.TestCase): def test_force_recreate_keeps_graph_config_in_data_volume(self): compose = ["docker", "compose", "-p", "fixture", "-f", "/repo/docker/docker-compose.yml"] with patch.object(deployment, "command", - side_effect=["", "replacement", "healthy", "persisted"]) as command, \ + side_effect=[deployment.GRAPH_CONFIG_LINK, "replacement", "healthy"]) as command, \ patch.object(deployment.subprocess, "run") as run, \ patch.object(deployment, "verify_image") as image, \ - patch.object(deployment, "verify_server") as server: + patch.object(deployment, "verify_server") as server, \ + patch.object(deployment, "create_persisted_graph") as create, \ + patch.object(deployment, "verify_persisted_graph") as verify: deployment.verify_standalone_recreate( compose, "original", "hugegraph/hugegraph:ci-1-1", "sha256:pr") @@ -121,13 +123,21 @@ def test_force_recreate_keeps_graph_config_in_data_volume(self): self.assertEqual(recreate[-1], "server") image.assert_called_once_with("replacement", "hugegraph/hugegraph:ci-1-1", "sha256:pr") server.assert_called_once_with() - write = command.call_args_list[0].args - self.assertEqual(write[:4], ("docker", "exec", "original", "sh")) - self.assertIn(deployment.GRAPH_CONFIG_LINK, write[-1]) - self.assertIn(deployment.GRAPH_CONFIG_PROBE, write[-1]) - self.assertEqual(command.call_args_list[-1].args, - ("docker", "exec", "replacement", "cat", - deployment.GRAPH_CONFIG_PROBE)) + create.assert_called_once_with() + verify.assert_called_once_with() + self.assertEqual(command.call_args_list[0].args, + ("docker", "exec", "original", "readlink", + "/hugegraph-server/conf/graphs")) + + def test_rejects_missing_graph_after_recreate(self): + with patch.object(deployment, "response", return_value=(404, "missing graph")): + with self.assertRaisesRegex(RuntimeError, "expected 200"): + deployment.verify_persisted_graph() + + def test_rejects_missing_vertex_even_when_graph_exists(self): + with patch.object(deployment, "response", return_value=(200, '{"properties":{}}')): + with self.assertRaisesRegex(RuntimeError, "did not survive"): + deployment.verify_persisted_graph() class GraphSmokeTest(unittest.TestCase): diff --git a/.github/workflows/docker-build-ci.yml b/.github/workflows/docker-build-ci.yml index 5ed8fd5acc..d5d393445e 100644 --- a/.github/workflows/docker-build-ci.yml +++ b/.github/workflows/docker-build-ci.yml @@ -30,9 +30,14 @@ jobs: docker-build: runs-on: ubuntu-24.04 timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + runtime: [standard, topling] env: - IMAGE_TAG: ci-${{ github.run_id }}-${{ github.run_attempt }} + IMAGE_TAG: ci-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.runtime }} SOURCE_REVISION: ${{ github.sha }} + RUNTIME_VARIANT: ${{ matrix.runtime }} steps: - name: Checkout id: checkout @@ -89,7 +94,7 @@ jobs: set -euo pipefail docker buildx bake --file docker/bake.hcl --print > /tmp/hugegraph-bake.json - jq -e ' + jq -e --arg runtime "$RUNTIME_VARIANT" ' .group.default.targets == [ "build-cache", "pd", @@ -98,7 +103,8 @@ jobs: "store" ] and ([.target[] | .platforms] | - all(. == ["linux/amd64", "linux/arm64"])) and + all(. == (if $runtime == "topling" then ["linux/amd64"] + else ["linux/amd64", "linux/arm64"] end))) and .target["build-cache"].target == "build" and .target["build-cache"].output[0].type == "cacheonly" and ([.target | to_entries[] | diff --git a/docker/README.md b/docker/README.md index 23d0273e38..e345e185b0 100644 --- a/docker/README.md +++ b/docker/README.md @@ -314,11 +314,18 @@ checks schema, indexes, graph operations and restart persistence. See [Topling runtime preparation and validation](../docs/storage-lifecycle.md#storage-provider-selection-and-toplingdb) for the native configuration and tested compatibility boundary. -For an existing standalone deployment, first preserve **graph definitions as well -as database files**. API-created graphs write `conf/graphs/*.properties`; the -supplied named `server-data` volume does not persist that directory. Before the -first image replacement, stop incoming work and the old Server, then copy its -definitions out while the old container still exists: +Standalone images retain graph definitions in +`rocksdb-data/.hugegraph-config/graphs`, reached through the `conf/graphs` symlink. +The supplied `server-data` volume therefore keeps API-created graph definitions +as well as database files through container replacement. If an older nonempty +volume or a bind mount lacks this directory, the entrypoint initializes it once +from the image's `conf/graphs.default`. Existing definitions and database files +are preserved; explicitly mounted `conf/graphs` directories take precedence. + +For the first upgrade from an older image, preserve **custom graph definitions +and settings** that existed only in its installation directory. Default seeding +does not recover those files from the database. Stop the old Server and copy its +definitions while that container still exists: ```bash docker compose -f docker-compose.yml stop server @@ -338,10 +345,11 @@ services: Apply this override alongside the normal Compose file when starting the new image. Verify all expected definitions and their data/WAL paths before replacement. -Custom `graphs` directories need equivalent backup and mounts. Database directories -alone cannot rediscover missing graph definitions. This is a manual prerequisite; -the image does not migrate old anonymous volumes or automatically persist future -API-created graph definitions outside the mounted configuration directory. +Custom `graphs` directories need equivalent backup and mounts. This one-time step +preserves older configuration files that are outside the data volume; it does not +migrate old anonymous volumes. Subsequent API-created definitions persist in the +selected configuration directory, either the default data-volume location above +or this explicit mount. Then stop incoming work, stop the services normally, and select the matching Topling images. Keep the same Compose project, credentials and @@ -355,13 +363,12 @@ HUGEGRAPH_VERSION='-topling' \ Use `docker-compose-hstore.yml` for the corresponding distributed deployment. The named data volumes are reattached automatically; no export/import or new -project is needed for the compatible database files. When returning to standard -RocksDB with mounted graph definitions, first set `rocksdb.provider=rocksdb` in -each `backend=rocksdb` working configuration while the Server is stopped. The -Topling entrypoint writes `rocksdb.provider=topling` into these writable files; -the standard image intentionally preserves mounted values and does not reset -them. Do not edit `backend=hstore` definitions. Then use the same stop/up sequence -and matching unsuffixed tag, retaining the configuration override above. +project is needed for the compatible database files. To return to standard +RocksDB, use the same stop/up sequence and matching unsuffixed tag. Standard +images explicitly select `rocksdb`, so the entrypoint resets the provider in all +persisted or writable mounted `backend=rocksdb` graph definitions. Topling images +select `topling` in the same way. Other graph settings and `backend=hstore` +definitions remain unchanged. Retain any configuration override used above. Prefer versioned tags or verified digests: Topling `latest-topling` is published on demand and can lag standard `latest`. Changing storage providers does not bypass HugeGraph or RocksDB version compatibility. @@ -369,8 +376,9 @@ Back up the data and qualify representative existing data before production use. The images declare only database directories as Docker volumes, so replacing an image also replaces its binaries, configuration defaults and JNI. The supplied -Compose files mount these database directories separately, but graph definitions -still require the configuration backup and mount described above. If an older +Compose files mount these database directories separately and standalone graph +definitions live in that data volume. Older custom definitions require the +one-time configuration preservation described above. If an older custom deployment kept data only in an anonymous whole-application volume, first identify and preserve that volume's database directories; they are not copied automatically into a new data volume. Custom data/WAL paths must remain mounted. diff --git a/docker/test-topling-images.sh b/docker/test-topling-images.sh index c2a9a7d446..1ce9c67138 100755 --- a/docker/test-topling-images.sh +++ b/docker/test-topling-images.sh @@ -37,6 +37,7 @@ assert 'error_message = "RUNTIME_VARIANT must be standard or topling"' in bake standalone = (repo / "hugegraph-server/Dockerfile").read_text() assert "mv ./conf/graphs ./rocksdb-data/.hugegraph-config/graphs" in standalone assert "ln -s ../rocksdb-data/.hugegraph-config/graphs ./conf/graphs" in standalone +assert "cp -a ./conf/graphs ./conf/graphs.default" in standalone assert "VOLUME /hugegraph-server/rocksdb-data" in standalone for generic in (repo / "install-dist/release-docs/LICENSE", repo / "install-dist/release-docs/NOTICE"): diff --git a/docs/storage-lifecycle.md b/docs/storage-lifecycle.md index b168fdbb16..6dc08cdf5d 100644 --- a/docs/storage-lifecycle.md +++ b/docs/storage-lifecycle.md @@ -538,19 +538,21 @@ from one Apache HugeGraph commit. For an older custom Docker deployment with data stored only in an anonymous volume covering the entire installation, preserve its database directories explicitly before -changing to data-only mounts. The supplied Compose data mounts keep database -directories separate, but they do not persist `conf/graphs/*.properties`, including -API-created graph definitions. Before the first image replacement, copy those -definitions from the stopped old container and keep a separate backup plus a -writable working configuration mount on subsequent starts. Do not assume data -files alone can restore missing definitions. No automatic configuration migration -or new persistence model is implemented here. - -Topling startup also rewrites the provider in mounted local RocksDB graph files. -Before returning to a standard image, stop the Server and manually restore -`rocksdb.provider=rocksdb` in those working files. Standard images preserve mounted -provider values; selecting an unsuffixed tag alone does not undo the earlier write. -See the linked Compose guide for the complete backup/mount and switch prerequisites. +changing to data-only mounts. Standalone images now link `conf/graphs` to +`rocksdb-data/.hugegraph-config/graphs`, so the supplied Compose data volume keeps +API-created graph definitions across replacement. When an old nonempty volume or +bind mount lacks that directory, the entrypoint seeds it from `conf/graphs.default` +without replacing existing definitions or changing database files. Explicit +configuration mounts keep precedence. Custom definitions and settings that existed +only in the older installation still need one-time backup and a writable working +configuration mount before the first replacement; database files cannot recreate +those missing definitions. See the linked Compose guide for that upgrade path. + +Both image variants explicitly select their provider. The entrypoint synchronizes +`rocksdb.provider` in all persisted or writable mounted `backend=rocksdb` graph +files to that selection, including when returning to the unsuffixed standard +image. Other settings and `backend=hstore` definitions are preserved. PD/Store +apply the selected provider through their runtime configuration overrides. ### Standalone Server setup diff --git a/hugegraph-pd/Dockerfile b/hugegraph-pd/Dockerfile index 3c69d793c1..d575bda38a 100644 --- a/hugegraph-pd/Dockerfile +++ b/hugegraph-pd/Dockerfile @@ -56,6 +56,7 @@ FROM scratch AS topling-jni # 2nd stage: runtime env # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 FROM eclipse-temurin:17-jre-jammy AS standard-base +ENV TOPLINGDB_ROCKSDB_PROVIDER=rocksdb # The fork's JNI requires glibc 2.38+ and a Java 17 JDK for its capability probe. FROM eclipse-temurin:17-jdk-noble AS topling-base diff --git a/hugegraph-server/Dockerfile b/hugegraph-server/Dockerfile index 0fff523a6a..a5c75844a4 100644 --- a/hugegraph-server/Dockerfile +++ b/hugegraph-server/Dockerfile @@ -56,6 +56,7 @@ FROM scratch AS topling-jni # 2nd stage: runtime env # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 FROM eclipse-temurin:17-jdk-jammy AS standard-base +ENV TOPLINGDB_ROCKSDB_PROVIDER=rocksdb # The fork's JNI requires glibc 2.38+ and a Java 17 JDK for its capability probe. FROM eclipse-temurin:17-jdk-noble AS topling-base @@ -106,6 +107,7 @@ RUN --mount=type=secret,id=proxy_ca,required=false \ COPY --from=build /pkg/hugegraph-server/apache-hugegraph-server-*/ /hugegraph-server/ RUN sed -i "s/^restserver.url.*$/restserver.url=http:\/\/0.0.0.0:8080/g" ./conf/rest-server.properties \ + && cp -a ./conf/graphs ./conf/graphs.default \ && mkdir -p ./rocksdb-data/.hugegraph-config \ && mv ./conf/graphs ./rocksdb-data/.hugegraph-config/graphs \ && ln -s ../rocksdb-data/.hugegraph-config/graphs ./conf/graphs diff --git a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh index acaf8bb046..c738678382 100755 --- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh +++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh @@ -572,13 +572,21 @@ for graph in hugegraph analytics; do grep -qx 'rocksdb.provider=topling' "${TEST_HOME}/conf/graphs/${graph}.properties" done grep -qx 'rocksdb.provider=rocksdb' "${TEST_HOME}/conf/graphs/remote.properties" -# Default RocksDB startup retains mounted settings so business/launcher mismatch -# checks can still reject an incorrectly selected runtime instead of hiding it. +# A bare launcher with no selection retains the supplied provider. Docker's +# standard image explicitly selects rocksdb, including when returning from Topling. ( cd "${TEST_HOME}" env -u TOPLINGDB_ROCKSDB_PROVIDER bash ./docker-entrypoint.sh ) grep -qx 'rocksdb.provider=topling' "${TEST_HOME}/conf/graphs/analytics.properties" +( + cd "${TEST_HOME}" + TOPLINGDB_ROCKSDB_PROVIDER=rocksdb bash ./docker-entrypoint.sh +) +for graph in hugegraph analytics; do + grep -qx 'rocksdb.provider=rocksdb' "${TEST_HOME}/conf/graphs/${graph}.properties" +done +grep -qx 'backend=hstore' "${TEST_HOME}/conf/graphs/remote.properties" cp "${TEST_HOME}/conf/graphs/analytics.properties" "${TEST_HOME}/analytics.before-invalid-provider" if ( cd "${TEST_HOME}" @@ -589,5 +597,43 @@ if ( fi cmp "${TEST_HOME}/analytics.before-invalid-provider" "${TEST_HOME}/conf/graphs/analytics.properties" +# Reproduce the image's symlink with an already populated legacy data directory. +# The default copy must bootstrap it, retain data, and never replace later user +# definitions. An explicit conf/graphs directory must also bypass bootstrapping. +mv "${TEST_HOME}/conf/graphs" "${TEST_HOME}/conf/graphs.default" +mkdir -p "${TEST_HOME}/rocksdb-data/data" "${TEST_HOME}/rocksdb-data/wal" +printf 'legacy data\n' > "${TEST_HOME}/rocksdb-data/data/keep" +ln -s ../rocksdb-data/.hugegraph-config/graphs "${TEST_HOME}/conf/graphs" +( + cd "${TEST_HOME}" + TOPLINGDB_ROCKSDB_PROVIDER=topling bash ./docker-entrypoint.sh +) +grep -qx 'legacy data' "${TEST_HOME}/rocksdb-data/data/keep" +grep -qx 'rocksdb.provider=topling' "${TEST_HOME}/conf/graphs/hugegraph.properties" +printf 'custom=true\n' >> "${TEST_HOME}/conf/graphs/analytics.properties" +cp "${TEST_HOME}/conf/graphs/analytics.properties" "${TEST_HOME}/analytics.persisted" +printf 'unrelated image default\n' > "${TEST_HOME}/conf/graphs.default/analytics.properties" +( + cd "${TEST_HOME}" + TOPLINGDB_ROCKSDB_PROVIDER=topling bash ./docker-entrypoint.sh +) +cmp "${TEST_HOME}/analytics.persisted" "${TEST_HOME}/conf/graphs/analytics.properties" +( + cd "${TEST_HOME}" + TOPLINGDB_ROCKSDB_PROVIDER=rocksdb bash ./docker-entrypoint.sh +) +grep -qx 'custom=true' "${TEST_HOME}/conf/graphs/analytics.properties" +grep -qx 'rocksdb.provider=rocksdb' "${TEST_HOME}/conf/graphs/analytics.properties" +grep -qx 'rocksdb.provider=rocksdb' "${TEST_HOME}/conf/graphs/hugegraph.properties" +[[ -z "$(find "${TEST_HOME}/rocksdb-data/.hugegraph-config" -name '.graphs-init.*' -print -quit)" ]] +unlink "${TEST_HOME}/conf/graphs" +mv "${TEST_HOME}/rocksdb-data/.hugegraph-config/graphs" "${TEST_HOME}/conf/graphs" +( + cd "${TEST_HOME}" + TOPLINGDB_ROCKSDB_PROVIDER=rocksdb bash ./docker-entrypoint.sh +) +[[ ! -e "${TEST_HOME}/rocksdb-data/.hugegraph-config/graphs" ]] +grep -qx 'custom=true' "${TEST_HOME}/conf/graphs/analytics.properties" + echo "PASS: Docker entrypoint configures HStore discovery and authentication" bash "${SCRIPT_DIR}/test/test-docker-entrypoint-process.sh" diff --git a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh index eec6f53be6..74ddf48606 100755 --- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh +++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh @@ -228,6 +228,32 @@ case "${DEPLOY_IN_K8S}" in exit 1 ;; esac +# An old nonempty volume or a bind mount hides the image's seeded graphs. +# Only initialize a missing target of the image-owned link; existing graph +# definitions and explicitly mounted conf/graphs directories remain authoritative. +prepare_graph_directory() { + local graphs=./conf/graphs defaults=./conf/graphs.default + local parent=./rocksdb-data/.hugegraph-config target staging + [[ -L "$graphs" && "$(readlink "$graphs")" == ../rocksdb-data/.hugegraph-config/graphs ]] || return 0 + [[ ! -d "$graphs" ]] || return 0 + target="$parent/graphs" + if [[ -e "$target" || -L "$target" || ! -d "$defaults" ]]; then + log "ERROR: cannot initialize graph definitions at $target from $defaults" + return 1 + fi + mkdir -p "$parent" + staging=$(mktemp -d "$parent/.graphs-init.XXXXXX") + if ! cp -a "$defaults/." "$staging/" || ! mv -T -n "$staging" "$target"; then + rm -rf -- "$staging" + log "ERROR: failed to initialize graph definitions at $target" + return 1 + fi + rm -rf -- "$staging" + [[ -d "$graphs" ]] || { log "ERROR: graph definitions target is not a directory: $target"; return 1; } + log "Initialized graph definitions in the existing data volume" +} +prepare_graph_directory + # ── Map env → properties file ───────────────────────────────────────── # TODO: drop the Helm chart's (helm/hugegraph) rest-server.properties wrapper once its minimum # Server image has the server.urls_to_pd and server.deploy_in_k8s mappings below. A read-only root diff --git a/hugegraph-store/Dockerfile b/hugegraph-store/Dockerfile index 0daf7af844..56da8ed1ad 100644 --- a/hugegraph-store/Dockerfile +++ b/hugegraph-store/Dockerfile @@ -56,6 +56,7 @@ FROM scratch AS topling-jni # 2nd stage: runtime env # Note: ZGC (The Z Garbage Collector) is only supported on ARM-Mac with java > 13 FROM eclipse-temurin:17-jdk-jammy AS standard-base +ENV TOPLINGDB_ROCKSDB_PROVIDER=rocksdb # The fork's JNI requires glibc 2.38+ and a Java 17 JDK for its capability probe. FROM eclipse-temurin:17-jdk-noble AS topling-base From 31eadd5848ac892046202809a1cb2eadd68957f9 Mon Sep 17 00:00:00 2001 From: orion Date: Sun, 25 Oct 2026 10:18:31 +0000 Subject: [PATCH 21/29] fix(docker): seed graph configuration before reading authentication --- .../docker/docker-entrypoint-test.sh | 3 +- .../docker/docker-entrypoint.sh | 53 ++++++++++--------- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh index c738678382..ead02f8791 100755 --- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh +++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh @@ -606,7 +606,8 @@ printf 'legacy data\n' > "${TEST_HOME}/rocksdb-data/data/keep" ln -s ../rocksdb-data/.hugegraph-config/graphs "${TEST_HOME}/conf/graphs" ( cd "${TEST_HOME}" - TOPLINGDB_ROCKSDB_PROVIDER=topling bash ./docker-entrypoint.sh + env -u HG_SERVER_AUTH_TOKEN_SECRET PASSWORD=entrypoint-test-secret \ + TOPLINGDB_ROCKSDB_PROVIDER=topling bash ./docker-entrypoint.sh ) grep -qx 'legacy data' "${TEST_HOME}/rocksdb-data/data/keep" grep -qx 'rocksdb.provider=topling' "${TEST_HOME}/conf/graphs/hugegraph.properties" diff --git a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh index 74ddf48606..b4153a1ffd 100755 --- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh +++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh @@ -186,32 +186,6 @@ if [[ -n "${PASSWORD:-}" && exit 1 fi -AUTH_TOKEN_SECRET_ENCODED="" -if [[ -n "${PASSWORD:-}" && -z "${HG_SERVER_AUTH_TOKEN_SECRET:-}" ]]; then - rest_secret=$(get_prop_encoded "auth.token_secret" "${REST_SERVER_CONF}") - graph_secret=$(get_prop_encoded "auth.token_secret" "${GRAPH_CONF}") - if [[ -n "${rest_secret}" ]]; then - AUTH_TOKEN_SECRET_ENCODED="${rest_secret}" - if [[ -n "${graph_secret}" && "${graph_secret}" != "${rest_secret}" ]]; then - log "WARN: authentication token secrets differ; using REST secret" - fi - elif [[ -n "${graph_secret}" ]]; then - AUTH_TOKEN_SECRET_ENCODED="${graph_secret}" - else - HG_SERVER_AUTH_TOKEN_SECRET=$(head -c 32 /dev/urandom | base64 | tr -d '\n') - log "generated a shared authentication token secret" - fi -fi - -# Encode authentication strings before changing either configuration file. -if [[ -n "${HG_SERVER_AUTH_TOKEN_SECRET:-}" ]]; then - AUTH_TOKEN_SECRET_ENCODED=$(encode_auth_prop_value "${HG_SERVER_AUTH_TOKEN_SECRET}") -fi -AUTH_ADMIN_PASSWORD_ENCODED="" -if [[ -n "${PASSWORD:-}" ]]; then - AUTH_ADMIN_PASSWORD_ENCODED=$(encode_auth_prop_value "${PASSWORD}") -fi - # Same accepted spellings as HG_SERVER_INIT_STORE_ENABLED below, checked # before the mapping so an invalid value leaves both conf files untouched. # URLs hold no whitespace, and the Server splits the list on bare commas, so @@ -254,6 +228,33 @@ prepare_graph_directory() { } prepare_graph_directory +# Read persisted authentication only after a missing graph directory is seeded. +AUTH_TOKEN_SECRET_ENCODED="" +if [[ -n "${PASSWORD:-}" && -z "${HG_SERVER_AUTH_TOKEN_SECRET:-}" ]]; then + rest_secret=$(get_prop_encoded "auth.token_secret" "${REST_SERVER_CONF}") + graph_secret=$(get_prop_encoded "auth.token_secret" "${GRAPH_CONF}") + if [[ -n "${rest_secret}" ]]; then + AUTH_TOKEN_SECRET_ENCODED="${rest_secret}" + if [[ -n "${graph_secret}" && "${graph_secret}" != "${rest_secret}" ]]; then + log "WARN: authentication token secrets differ; using REST secret" + fi + elif [[ -n "${graph_secret}" ]]; then + AUTH_TOKEN_SECRET_ENCODED="${graph_secret}" + else + HG_SERVER_AUTH_TOKEN_SECRET=$(head -c 32 /dev/urandom | base64 | tr -d '\n') + log "generated a shared authentication token secret" + fi +fi + +# Encode authentication strings before changing either configuration file. +if [[ -n "${HG_SERVER_AUTH_TOKEN_SECRET:-}" ]]; then + AUTH_TOKEN_SECRET_ENCODED=$(encode_auth_prop_value "${HG_SERVER_AUTH_TOKEN_SECRET}") +fi +AUTH_ADMIN_PASSWORD_ENCODED="" +if [[ -n "${PASSWORD:-}" ]]; then + AUTH_ADMIN_PASSWORD_ENCODED=$(encode_auth_prop_value "${PASSWORD}") +fi + # ── Map env → properties file ───────────────────────────────────────── # TODO: drop the Helm chart's (helm/hugegraph) rest-server.properties wrapper once its minimum # Server image has the server.urls_to_pd and server.deploy_in_k8s mappings below. A read-only root From 04e4c97202f8f5c1bb994e684d09899f07c30e13 Mon Sep 17 00:00:00 2001 From: orion Date: Sun, 25 Oct 2026 10:35:16 +0000 Subject: [PATCH 22/29] docs(docker): distinguish default and legacy configuration mounts --- docker/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index e345e185b0..5ecd910c60 100644 --- a/docker/README.md +++ b/docker/README.md @@ -358,9 +358,13 @@ data mounts. For standalone, from this directory: ```bash docker compose -f docker-compose.yml stop HUGEGRAPH_VERSION='-topling' \ - docker compose -f docker-compose.yml -f docker-compose.graph-config.yml up -d --wait + docker compose -f docker-compose.yml up -d --wait ``` +If you use the working configuration mount above, retain +`-f docker-compose.graph-config.yml` after the base Compose file on subsequent +commands. Deployments using the default data-volume configuration need no override. + Use `docker-compose-hstore.yml` for the corresponding distributed deployment. The named data volumes are reattached automatically; no export/import or new project is needed for the compatible database files. To return to standard From b94829af6277f524cc3359d916201ce4b6a66a05 Mon Sep 17 00:00:00 2001 From: orion Date: Sun, 25 Oct 2026 10:52:27 +0000 Subject: [PATCH 23/29] fix(docker): propagate bootstrap context when reopening graphs --- .../test/test-docker-entrypoint-process.sh | 6 +- .../org/apache/hugegraph/cmd/InitStore.java | 15 +++- .../apache/hugegraph/cmd/InitStoreTest.java | 84 +++++++++++++++++++ 3 files changed, 103 insertions(+), 2 deletions(-) diff --git a/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh b/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh index 88a7a30b2b..09c35fb27e 100644 --- a/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh +++ b/hugegraph-server/hugegraph-dist/docker/test/test-docker-entrypoint-process.sh @@ -287,7 +287,11 @@ sys.exit(int((work / 'child-exit').read_text())) os.kill(int((work / 'bin/pid').read_text()), signal.SIGKILL) else: status = 42 if mode in ['partition42', 'partitionDiagnostic42'] else int(mode[4:]) - (work / 'child-exit').write_text(str(status)) + # Publish the exit status only after its contents are + # complete; the launcher polls for the final path. + pending = work / 'child-exit.pending' + pending.write_text(str(status)) + pending.replace(work / 'child-exit') actual = process.wait(timeout=5) assert actual == expected, (mode, expected, actual, (work / 'output').read_text()) diagnostic = (work / 'output').read_text() diff --git a/hugegraph-server/hugegraph-dist/src/main/java/org/apache/hugegraph/cmd/InitStore.java b/hugegraph-server/hugegraph-dist/src/main/java/org/apache/hugegraph/cmd/InitStore.java index 5362a1821d..05a5c75627 100644 --- a/hugegraph-server/hugegraph-dist/src/main/java/org/apache/hugegraph/cmd/InitStore.java +++ b/hugegraph-server/hugegraph-dist/src/main/java/org/apache/hugegraph/cmd/InitStore.java @@ -29,6 +29,8 @@ import org.apache.hugegraph.HugeFactory; import org.apache.hugegraph.HugeGraph; +import org.apache.hugegraph.auth.HugeAuthenticator.User; +import org.apache.hugegraph.auth.HugeGraphAuthProxy; import org.apache.hugegraph.auth.StandardAuthenticator; import org.apache.hugegraph.backend.store.BackendStoreInfo; import org.apache.hugegraph.config.CoreOptions; @@ -37,6 +39,7 @@ import org.apache.hugegraph.constant.ServiceConstant; import org.apache.hugegraph.dist.RegisterUtil; import org.apache.hugegraph.meta.PdMetaDriver.PDAuthConfig; +import org.apache.hugegraph.task.TaskManager; import org.apache.hugegraph.util.ConfigUtil; import org.apache.hugegraph.util.E; import org.apache.hugegraph.util.Log; @@ -292,7 +295,7 @@ private static HugeGraph initGraph(String configPath) throws Exception { * Init the required information for creating the admin account * (when switch from non-auth mode to auth mode) */ - graph.initSystemInfo(); + initSystemInfo(graph); LOG.info("Skip init-store due to the backend store of '{}' " + "had been initialized", graph.name()); } else { @@ -309,4 +312,14 @@ private static void initBackend(final HugeGraph graph) { // Only explicitly transient backend failures can be retried safely graph.initBackend(); } + + private static void initSystemInfo(HugeGraph graph) { + // Persisted API-created graphs use the auth proxy. Their task-schema + // initialization runs on a pre-existing executor, so it must receive + // this local bootstrap command's admin context explicitly. + HugeGraphAuthProxy.runAsAdmin(() -> { + TaskManager.setContext(User.ADMIN.toJson()); + graph.initSystemInfo(); + }); + } } diff --git a/hugegraph-server/hugegraph-dist/src/test/java/org/apache/hugegraph/cmd/InitStoreTest.java b/hugegraph-server/hugegraph-dist/src/test/java/org/apache/hugegraph/cmd/InitStoreTest.java index cf11df45f9..793ed15387 100644 --- a/hugegraph-server/hugegraph-dist/src/test/java/org/apache/hugegraph/cmd/InitStoreTest.java +++ b/hugegraph-server/hugegraph-dist/src/test/java/org/apache/hugegraph/cmd/InitStoreTest.java @@ -20,9 +20,15 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Proxy; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hugegraph.HugeGraph; +import org.apache.hugegraph.auth.AuthContext; +import org.apache.hugegraph.auth.HugeAuthenticator.User; +import org.apache.hugegraph.auth.HugeGraphAuthProxy; +import org.apache.hugegraph.task.TaskManager; import org.junit.Assert; import org.junit.Test; @@ -54,4 +60,82 @@ public void testInitBackendFailsFastForPermanentException() throws Exception { } Assert.assertEquals(1, invocations.get()); } + + @Test + public void testSystemInfoBootstrapPropagatesAdminToExistingTaskExecutor() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + // Create the worker before the bootstrap command establishes a context. + executor.submit(HugeGraphAuthProxy::resetContext).get(); + AuthContext.setContext("caller-auth-context"); + TaskManager.setContext("caller-task-context"); + HugeGraphAuthProxy.setRequestGraphSpace("caller_space"); + try { + AtomicInteger invocations = new AtomicInteger(); + HugeGraph graph = initializationGraph(() -> { + try { + executor.submit(new TaskManager.ContextCallable<>(() -> { + Assert.assertEquals(User.ADMIN, + HugeGraphAuthProxy.getContext().user()); + invocations.incrementAndGet(); + return null; + })).get(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + initializeSystemInfo(graph); + Assert.assertEquals(1, invocations.get()); + Assert.assertNull(executor.submit(TaskManager::getContext).get()); + assertCallerContextRestored(); + } finally { + executor.shutdownNow(); + HugeGraphAuthProxy.resetContext(); + TaskManager.resetContext(); + } + } + + @Test + public void testSystemInfoBootstrapRestoresContextAfterFailure() throws Exception { + IllegalStateException failure = new IllegalStateException("Task schema initialization failed"); + AuthContext.setContext("caller-auth-context"); + TaskManager.setContext("caller-task-context"); + HugeGraphAuthProxy.setRequestGraphSpace("caller_space"); + try { + try { + initializeSystemInfo(initializationGraph(() -> { + throw failure; + })); + Assert.fail("Expected initialization failure"); + } catch (InvocationTargetException e) { + Assert.assertSame(failure, e.getCause()); + } + assertCallerContextRestored(); + } finally { + HugeGraphAuthProxy.resetContext(); + TaskManager.resetContext(); + } + } + + private static HugeGraph initializationGraph(Runnable initialization) { + return (HugeGraph) Proxy.newProxyInstance( + HugeGraph.class.getClassLoader(), new Class[]{HugeGraph.class}, + (proxy, method, args) -> { + if (method.getName().equals("initSystemInfo")) { + initialization.run(); + } + return null; + }); + } + + private static void initializeSystemInfo(HugeGraph graph) throws Exception { + Method method = InitStore.class.getDeclaredMethod("initSystemInfo", HugeGraph.class); + method.setAccessible(true); + method.invoke(null, graph); + } + + private static void assertCallerContextRestored() { + Assert.assertEquals("caller-auth-context", AuthContext.getContext()); + Assert.assertEquals("caller-task-context", TaskManager.getContext()); + Assert.assertEquals("caller_space", HugeGraphAuthProxy.getRequestGraphSpace()); + } } From 5ec3fadd10f99b717fbf4bbf9294f137d0e3a661 Mon Sep 17 00:00:00 2001 From: orion Date: Sun, 25 Oct 2026 16:03:17 +0000 Subject: [PATCH 24/29] fix(docker): preserve matching graph providers Leave matching provider files untouched, including the implicit RocksDB default. Report required provider updates clearly when graph configuration cannot be written. Cover read-only directory and file mounts in Docker CI and document switch constraints. --- .github/scripts/docker-deployment.py | 82 +++++++++++++++++++ docker/README.md | 10 +++ docs/storage-lifecycle.md | 7 +- .../docker/docker-entrypoint-test.sh | 23 ++++++ .../docker/docker-entrypoint.sh | 17 +++- 5 files changed, 136 insertions(+), 3 deletions(-) diff --git a/.github/scripts/docker-deployment.py b/.github/scripts/docker-deployment.py index 15e87ea171..d68ac68342 100755 --- a/.github/scripts/docker-deployment.py +++ b/.github/scripts/docker-deployment.py @@ -20,8 +20,11 @@ import json import os from pathlib import Path +import re import subprocess import sys +import tempfile +import time import urllib.error import urllib.parse import urllib.request @@ -182,6 +185,84 @@ def verify_standalone_recreate(compose, container, image, expected): verify_persisted_graph() +def verify_readonly_graphs(image): + """Start actual images with directory and individual-file read-only binds.""" + environment = json.loads(command("docker", "image", "inspect", "--format", + "{{json .Config.Env}}", image)) + selected = dict(item.split("=", 1) for item in environment if "=" in item)[ + "TOPLINGDB_ROCKSDB_PROVIDER"] + if selected not in {"rocksdb", "topling"}: + raise RuntimeError(f"Unexpected image provider: {selected}") + default = command("docker", "run", "--rm", "--pull", "never", "--entrypoint", "cat", + image, "/hugegraph-server/conf/graphs.default/hugegraph.properties") + default = re.sub(r"(?m)^[ \t]*rocksdb\.provider\s*[:=].*$", "", default) + "\n" + with tempfile.TemporaryDirectory(prefix="hg-readonly-") as temporary: + root = Path(temporary) + for mount_kind in ("directory", "file"): + for configured in (None, "rocksdb", "topling"): + case = f"{mount_kind}-{configured or 'default'}" + graphs = root / case + graphs.mkdir() + config = graphs / "hugegraph.properties" + config.write_text(default + (f"rocksdb.provider={configured}\n" if configured else "")) + original = config.read_bytes() + source = graphs if mount_kind == "directory" else config + destination = "/hugegraph-server/conf/graphs" + if mount_kind == "file": + destination += "/hugegraph.properties" + name = root.name + "-" + case + should_start = (configured or "rocksdb") == selected + try: + container = command("docker", "run", "--detach", "--pull", "never", "--name", name, + "--publish", "127.0.0.1::8080", "--mount", + f"type=bind,src={source},dst={destination},readonly", image) + port = (command("docker", "port", container, "8080/tcp").rsplit(":", 1)[1] + if should_start else None) + deadline = time.monotonic() + 180 + while True: + state = json.loads(command("docker", "inspect", "--format", + "{{json .State}}", container)) + if not state["Running"]: + break + if should_start: + try: + if response(f"http://127.0.0.1:{port}/graphspaces/DEFAULT/graphs")[0] == 200: + break + except (urllib.error.URLError, ConnectionError, TimeoutError): + pass + if time.monotonic() >= deadline: + raise RuntimeError(f"Read-only graph case timed out: {case}") + time.sleep(1) + if should_start: + if not state["Running"]: + raise RuntimeError(f"Matching read-only graph did not start: {case}") + probe = subprocess.run(["docker", "exec", container, "sh", "-c", + "printf probe >> /hugegraph-server/conf/graphs/hugegraph.properties"], + capture_output=True, text=True, timeout=30) + if probe.returncode == 0 or "Read-only file system" not in probe.stderr: + raise RuntimeError(f"Graph configuration mount is not read-only: {case}") + subprocess.run(["docker", "stop", "--time", "60", container], check=True, timeout=90) + stopped = json.loads(command("docker", "inspect", "--format", + "{{json .State}}", container)) + if stopped["OOMKilled"] or stopped["ExitCode"] not in {0, 143}: + raise RuntimeError(f"Read-only graph shutdown failed: {case}: {stopped}") + else: + logs = command("docker", "logs", container) + expected = f"cannot change rocksdb.provider from '{configured or 'rocksdb'}'" + if state["ExitCode"] == 0 or expected not in logs or \ + f"'{selected}' in ./conf/graphs/hugegraph.properties" not in logs or \ + "Initializing HugeGraph Store" in logs: + raise RuntimeError(f"Mismatched read-only graph did not fail clearly: {case}: {logs}") + if config.read_bytes() != original: + raise RuntimeError(f"Read-only graph configuration changed: {case}") + print(f"PASS: {selected} {case} read-only graph configuration", flush=True) + except BaseException: + subprocess.run(["docker", "logs", name], check=False, timeout=30) + raise + finally: + subprocess.run(["docker", "rm", "--force", "--volumes", name], check=True, timeout=120) + + def smoke(tag, topology, images): root = Path(__file__).resolve().parents[2] project = f"hg-pr-{tag}-{Path(topology).stem}" @@ -235,6 +316,7 @@ def main(tag): HUGEGRAPH_AUTH_TOKEN_SECRET="0123456789abcdef" * 4, HG_PD_AUTH_SECRET_KEY=PD_PASSWORD) smoke(tag, "docker-compose.yml", {"server": "hugegraph/hugegraph"}) + verify_readonly_graphs(f"hugegraph/hugegraph:{tag}") smoke(tag, "docker-compose-hstore.yml", {"pd": "hugegraph/pd", "store": "hugegraph/store", "server": "hugegraph/server"}) diff --git a/docker/README.md b/docker/README.md index 5ecd910c60..5428c81114 100644 --- a/docker/README.md +++ b/docker/README.md @@ -322,6 +322,16 @@ volume or a bind mount lacks this directory, the entrypoint initializes it once from the image's `conf/graphs.default`. Existing definitions and database files are preserved; explicitly mounted `conf/graphs` directories take precedence. +An explicitly mounted graph directory or file may be read-only when its +`rocksdb.provider` already matches the selected image. An absent property means +`rocksdb`, so a Topling configuration must explicitly select `topling`. Matching +files are left unchanged. If the provider differs, startup reports the graph +file and required change; supply a matching configuration or use a writable +directory mount before switching images. A single-file bind mount cannot be +replaced by the entrypoint's in-place editor, even when mounted writable. +Other environment-driven configuration changes, authentication setup, and API +graph creation/deletion still require writable configuration where they apply. + For the first upgrade from an older image, preserve **custom graph definitions and settings** that existed only in its installation directory. Default seeding does not recover those files from the database. Stop the old Server and copy its diff --git a/docs/storage-lifecycle.md b/docs/storage-lifecycle.md index 6dc08cdf5d..52c81f029b 100644 --- a/docs/storage-lifecycle.md +++ b/docs/storage-lifecycle.md @@ -551,7 +551,12 @@ those missing definitions. See the linked Compose guide for that upgrade path. Both image variants explicitly select their provider. The entrypoint synchronizes `rocksdb.provider` in all persisted or writable mounted `backend=rocksdb` graph files to that selection, including when returning to the unsuffixed standard -image. Other settings and `backend=hstore` definitions are preserved. PD/Store +image. Files whose provider already matches are not written; an absent property +defaults to `rocksdb`. Matching read-only graph mounts can start, while a required +provider change fails with the graph path and current/selected providers. Use a +writable directory or supply a matching configuration before switching; see the +[Compose guide](../docker/README.md) for configuration-mount constraints. +Other settings and `backend=hstore` definitions are preserved. PD/Store apply the selected provider through their runtime configuration overrides. ### Standalone Server setup diff --git a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh index ead02f8791..eddb4c55c1 100755 --- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh +++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh @@ -597,6 +597,29 @@ if ( fi cmp "${TEST_HOME}/analytics.before-invalid-provider" "${TEST_HOME}/conf/graphs/analytics.properties" +# Matching providers must leave the configuration byte-for-byte unchanged. +# Real read-only directory/file mounts are also exercised by Docker deployment CI. +for provider_case in default rocksdb topling; do + selected_provider="${provider_case}" + [[ "${provider_case}" != default ]] || selected_provider=rocksdb + readonly_graph="${TEST_HOME}/conf/graphs/readonly.properties" + printf 'backend=rocksdb\n# Preserve formatting and unrelated settings\ncustom=true\n' > "${readonly_graph}" + if [[ "${provider_case}" != default ]]; then + printf 'rocksdb.provider : %s \n' "${provider_case}" >> "${readonly_graph}" + fi + cp "${readonly_graph}" "${TEST_HOME}/readonly.before" + chmod 444 "${readonly_graph}" + readonly_identity=$(ls -di "${readonly_graph}") + ( + cd "${TEST_HOME}" + TOPLINGDB_ROCKSDB_PROVIDER="${selected_provider}" bash ./docker-entrypoint.sh + ) + cmp "${TEST_HOME}/readonly.before" "${readonly_graph}" + [[ "$(ls -di "${readonly_graph}")" == "${readonly_identity}" ]] + chmod 644 "${readonly_graph}" + rm "${readonly_graph}" +done + # Reproduce the image's symlink with an already populated legacy data directory. # The default copy must bootstrap it, retain data, and never replace later user # definitions. An explicit conf/graphs directory must also bypass bootstrapping. diff --git a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh index b4153a1ffd..e80f800fb5 100755 --- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh +++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh @@ -74,7 +74,7 @@ set_prop_encoded() { key_re="^[[:space:]]*${esc_key}([[:space:]]*[:=]|[[:space:]]+|[[:space:]]*$)" if grep -qE "${key_re}" "${file}"; then - sed -ri "0,/${key_re}/!{/${key_re}/d;}" "${file}" + sed -ri "0,/${key_re}/!{/${key_re}/d;}" "${file}" || return sed -ri "0,/${key_re}/s~${key_re}.*~${key}=${esc_val}~" "${file}" else printf '%s=%s\n' "$key" "$encoded_val" >> "${file}" @@ -268,7 +268,20 @@ if [[ -n "${TOPLINGDB_ROCKSDB_PROVIDER:-}" ]]; then for graph_conf in ./conf/graphs/*.properties; do [[ -f "${graph_conf}" ]] || continue if [[ "$(get_prop_encoded backend "${graph_conf}")" == rocksdb ]]; then - set_prop "rocksdb.provider" "${ROCKSDB_PROVIDER}" "${graph_conf}" + current_provider=$(get_prop_encoded rocksdb.provider "${graph_conf}") + current_provider="${current_provider%"${current_provider##*[![:space:]]}"}" + # RocksDBOptions defaults an absent property to rocksdb. Do not + # touch matching configurations, including read-only mounts. + if ! grep -qE '^[[:space:]]*rocksdb[.]provider([[:space:]]*[:=]|[[:space:]]|$)' "${graph_conf}"; then + current_provider=rocksdb + fi + if [[ "${current_provider}" != "${ROCKSDB_PROVIDER}" ]] && + ! set_prop "rocksdb.provider" "${ROCKSDB_PROVIDER}" "${graph_conf}"; then + log "ERROR: cannot change rocksdb.provider from '${current_provider}' to" \ + "'${ROCKSDB_PROVIDER}' in ${graph_conf}; make the graph configuration" \ + "writable or configure the provider to match the selected image" + exit 1 + fi fi done fi From e42561ffcc53340d996a45dc3c8c1ceaf93ec0e8 Mon Sep 17 00:00:00 2001 From: orion Date: Sun, 25 Oct 2026 16:18:11 +0000 Subject: [PATCH 25/29] fix(docker): propagate JNI preparation failures Keep native capability and license failures visible before changing payload ownership. Exercise actual Docker RUN chains with preparer and ownership failures injected. Retain the numeric non-root user selected by the upstream image baseline. --- docker/test-topling-images.sh | 21 +++++++++++++++++++++ hugegraph-pd/Dockerfile | 4 ++-- hugegraph-server/Dockerfile | 4 ++-- hugegraph-store/Dockerfile | 4 ++-- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/docker/test-topling-images.sh b/docker/test-topling-images.sh index 1ce9c67138..5badc80ec0 100755 --- a/docker/test-topling-images.sh +++ b/docker/test-topling-images.sh @@ -49,6 +49,27 @@ for entry in ("installed_license_directory=topling/licenses/", "installed_notice=topling/NOTICE"): assert entry in inventory, entry +# Execute the actual RUN chain with failures injected after payload creation. +# Ownership adjustment must never hide a failed capability/license preparation. +for dockerfile in ("hugegraph-server/Dockerfile", "hugegraph-pd/Dockerfile", "hugegraph-store/Dockerfile"): + commands = re.findall(r"(?m)^RUN (.*(?:\n[ \t].*)*)", (repo / dockerfile).read_text()) + preparation = next(command for command in commands if "bash /tmp/prepare-topling-image.sh" in command) + preparation = re.sub(r"--mount=\S+\s*", "", preparation.replace("\\\n", " ")) + stubs = ('bash() { echo PREPARE; return "$PREPARE_STATUS"; }; ' + 'chown() { echo OWNERSHIP; return "$CHOWN_STATUS"; }; ' + 'rm() { echo CLEANUP; }; ') + for variant, prepare_status, chown_status, expected in ( + ("topling", 17, 0, 17), ("topling", 0, 23, 23), + ("topling", 0, 0, 0), ("standard", 17, 23, 0)): + environment = dict(os.environ, RUNTIME_VARIANT=variant, HG_UID="10001", HG_GID="10001", + PREPARE_STATUS=str(prepare_status), CHOWN_STATUS=str(chown_status)) + result = subprocess.run(["sh", "-c", stubs + preparation], env=environment, + capture_output=True, text=True) + assert result.returncode == expected, (dockerfile, variant, result.returncode, result.stdout) + assert ("CLEANUP" in result.stdout) == (expected == 0), result.stdout + assert ("OWNERSHIP" in result.stdout) == (variant == "topling" and prepare_status == 0) + print(f"PASS: {dockerfile} preserves preparation/ownership failures before non-root runtime") + docker = shutil.which("docker") if docker is not None: buildx = subprocess.run([docker, "buildx", "version"], capture_output=True, text=True) diff --git a/hugegraph-pd/Dockerfile b/hugegraph-pd/Dockerfile index ec95bcc13f..b484765405 100644 --- a/hugegraph-pd/Dockerfile +++ b/hugegraph-pd/Dockerfile @@ -127,8 +127,8 @@ RUN --mount=type=secret,id=proxy_ca,required=false \ --mount=type=bind,from=topling-jni,target=/run/topling-jni \ if [ "$RUNTIME_VARIANT" = topling ]; then \ if [ -f /run/secrets/proxy_ca ]; then export CURL_CA_BUNDLE=/run/secrets/proxy_ca; fi; \ - bash /tmp/prepare-topling-image.sh; \ - chown -R "${HG_UID}:${HG_GID}" topling conf/toplingdb.yaml; \ + bash /tmp/prepare-topling-image.sh \ + && chown -R "${HG_UID}:${HG_GID}" topling conf/toplingdb.yaml; \ fi \ && rm /tmp/prepare-topling-image.sh /tmp/topling-runtime.env /tmp/toplingdb.yaml diff --git a/hugegraph-server/Dockerfile b/hugegraph-server/Dockerfile index cfd6acf434..f7d8ded3f0 100644 --- a/hugegraph-server/Dockerfile +++ b/hugegraph-server/Dockerfile @@ -137,8 +137,8 @@ RUN --mount=type=secret,id=proxy_ca,required=false \ --mount=type=bind,from=topling-jni,target=/run/topling-jni \ if [ "$RUNTIME_VARIANT" = topling ]; then \ if [ -f /run/secrets/proxy_ca ]; then export CURL_CA_BUNDLE=/run/secrets/proxy_ca; fi; \ - bash /tmp/prepare-topling-image.sh; \ - chown -R "${HG_UID}:${HG_GID}" topling conf/toplingdb.yaml; \ + bash /tmp/prepare-topling-image.sh \ + && chown -R "${HG_UID}:${HG_GID}" topling conf/toplingdb.yaml; \ fi \ && rm /tmp/prepare-topling-image.sh /tmp/topling-runtime.env /tmp/toplingdb.yaml diff --git a/hugegraph-store/Dockerfile b/hugegraph-store/Dockerfile index 7e0af8c75c..386e3ad528 100644 --- a/hugegraph-store/Dockerfile +++ b/hugegraph-store/Dockerfile @@ -128,8 +128,8 @@ RUN --mount=type=secret,id=proxy_ca,required=false \ --mount=type=bind,from=topling-jni,target=/run/topling-jni \ if [ "$RUNTIME_VARIANT" = topling ]; then \ if [ -f /run/secrets/proxy_ca ]; then export CURL_CA_BUNDLE=/run/secrets/proxy_ca; fi; \ - bash /tmp/prepare-topling-image.sh; \ - chown -R "${HG_UID}:${HG_GID}" topling conf/toplingdb.yaml; \ + bash /tmp/prepare-topling-image.sh \ + && chown -R "${HG_UID}:${HG_GID}" topling conf/toplingdb.yaml; \ fi \ && rm /tmp/prepare-topling-image.sh /tmp/topling-runtime.env /tmp/toplingdb.yaml From 8ee68b82dd3073dc69b297978307597a927ae4af Mon Sep 17 00:00:00 2001 From: orion Date: Sun, 25 Oct 2026 16:31:36 +0000 Subject: [PATCH 26/29] test(docker): make read-only fixtures readable Set explicit fixture read and directory traversal permissions for the image UID. Keep read-only bind enforcement while avoiding restrictive runner umask failures. Check fixture readability and cleanup before Docker startup under umask 077. --- .github/scripts/docker-deployment.py | 4 +++ .github/scripts/test_docker_deployment.py | 32 +++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/.github/scripts/docker-deployment.py b/.github/scripts/docker-deployment.py index d68ac68342..1444cf8575 100755 --- a/.github/scripts/docker-deployment.py +++ b/.github/scripts/docker-deployment.py @@ -203,8 +203,12 @@ def verify_readonly_graphs(image): case = f"{mount_kind}-{configured or 'default'}" graphs = root / case graphs.mkdir() + # A restrictive runner umask must not hide the fixture from + # the image's distinct non-root UID. The bind itself is read-only. + graphs.chmod(0o755) config = graphs / "hugegraph.properties" config.write_text(default + (f"rocksdb.provider={configured}\n" if configured else "")) + config.chmod(0o644) original = config.read_bytes() source = graphs if mount_kind == "directory" else config destination = "/hugegraph-server/conf/graphs" diff --git a/.github/scripts/test_docker_deployment.py b/.github/scripts/test_docker_deployment.py index 33926041f7..11321dcde0 100755 --- a/.github/scripts/test_docker_deployment.py +++ b/.github/scripts/test_docker_deployment.py @@ -20,6 +20,7 @@ import json import os from pathlib import Path +import stat import unittest from unittest.mock import patch @@ -103,6 +104,37 @@ def test_hstore_starts_only_pr_services_and_cleans_after_checks(self): self.assertEqual(run.call_args_list[-1].args[0][-3:], ["down", "-v", "--remove-orphans"]) +class ReadOnlyGraphFixtureTest(unittest.TestCase): + def test_fixture_is_readable_by_image_uid_under_restrictive_runner_umask(self): + inspected = [] + + def command(*args): + if args[:3] == ("docker", "image", "inspect"): + return json.dumps(["TOPLINGDB_ROCKSDB_PROVIDER=rocksdb"]) + if "--entrypoint" in args: + return "backend=rocksdb\n" + mount = args[args.index("--mount") + 1] + source = Path(dict(option.split("=", 1) for option in mount.split(",") if "=" in option)["src"]) + directory_mode = stat.S_IMODE(source.stat().st_mode) + file_mode = stat.S_IMODE((source / "hugegraph.properties").stat().st_mode) + self.assertEqual(directory_mode & (stat.S_IROTH | stat.S_IXOTH), + stat.S_IROTH | stat.S_IXOTH) + self.assertTrue(file_mode & stat.S_IROTH) + inspected.append(source) + raise RuntimeError("fixture inspected before Docker startup") + + previous_umask = os.umask(0o077) + try: + with patch.object(deployment, "command", side_effect=command), \ + patch.object(deployment.subprocess, "run"): + with self.assertRaisesRegex(RuntimeError, "fixture inspected"): + deployment.verify_readonly_graphs("hugegraph/hugegraph:fixture") + self.assertEqual(len(inspected), 1) + self.assertFalse(inspected[0].exists()) + finally: + os.umask(previous_umask) + + class GraphDefinitionPersistenceTest(unittest.TestCase): def test_force_recreate_keeps_graph_config_in_data_volume(self): compose = ["docker", "compose", "-p", "fixture", "-f", "/repo/docker/docker-compose.yml"] From 1f8ddb2ceaf9a5d0db9aa821e0ad8ed55f27f96c Mon Sep 17 00:00:00 2001 From: orion Date: Sun, 25 Oct 2026 16:38:43 +0000 Subject: [PATCH 27/29] test(docker): isolate read-only mount failures Give private non-secret test fixtures baseline access for the image UID. Ensure write rejection comes from the read-only bind rather than host inode permissions. Verify baseline read/write/traversal access under a restrictive runner umask. --- .github/scripts/docker-deployment.py | 9 +++++---- .github/scripts/test_docker_deployment.py | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/scripts/docker-deployment.py b/.github/scripts/docker-deployment.py index 1444cf8575..e220b5b4de 100755 --- a/.github/scripts/docker-deployment.py +++ b/.github/scripts/docker-deployment.py @@ -203,12 +203,13 @@ def verify_readonly_graphs(image): case = f"{mount_kind}-{configured or 'default'}" graphs = root / case graphs.mkdir() - # A restrictive runner umask must not hide the fixture from - # the image's distinct non-root UID. The bind itself is read-only. - graphs.chmod(0o755) + # The private temp parent stays 0700. Give the image UID baseline + # access to these non-secret fixtures so only the read-only bind + # denies writes, independently of the runner's umask/identity. + graphs.chmod(0o777) config = graphs / "hugegraph.properties" config.write_text(default + (f"rocksdb.provider={configured}\n" if configured else "")) - config.chmod(0o644) + config.chmod(0o666) original = config.read_bytes() source = graphs if mount_kind == "directory" else config destination = "/hugegraph-server/conf/graphs" diff --git a/.github/scripts/test_docker_deployment.py b/.github/scripts/test_docker_deployment.py index 11321dcde0..3c9a36976e 100755 --- a/.github/scripts/test_docker_deployment.py +++ b/.github/scripts/test_docker_deployment.py @@ -105,7 +105,7 @@ def test_hstore_starts_only_pr_services_and_cleans_after_checks(self): class ReadOnlyGraphFixtureTest(unittest.TestCase): - def test_fixture_is_readable_by_image_uid_under_restrictive_runner_umask(self): + def test_fixture_is_usable_by_image_uid_before_readonly_mount_under_restrictive_umask(self): inspected = [] def command(*args): @@ -117,9 +117,9 @@ def command(*args): source = Path(dict(option.split("=", 1) for option in mount.split(",") if "=" in option)["src"]) directory_mode = stat.S_IMODE(source.stat().st_mode) file_mode = stat.S_IMODE((source / "hugegraph.properties").stat().st_mode) - self.assertEqual(directory_mode & (stat.S_IROTH | stat.S_IXOTH), - stat.S_IROTH | stat.S_IXOTH) - self.assertTrue(file_mode & stat.S_IROTH) + self.assertEqual(directory_mode & (stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH), + stat.S_IROTH | stat.S_IWOTH | stat.S_IXOTH) + self.assertEqual(file_mode & (stat.S_IROTH | stat.S_IWOTH), stat.S_IROTH | stat.S_IWOTH) inspected.append(source) raise RuntimeError("fixture inspected before Docker startup") From 4c00ca0bbf211981c1c9a24aa90c001d07f5790c Mon Sep 17 00:00:00 2001 From: imbajin Date: Mon, 26 Oct 2026 01:17:50 +0800 Subject: [PATCH 28/29] fix(rocksdb): apply image provider at runtime store open --- .../store/rocksdb/RocksDBStoreProvider.java | 22 +++++++++ .../unit/core/BackendProviderFactoryTest.java | 45 ++++++++++++++++++- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStoreProvider.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStoreProvider.java index 82075919ad..c50f3c91e1 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStoreProvider.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStoreProvider.java @@ -23,20 +23,25 @@ import org.apache.hugegraph.backend.store.BackendStore; import org.apache.hugegraph.config.HugeConfig; import org.apache.hugegraph.util.ConfigUtil; +import org.apache.hugegraph.util.E; public class RocksDBStoreProvider extends AbstractBackendStoreProvider { + private static final String IMAGE_PROVIDER = "TOPLINGDB_ROCKSDB_PROVIDER"; + protected String database() { return this.graph().toLowerCase(); } @Override protected BackendStore newSchemaStore(HugeConfig config, String store) { + applyImageProvider(config); return new RocksDBStore.RocksDBSchemaStore(this, this.database(), store); } @Override protected BackendStore newGraphStore(HugeConfig config, String store) { + applyImageProvider(config); return new RocksDBStore.RocksDBGraphStore(this, this.database(), store); } @@ -65,9 +70,26 @@ public void onDeleteConfig(HugeConfig config) { @Override protected BackendStore newSystemStore(HugeConfig config, String store) { + applyImageProvider(config); return new RocksDBStore.RocksDBSystemStore(this, this.database(), store); } + private static void applyImageProvider(HugeConfig config) { + applyImageProvider(config, System.getenv(IMAGE_PROVIDER)); + } + + private static void applyImageProvider(HugeConfig config, String provider) { + if (config.containsKey(RocksDBOptions.PROVIDER.name()) || + provider == null || provider.isBlank()) { + return; + } + E.checkArgument("rocksdb".equals(provider) || "topling".equals(provider), + "Unsupported %s value '%s'", IMAGE_PROVIDER, provider); + // Keep image selection above persisted graph/PD configuration so an + // image rollback can switch the runtime without rewriting metadata. + config.setProperty(RocksDBOptions.PROVIDER.name(), provider); + } + @Override public String type() { return "rocksdb"; diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/BackendProviderFactoryTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/BackendProviderFactoryTest.java index 58bd977b6e..fbe46653d2 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/BackendProviderFactoryTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/BackendProviderFactoryTest.java @@ -25,11 +25,16 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import org.apache.hugegraph.exception.BackendException; +import org.apache.commons.configuration2.PropertiesConfiguration; import org.apache.hugegraph.backend.store.BackendProviderFactory; import org.apache.hugegraph.backend.store.memory.InMemoryDBStoreProvider; import org.apache.hugegraph.backend.store.raft.RaftBackendStoreProvider; +import org.apache.hugegraph.backend.store.rocksdb.RocksDBOptions; +import org.apache.hugegraph.backend.store.rocksdb.RocksDBStoreProvider; +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.exception.BackendException; import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.testutil.Whitebox; import org.apache.hugegraph.unit.BaseUnitTest; import org.junit.Test; @@ -105,4 +110,42 @@ public void testRegisterConcurrently() throws Exception { executor.shutdownNow(); } } + + @Test + public void testRocksDBImageProviderIsRuntimeOnlyAndPreservesExplicitConfig() { + HugeConfig implicit = rocksConfig(); + applyImageProvider(implicit, "topling"); + Assert.assertEquals("topling", implicit.get(RocksDBOptions.PROVIDER)); + + HugeConfig standard = rocksConfig(); + applyImageProvider(standard, "rocksdb"); + Assert.assertEquals("rocksdb", standard.get(RocksDBOptions.PROVIDER)); + + HugeConfig explicit = rocksConfig(); + explicit.setProperty(RocksDBOptions.PROVIDER.name(), "rocksdb"); + applyImageProvider(explicit, "topling"); + Assert.assertEquals("rocksdb", explicit.get(RocksDBOptions.PROVIDER)); + + HugeConfig absent = rocksConfig(); + applyImageProvider(absent, null); + applyImageProvider(absent, ""); + Assert.assertFalse(absent.containsKey(RocksDBOptions.PROVIDER.name())); + + HugeConfig invalid = rocksConfig(); + Assert.assertThrows(IllegalArgumentException.class, + () -> applyImageProvider(invalid, "invalid")); + Assert.assertFalse(invalid.containsKey(RocksDBOptions.PROVIDER.name())); + } + + private static HugeConfig rocksConfig() { + PropertiesConfiguration properties = new PropertiesConfiguration(); + properties.setProperty("backend", "rocksdb"); + return new HugeConfig(properties); + } + + private static void applyImageProvider(HugeConfig config, String provider) { + Whitebox.invoke(RocksDBStoreProvider.class, + new Class[]{HugeConfig.class, String.class}, + "applyImageProvider", null, config, provider); + } } From 693470e05e6e3ae876353f254a9450eb55ddce8e Mon Sep 17 00:00:00 2001 From: imbajin Date: Mon, 26 Oct 2026 01:23:51 +0800 Subject: [PATCH 29/29] fix(rocksdb): apply image provider on every config load --- .../store/rocksdb/RocksDBStoreProvider.java | 23 +++++++++++++++---- .../unit/core/BackendProviderFactoryTest.java | 2 ++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStoreProvider.java b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStoreProvider.java index c50f3c91e1..0b92f6aaaa 100644 --- a/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStoreProvider.java +++ b/hugegraph-server/hugegraph-rocksdb/src/main/java/org/apache/hugegraph/backend/store/rocksdb/RocksDBStoreProvider.java @@ -34,14 +34,30 @@ protected String database() { } @Override - protected BackendStore newSchemaStore(HugeConfig config, String store) { + public BackendStore loadSchemaStore(HugeConfig config) { + applyImageProvider(config); + return super.loadSchemaStore(config); + } + + @Override + public BackendStore loadGraphStore(HugeConfig config) { applyImageProvider(config); + return super.loadGraphStore(config); + } + + @Override + public BackendStore loadSystemStore(HugeConfig config) { + applyImageProvider(config); + return super.loadSystemStore(config); + } + + @Override + protected BackendStore newSchemaStore(HugeConfig config, String store) { return new RocksDBStore.RocksDBSchemaStore(this, this.database(), store); } @Override protected BackendStore newGraphStore(HugeConfig config, String store) { - applyImageProvider(config); return new RocksDBStore.RocksDBGraphStore(this, this.database(), store); } @@ -70,7 +86,6 @@ public void onDeleteConfig(HugeConfig config) { @Override protected BackendStore newSystemStore(HugeConfig config, String store) { - applyImageProvider(config); return new RocksDBStore.RocksDBSystemStore(this, this.database(), store); } @@ -79,7 +94,7 @@ private static void applyImageProvider(HugeConfig config) { } private static void applyImageProvider(HugeConfig config, String provider) { - if (config.containsKey(RocksDBOptions.PROVIDER.name()) || + if (config == null || config.containsKey(RocksDBOptions.PROVIDER.name()) || provider == null || provider.isBlank()) { return; } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/BackendProviderFactoryTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/BackendProviderFactoryTest.java index fbe46653d2..563fa45e2f 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/BackendProviderFactoryTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/core/BackendProviderFactoryTest.java @@ -113,6 +113,8 @@ public void testRegisterConcurrently() throws Exception { @Test public void testRocksDBImageProviderIsRuntimeOnlyAndPreservesExplicitConfig() { + applyImageProvider(null, "topling"); + HugeConfig implicit = rocksConfig(); applyImageProvider(implicit, "topling"); Assert.assertEquals("topling", implicit.get(RocksDBOptions.PROVIDER));