diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index d3e75dad2e..82bb089200 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -108,6 +108,9 @@ TRAVIS + "test-check-jacoco-report.sh": {"struct"}, TRAVIS + "test-codecov-upload-config.sh": {"struct"}, TRAVIS + "test-riscv64-java-runtime.sh": {"server", "riscv64"}, + TRAVIS + "test-store-docker-entrypoint.sh": {"store"}, + TRAVIS + "test-start-hugegraph-store.sh": {"store"}, + TRAVIS + "test-start-server.sh": {"server"}, } diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index 5c1d84f2a7..7e1f511111 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -370,14 +370,17 @@ def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self): ("test-check-java-runtime-logs.sh", {"struct"}), ("test-check-jacoco-report.sh", {"struct"}), ("test-codecov-upload-config.sh", {"struct"}), - ("test-riscv64-java-runtime.sh", {"server", "riscv64"})] + ("test-riscv64-java-runtime.sh", {"server", "riscv64"}), + ("test-store-docker-entrypoint.sh", {"store"}), + ("test-start-hugegraph-store.sh", {"store"}), + ("test-start-server.sh", {"server"})] for name, suites in cases: script = policy.TRAVIS + name with self.subTest(script=script): plan = self.plan_for([script]) self.assertEqual(suites, set(plan["selected"])) callers = self.coordinator_jobs(plan) - if "struct" in suites: + if "struct" in suites or "store" in suites: self.assertEqual({"pd_store"}, callers) root = (workflows / "server-ci.yml").read_text() caller = re.search(r"^ pd_store:\n(.*?)(?=^ [\w-]+:|\Z)", root, re.M | re.S).group(1) @@ -387,13 +390,21 @@ def test_contract_scripts_reach_their_actual_inner_jobs_and_steps(self): binding = binding.replace("needs.preflight.outputs." + key, repr(str(plan[key]).lower())) inputs[key] = eval(binding, {"__builtins__": {}}) workflow = (workflows / "pd-store-ci.yml").read_text() - struct = workflow.split("\n struct:\n", 1)[1].split("\n pd:\n", 1)[0] - condition = re.search(r"^ if: (.+)$", struct, re.M).group(1) + job = "struct" if "struct" in suites else "store" + body = re.search(r"^ " + job + r":\n(.*?)(?=^ [\w-]+:|\Z)", workflow, re.M | re.S).group(1) + condition = re.search(r"^ if: (.+)$", body, re.M).group(1) condition = condition.replace("inputs.run-struct", str(inputs["struct"])) condition = condition.replace("inputs.run-store", str(inputs["store"])) - self.assertTrue(eval(condition.replace("||", "or").replace("&&", "and").replace("!", "not "), {"__builtins__": {}})) - step = next(step for step in struct.split("\n - ") if Path(script).name in step) - self.assertNotRegex(step, r"(?m)^ if:") + self.assertTrue(eval(condition.replace("||", "or").replace("&&", "and").replace("!", "not "), + {"__builtins__": {}})) + step = next(step for step in body.split("\n - ") if Path(script).name in step) + gate = re.search(r"(?m)^ if: (.+)$", step) + if gate: + # Only the job's own tool preflight may gate a contract step, and that + # preflight step must itself always run + self.assertEqual("steps.store-preflight.outputs.can_run == 'true'", gate.group(1)) + preflight = next(s for s in body.split("\n - ") if "id: store-preflight" in s) + self.assertNotRegex(preflight, r"(?m)^ if:") else: self.assertIn("server-rocksdb", callers) workflow = (workflows / "server-ci.yml").read_text() diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index e4dd1809ea..5ea7c1f05a 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -364,6 +364,10 @@ jobs: run: | mvn editorconfig:check -pl hugegraph-store/hg-store-test -am -ntp + - name: Run Store docker entrypoint rocksdb memory tests + run: | + $TRAVIS_DIR/test-store-docker-entrypoint.sh + - name: Check startup test prerequisites (Store) id: store-preflight run: | diff --git a/helm/hugegraph/values-cluster.yaml b/helm/hugegraph/values-cluster.yaml index 98e7d99058..1a9f0014d4 100644 --- a/helm/hugegraph/values-cluster.yaml +++ b/helm/hugegraph/values-cluster.yaml @@ -57,9 +57,11 @@ store: -XX:MaxMetaspaceSize=256m -XX:MaxDirectMemorySize=512m -XX:+UseContainerSupport - # Container limits do not configure JVM heap or RocksDB native cache budgets. - # Size this preset for the workload and configure those application budgets - # separately; javaOpts accepts explicit JVM/system-property overrides. + # Container limits do not configure the JVM heap; javaOpts does. Store images + # with auto-sized RocksDB memory (apache/hugegraph#3254) give RocksDB half of + # the container limit left after heap, direct memory and a 1.5 GiB reserve; + # with this preset that is 2.5 GiB. Set store.extraEnv + # HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE (bytes) to use a different value. resources: requests: cpu: "1" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh index 6e1bc0394d..6c30369c61 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-store.sh @@ -172,21 +172,50 @@ JAVA printf '%s\n' '#!/bin/sh' 'echo launcher-test' > "$fixture/jdk/bin/uname" chmod +x "$fixture/jdk/bin/"* "$fixture/bin/start-hugegraph-store.sh" for option in default custom-java-options user-options default-serial default-zgc \ - g1 G1 zgc ZGC invalid missing-module-options; do + g1 G1 zgc ZGC direct-default direct-heap-lower direct-heap-raised \ + direct-heap-tab direct-heap-alias direct-explicit-cap \ + direct-tool-options-cap direct-jdk-options-cap direct-tool-options-heap \ + direct-tool-options-single direct-tool-options-double \ + direct-jdk-options-single direct-jdk-options-double \ + direct-tool-options-inner direct-tool-options-tab direct-tool-options-other \ + invalid missing-module-options; do capture="$fixture/$option.args" local -a args=(-d false) + local tool_options="" jdk_options="" jvm_options="-Xms64m -Xmx64m" case "$option" in default) jvm_options="" ;; custom-java-options) jvm_options="$jvm_options -Dlauncher.test.custom=true" ;; user-options) jvm_options=""; args+=(-j "-Xmx96m -Dlauncher.test=true") ;; + # The direct-memory cap follows the heap the JVM actually gets (calc_xmx: 1024) + direct-default) jvm_options="" ;; + direct-heap-lower) jvm_options=""; args+=(-j "-Xmx512m") ;; + direct-heap-raised) jvm_options=""; args+=(-j "-Dlauncher.test=true -Xmx4g") ;; + direct-heap-tab) jvm_options=""; args+=(-j $'-Dlauncher.test=true\t-Xmx4g') ;; + direct-heap-alias) jvm_options=""; args+=(-j "-XX:MaxHeapSize=4g") ;; + direct-explicit-cap) jvm_options=""; args+=(-j "-XX:MaxDirectMemorySize=256m") ;; + # The JVM reads these before the command line, so a launcher cap would override them + direct-tool-options-cap) jvm_options=""; tool_options="-Dt=1 -XX:MaxDirectMemorySize=512m" ;; + direct-jdk-options-cap) jvm_options=""; jdk_options="-XX:MaxDirectMemorySize=512m" ;; + # The JVM drops quotes anywhere in a token of these variables + direct-tool-options-single) jvm_options=""; tool_options="-Dt=1 '-XX:MaxDirectMemorySize=64m'" ;; + direct-tool-options-double) jvm_options=""; tool_options='"-XX:MaxDirectMemorySize=64m"' ;; + direct-jdk-options-single) jvm_options=""; jdk_options="'-XX:MaxDirectMemorySize=64m'" ;; + direct-jdk-options-double) jvm_options=""; jdk_options='-Dt=1 "-XX:MaxDirectMemorySize=64m"' ;; + direct-tool-options-inner) jvm_options=""; tool_options='-XX:"MaxDirectMemorySize"=64m' ;; + direct-tool-options-tab) jvm_options=""; tool_options=$'-Dt=1\t-XX:MaxDirectMemorySize=64m' ;; + # Another option that merely contains the name doesn't count + direct-tool-options-other) jvm_options=""; tool_options="-Dt=-XX:MaxDirectMemorySize=64m" ;; + # The launcher -Xmx overrides an environment heap, so the cap still follows calc_xmx + direct-tool-options-heap) jvm_options=""; tool_options="-Xmx4g" ;; missing-module-options) rm "$fixture/bin/jvm-module.options" ;; default-serial) jvm_options="$jvm_options -XX:+UseSerialGC" ;; default-zgc) jvm_options="$jvm_options -XX:+UseZGC" ;; *) args+=(-g "$option") ;; esac - env -u GC_OPTION -u JAVA_TOOL_OPTIONS -u LD_PRELOAD \ + env -u GC_OPTION -u LD_PRELOAD \ -u TOPLINGDB_ROCKSDB_PROVIDER -u TOPLING_RUNTIME_CLASSPATH -u TOPLING_ACTIVE_NATIVE \ + JAVA_TOOL_OPTIONS="$tool_options" JDK_JAVA_OPTIONS="$jdk_options" \ JAVA_HOME="$fixture/jdk" JAVA_OPTIONS="$jvm_options" \ OPEN_TELEMETRY=false STDOUT_MODE=true CAPTURE_FILE="$capture" \ PATH="$fixture/jdk/bin:$PATH" \ @@ -217,6 +246,23 @@ JAVA elif [[ "$option" == user-options ]] && ! grep -Fxq -- '-Dlauncher.test=true' "$capture"; then fail "launcher dropped custom -j options" + elif [[ "$option" == direct-* ]]; then + local cap want + # The JVM uses the last -XX:MaxDirectMemorySize on the command line + cap=$(grep -E '^-XX:MaxDirectMemorySize=' "$capture" | tail -1) + case "$option" in + direct-default | direct-tool-options-heap | direct-tool-options-other) + want='-XX:MaxDirectMemorySize=1024m' ;; + direct-explicit-cap) want='-XX:MaxDirectMemorySize=256m' ;; + # A heap-only -j leaves the cap to the JVM, which defaults it to that heap; + # a cap in JAVA_TOOL_OPTIONS or JDK_JAVA_OPTIONS is left to apply as given + *) want='' ;; + esac + if [[ "$cap" == "$want" ]]; then + pass "launcher '$option' sets direct memory to '${want:-no launcher cap}'" + else + fail "launcher '$option' set direct memory to '${cap:-none}', expected '${want:-none}'" + fi elif [[ "$option" == zgc || "$option" == ZGC ]]; then if grep -Fxq -- '-XX:+UseZGC' "$capture" && \ ! grep -Fxq -- '-XX:InitiatingHeapOccupancyPercent=50' "$capture"; then diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-store-docker-entrypoint.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-store-docker-entrypoint.sh new file mode 100755 index 0000000000..ce20947cc8 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-store-docker-entrypoint.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Checks that the Store Docker entrypoint passes HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE +# into SPRING_APPLICATION_JSON as rocksdb.total_memory_size, leaves the rocksdb key +# out when the variable is unset (so conf/application-pd.yml or auto-sizing applies), +# and refuses values other than auto or a plain number of bytes. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../.." && pwd)" +ENTRYPOINT="${REPO_ROOT}/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh" +TMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/store-entrypoint-test.XXXXXX") +trap 'rm -rf "${TMP_DIR}"' EXIT + +PASS=0 +FAIL=0 + +[[ -f "${ENTRYPOINT}" ]] || { echo "entrypoint not found at ${ENTRYPOINT}" >&2; exit 1; } +command -v python3 >/dev/null || { echo "python3 required" >&2; exit 1; } + +mkdir -p "${TMP_DIR}/bin" +cp "${ENTRYPOINT}" "${TMP_DIR}/docker-entrypoint.sh" +# Stand in for the launcher: record the generated config instead of starting Store +cat > "${TMP_DIR}/bin/start-hugegraph-store.sh" <<'STUB' +#!/usr/bin/env bash +printf '%s' "${SPRING_APPLICATION_JSON}" > ./spring.json +STUB +chmod +x "${TMP_DIR}/bin/start-hugegraph-store.sh" "${TMP_DIR}/docker-entrypoint.sh" + +run_entrypoint() { + rm -f "${TMP_DIR}/spring.json" + # Drop any value from the caller's environment so "unset" really is unset + (cd "${TMP_DIR}" && env -u HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE \ + HG_STORE_PD_ADDRESS=pd0:8686 \ + HG_STORE_GRPC_HOST=store0 \ + HG_STORE_RAFT_ADDRESS=store0:8510 \ + "$@" \ + ./docker-entrypoint.sh 2>&1) +} + +expect_memory() { + # expect_memory [NAME=VALUE ...] + local name="$1" expected="$2" + shift 2 + local out + if ! out=$(run_entrypoint "$@"); then + echo " FAIL ${name}: entrypoint exited non-zero" + printf '%s\n' "${out}" | tail -3 + FAIL=$((FAIL + 1)) + return + fi + if ! EXPECTED="${expected}" python3 - "${TMP_DIR}/spring.json" <<'PY' +import json, os, sys +with open(sys.argv[1], encoding="utf-8") as fh: + doc = json.load(fh) +want = os.environ["EXPECTED"] +got = doc.get("rocksdb", {}).get("total_memory_size", "absent") +if got != want: + print(" rocksdb.total_memory_size is %r, expected %r" % (got, want)) + sys.exit(1) +if doc["app"]["data-path"] != "/hugegraph-store/storage": + print(" app.data-path was lost") + sys.exit(1) +PY + then + echo " FAIL ${name}: invalid JSON or wrong rocksdb.total_memory_size" + FAIL=$((FAIL + 1)) + return + fi + echo " PASS ${name}" + PASS=$((PASS + 1)) +} + +echo "Store docker-entrypoint rocksdb memory override" +expect_memory "unset leaves the key out" absent +expect_memory "empty leaves the key out" absent HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE= +expect_memory "bytes are passed through" 2147483648 \ + HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE=2147483648 +expect_memory "auto is passed through" auto HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE=auto +expect_memory "legacy 0 is passed through for auto-sizing" 0 HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE=0 + +for bad in '4GB' '-1' '1.5e9' '1 2' '1"}' 'automatic'; do + if run_entrypoint HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE="${bad}" >/dev/null; then + echo " FAIL '${bad}' was accepted" + FAIL=$((FAIL + 1)) + else + echo " PASS '${bad}' is refused" + PASS=$((PASS + 1)) + fi +done + +echo "${PASS} passed, ${FAIL} failed" +[[ "${FAIL}" -eq 0 ]] diff --git a/hugegraph-store/README.md b/hugegraph-store/README.md index 531ff49079..4c0744c2de 100644 --- a/hugegraph-store/README.md +++ b/hugegraph-store/README.md @@ -250,7 +250,9 @@ bin/start-hugegraph-store.sh [-g GC_TYPE] [-j "JVM_OPTIONS"] [-d DAEMON] set in `JAVA_OPTIONS` apply. Explicit `-g g1` selects G1 even when JVM ergonomics would choose Serial GC; unknown values are rejected. When using explicit `-g`, do not select another collector in `JAVA_OPTIONS` or `-j`; conflicting selections are rejected by the JVM. -- `-j`: Custom JVM options (e.g., `-j "-Xmx16g -Xms8g"`) +- `-j`: Custom JVM options (e.g., `-j "-Xmx16g -Xms8g -XX:MaxDirectMemorySize=2g"`). A `-j` heap replaces the + launcher's direct-memory cap, so pass one with it; otherwise the RocksDB budget counts a second heap + of direct memory (see the deployment guide's Store Configuration). - `-d`: Daemon mode (`true` = daemon, `false` = foreground; default: `true`). Set to `false` when running under Docker or a process supervisor so the container exits if Java dies. The launcher always loads `bin/jvm-module.options`, including with custom `JAVA_OPTIONS` or `-j`. @@ -394,6 +396,10 @@ docker run -d \ hugegraph/store:latest ``` +Add `-e HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE=` to pin the off-heap RocksDB budget. +Without it the Store sizes the budget from the container memory limit; see +[Store Configuration](docs/deployment-guide.md#store-configuration) in the deployment guide. + **Exposed Ports**: - `8520`: REST API (management, metrics) - `8500`: gRPC (client connections) diff --git a/hugegraph-store/docs/best-practices.md b/hugegraph-store/docs/best-practices.md index 47d02521c3..74524943e7 100644 --- a/hugegraph-store/docs/best-practices.md +++ b/hugegraph-store/docs/best-practices.md @@ -85,7 +85,7 @@ app: **JVM Settings**: ```bash -JAVA_OPTS="-Xms32g -Xmx32g -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:G1ReservePercent=20" +JAVA_OPTS="-Xms32g -Xmx32g -XX:MaxDirectMemorySize=2g -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:G1ReservePercent=20" ``` ### PD Node Recommendations @@ -252,6 +252,7 @@ store.rpc_timeout=30000 # 30 seconds ```bash JAVA_OPTS=" -Xms32g -Xmx32g # Fixed heap size + -XX:MaxDirectMemorySize=2g # Direct-memory cap; RocksDB auto-sizing counts it -XX:+UseG1GC # G1 garbage collector -XX:MaxGCPauseMillis=200 # Target pause time -XX:G1ReservePercent=20 # Reserve for to-space @@ -267,6 +268,7 @@ JAVA_OPTS=" ```bash JAVA_OPTS=" -Xms32g -Xmx32g + -XX:MaxDirectMemorySize=2g # Direct-memory cap; RocksDB auto-sizing counts it -XX:+UseZGC # ZGC (sub-10ms pauses) -XX:ZCollectionInterval=120 # GC interval (seconds) -XX:+UnlockDiagnosticVMOptions diff --git a/hugegraph-store/docs/deployment-guide.md b/hugegraph-store/docs/deployment-guide.md index 073a2624f6..1e31c92137 100644 --- a/hugegraph-store/docs/deployment-guide.md +++ b/hugegraph-store/docs/deployment-guide.md @@ -379,7 +379,7 @@ File: `hugegraph-store/conf/application-pd.yml` (RocksDB tuning) ```yaml rocksdb: # Memory Configuration - total_memory_size: 32000000000 # Total memory for RocksDB (32GB) + total_memory_size: auto # Off-heap RocksDB budget: auto or bytes write_buffer_size: 134217728 # Memtable size (128MB) max_write_buffer_number: 6 # Max memtables min_write_buffer_number_to_merge: 2 # Min memtables to merge @@ -388,14 +388,87 @@ rocksdb: level0_file_num_compaction_trigger: 4 max_background_jobs: 8 # Background compaction/flush threads - # Block Cache - block_cache_size: 16000000000 # Block cache (16GB) + # Budget split: this share goes to memtables, the rest to the block cache + write_buffer_ratio: 0.66 # SST File Size target_file_size_base: 134217728 # Target SST size (128MB) max_bytes_for_level_base: 1073741824 # L1 size (1GB) ``` +`total_memory_size` bounds the memtables (`write_buffer_ratio` of it) and the block cache +(the rest). Both live outside the Java heap, so `-Xmx` does not limit them. A fixed 1 GiB +cache for the Raft log comes on top of this budget; the Raft log memtables of all partitions +share a 256 MiB write buffer charged to that cache, so they don't add to it. + +With `total_memory_size: auto` (the shipped value; a missing, blank or `0` value means the same) +the Store sizes the budget at startup as half of what is left after the heap, the maximum direct +memory and a 1.5 GiB reserve (the 1 GiB Raft log cache plus 512 MiB for other native memory), +clamped to 256 MiB–32 GB. Inside a container the starting point is the cgroup memory limit, including a +limit set only on a parent cgroup when the Store can see that cgroup; inside a private cgroup +namespace, or when only a subtree of the cgroup2 hierarchy is mounted, a limit above it is not +visible, so pin the value there if only the pod or slice is limited. The Store uses the lowest +limit it can see and logs a warning when it can't see the whole hierarchy; it doesn't guess at +limits it can't read. A Store with a 4 GiB limit, `-Xmx1g` and `-XX:MaxDirectMemorySize=512m` +gets 512 MiB. Without +`-XX:MaxDirectMemorySize` the JVM allows as much direct memory as heap, and the budget counts +it that way. `bin/start-hugegraph-store.sh` sets the flag to the heap it picks; when `-j` sets +`-Xmx`, `-XX:MaxHeapSize` or `-XX:MaxDirectMemorySize` itself, or `JAVA_TOOL_OPTIONS` or +`JDK_JAVA_OPTIONS` sets `-XX:MaxDirectMemorySize`, the launcher leaves the flag out and the JVM's +default (that heap) or the given cap applies. Either way the budget counts a limit the JVM +enforces. In small containers set a +cap below the heap, or the budget falls to the 256 MiB floor. + +On a host without a container limit the starting point is physical memory. The default +`bin/start-hugegraph-store.sh` caps the heap at 2 GiB and direct memory at the same 2 GiB, so a +16 GiB host gets about 5.25 GiB. When you raise the heap, pass a direct-memory cap with it, +for example `-j "-Xmx36g -XX:MaxDirectMemorySize=2g"`; otherwise direct memory counts as a +second heap. With a 2 GiB cap a 16 GiB host with `-Xmx8g` gets 2.25 GiB and a 64 GiB host with +`-Xmx36g` gets 12.25 GiB. Before this default every host got a fixed `32000000000`, so pin the +value after upgrading if a large node relied on it. + +At startup the Store logs the budget and what it was computed from, for example: + +```text +RocksDB memory budget: 512 MiB (536870912 bytes), mode=auto, physical-limit=4 GiB, source=cgroup-v2, heap=1 GiB, direct=512 MiB, reserve=1.5 GiB, cgroup-view=complete +``` + +It warns when the floor applies, or when an explicit value is under 256 MiB or exceeds the +memory left after the heap, direct memory and the reserve. Set a number of bytes, at least +1048576 (1 MiB), to pin the budget; a smaller value or a unit suffix such as `4GB` is rejected. Before this default existed the +shipped value was `32000000000`, which can exceed a container limit and get the Store +OOM-killed. Both caps are soft: RocksDB can briefly go over them under heavy writes, and a +small budget with many partitions means frequent memtable flushes. + +**Upgrading an existing deployment.** Auto-sizing applies only when the loaded configuration +has `total_memory_size: auto` (or `0`, or no value). A Store that mounts its own `conf/application-pd.yml`, +or a tarball install that kept its `conf/` from an earlier release, still has `32000000000` and +keeps that fixed budget; the Store logs a warning naming the old value at startup. Set the key +to `auto`, or set `HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE` in Docker. + +PD can also hold the old value. When PD has no shared copy yet, the first Store to start uploads +its local `application-pd.yml` to PD (KV key `S:FS`). At startup a Store loads that copy over its +local file only when it has no local file or the file is no newer than PD's `S:Timestamp`. The +upload sets no timestamp and a missing one counts as 0, so changing `S:FS` alone changes nothing. +The simplest upgrade is to set `HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE=auto` (or +`-Drocksdb.total_memory_size=auto`) on every Store and restart it; the Store keeps this value +above both its local file and PD's copy. To change the shared copy instead, set +`rocksdb.total_memory_size` to `0` or remove it in `S:FS`, then put the current time in +milliseconds into `S:Timestamp` through PD's KV service (gRPC `KvService.put`). Use `auto` there +only once every Store runs this release: an earlier release can't parse `auto` and fails to start, +while it reads `0` as its old heap-sized default. This also applies to a copy that a Store of this +release uploaded when PD had none, since the shipped file says `auto`. Every running Store watches +`S:Timestamp`: it reloads PD's copy, writes it over its local file and runs +`bin/restart-hugegraph-store.sh` within 60 seconds, so all Stores restart within the same minute. +A stopped Store loads the copy at its next start. That self-restart passes no arguments, so `-j` +and `-g` from the original start are dropped and the launcher's default heap applies; only JVM +settings in an exported `JAVA_OPTIONS` carry over. A Docker Store doesn't restart at all: Java is +the entrypoint's last process, so stopping it ends the container with exit code 143. The container +stays down unless a restart policy (the bundled compose files use `restart: unless-stopped`) starts +it again through the entrypoint. On Docker, set `HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE` instead of +publishing `S:Timestamp`. Elsewhere, prefer the per-Store override on Stores started with a custom +heap. + ### Server Configuration File: `hugegraph-server/conf/graphs/hugegraph.properties` @@ -758,8 +831,22 @@ environment: HG_STORE_REST_PORT: "8520" # maps to server.port HG_STORE_RAFT_ADDRESS: store0:8510 # maps to raft.address HG_STORE_DATA_PATH: /hugegraph-store/storage # maps to app.data-path + # optional; maps to rocksdb.total_memory_size (auto or bytes); unset = config file + # HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE: "2147483648" ``` +`HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE` must be `auto` or a plain number of bytes; the entrypoint +refuses anything else. When set it outranks `rocksdb.total_memory_size` in a mounted +`conf/application-pd.yml`, a `-Drocksdb.total_memory_size` in `JAVA_OPTS`, and an +`application-pd.yml` that PD distributes to every Store. Leave it unset +to let the Store size the budget from the container memory limit, provided the loaded config +also has `total_memory_size: auto` (or `0`, or no value); a mounted `application-pd.yml` from an earlier +release still has `32000000000` (see [Store Configuration](#store-configuration)). A container +limit bounds only the RocksDB budget: the launcher still picks `-Xmx` from the host's free +memory (up to 2 GiB) and sets direct memory to the same size, so size both for the limit through +`JAVA_OPTS`, for example `JAVA_OPTS: "-Xmx1g -XX:MaxDirectMemorySize=512m"` for a 4 GiB limit. +With the defaults, heap and direct memory plus the 1.5 GiB reserve already exceed a 2 GiB limit. + **Server environment variables**: ```yaml @@ -858,6 +945,10 @@ spec: value: "$(POD_NAME).hugegraph-store" - name: HG_STORE_RAFT_ADDRESS value: "$(POD_NAME).hugegraph-store:8510" + # Optional: pin the off-heap RocksDB budget (bytes). Unset, it is sized + # from the container memory limit below. + # - name: HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE + # value: "4294967296" volumeMounts: - name: data mountPath: /hugegraph-store/storage diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/CgroupMemoryLimit.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/CgroupMemoryLimit.java new file mode 100644 index 0000000000..d17917feef --- /dev/null +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/CgroupMemoryLimit.java @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.options; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +/** + * The lowest cgroup memory limit visible to this process. cgroup v2 walks memory.max from + * the process's cgroup up to the mount root; cgroup v1 reads hierarchical_memory_limit, + * which already covers the ancestors. A limit above what the mounts show, as inside a + * private cgroup namespace or under a subtree-only cgroup2 mount, can't be read: the + * result is then marked incomplete so the caller can warn, and nothing is guessed. + */ +final class CgroupMemoryLimit { + + static final CgroupMemoryLimit NONE = new CgroupMemoryLimit(-1L, 0, false); + + // Bytes, or -1 when no limit is set or none could be read + final long bytes; + // 1 or 2 for the cgroup version the limit came from, 0 when there is no limit + final int version; + // Whether a stricter limit may sit above the part of the hierarchy that is visible + final boolean incomplete; + + CgroupMemoryLimit(long bytes, int version, boolean incomplete) { + this.bytes = bytes; + this.version = version; + this.incomplete = incomplete; + } + + static CgroupMemoryLimit detect() { + return read(Paths.get("/proc/self/cgroup"), Paths.get("/proc/self/mountinfo"), + Paths.get("/sys/fs/cgroup")); + } + + /** + * The membership path from /proc/self/cgroup is rebased onto every mount of the + * controller in /proc/self/mountinfo whose root contains it, and the lowest limit over + * all of them is taken, so mount order doesn't matter. Without a matching mount + * defaultRoot is used. + */ + static CgroupMemoryLimit read(Path procSelfCgroup, Path mountinfo, Path defaultRoot) { + List lines; + try { + lines = Files.readAllLines(procSelfCgroup, StandardCharsets.UTF_8); + } catch (IOException | RuntimeException e) { + return NONE; + } + List v2Mounts = mounts(mountinfo, true); + List v1Mounts = mounts(mountinfo, false); + long limit = -1L; + int version = 0; + boolean incomplete = false; + for (String line : lines) { + String[] parts = line.split(":", 3); + if (parts.length < 3) { + continue; + } + boolean v2 = parts[0].equals("0") && parts[1].isEmpty(); + if (!v2 && !("," + parts[1] + ",").contains(",memory,")) { + continue; + } + List mounts = v2 ? v2Mounts : v1Mounts; + boolean matched = false; + boolean fullView = false; + long lineLimit = -1L; + for (String[] mount : mounts) { + if (!contains(mount[0], parts[2])) { + // A mount of another subtree says nothing about this cgroup + continue; + } + matched = true; + Path point = Paths.get(mount[1]); + String relative = rebase(parts[2], mount[0]); + // A full-hierarchy mount only covers the ancestors if this process's own + // directory is reachable through it + fullView |= mount[0].equals("/") && visible(point, relative); + lineLimit = lowest(lineLimit, v2 ? v2Limit(point, relative) : + v1Limit(point, relative)); + } + if (!matched && !mounts.isEmpty()) { + // In a cgroup namespace /proc/self/cgroup says "/" while a mount made outside + // it has a root such as "/..", so the process's own cgroup can't be located. + // The limits at the mount roots still bound it + incomplete = true; + for (String[] mount : mounts) { + Path point = Paths.get(mount[1]); + lineLimit = lowest(lineLimit, v2 ? readLimit(point.resolve("memory.max")) : + v1Limit(point, "")); + } + } else if (!matched) { + String relative = rebase(parts[2], "/"); + lineLimit = v2 ? v2Limit(defaultRoot, relative) : + v1Limit(defaultRoot.resolve("memory"), relative); + } else if (v2 && (!fullView || parts[2].equals("/"))) { + // Only subtree mounts such as /kubepods, or the process sits at the root it + // can see, which inside a private cgroup namespace is the namespace root: a + // memory.max above is hidden. v1's hierarchical limit has no such gap + incomplete = true; + } + if (lineLimit > 0 && (limit <= 0 || lineLimit < limit)) { + limit = lineLimit; + version = v2 ? 2 : 1; + } + } + return new CgroupMemoryLimit(limit, version, incomplete); + } + + /** + * The {root, mount point} of every cgroup v2 mount, or of every cgroup v1 mount + * carrying the memory controller, in /proc/self/mountinfo order. + */ + private static List mounts(Path mountinfo, boolean v2) { + List mounts = new ArrayList<>(); + List lines; + try { + lines = Files.readAllLines(mountinfo, StandardCharsets.UTF_8); + } catch (IOException | RuntimeException e) { + return mounts; + } + for (String line : lines) { + // id parent major:minor root mount-point options [optional...] - type source super + int separator = line.indexOf(" - "); + if (separator < 0) { + continue; + } + String[] fields = line.substring(0, separator).split(" "); + String[] tail = line.substring(separator + 3).split(" "); + if (fields.length < 5 || tail.length < 1) { + continue; + } + boolean match = v2 ? tail[0].equals("cgroup2") : + tail[0].equals("cgroup") && tail.length >= 3 && + ("," + tail[2] + ",").contains(",memory,"); + if (match) { + mounts.add(new String[]{unescape(fields[3]), unescape(fields[4])}); + } + } + return mounts; + } + + // Whether a mount whose root is mountRoot shows the cgroup at membership + static boolean contains(String mountRoot, String membership) { + String path = membership.startsWith("/") ? membership : "/" + membership; + String root = trimSlash(mountRoot); + return root.equals("/") || path.equals(root) || path.startsWith(root + "/"); + } + + /** + * The membership path relative to a mount whose root is mountRoot: /kubepods/pod1/ctr + * under root /kubepods is pod1/ctr. A path outside the mount root is kept as is. + */ + static String rebase(String membership, String mountRoot) { + String path = membership.startsWith("/") ? membership : "/" + membership; + String root = trimSlash(mountRoot); + if (!root.equals("/")) { + if (path.equals(root)) { + return ""; + } + if (path.startsWith(root + "/")) { + path = path.substring(root.length()); + } + } + return path.substring(1); + } + + private static String trimSlash(String root) { + return root.endsWith("/") && root.length() > 1 ? + root.substring(0, root.length() - 1) : root; + } + + // mountinfo writes space, tab, newline and backslash as octal escapes such as \040 + private static String unescape(String field) { + StringBuilder out = new StringBuilder(field.length()); + for (int i = 0; i < field.length(); i++) { + char c = field.charAt(i); + if (c == '\\' && i + 4 <= field.length() && + field.substring(i + 1, i + 4).matches("[0-7]{3}")) { + out.append((char) Integer.parseInt(field.substring(i + 1, i + 4), 8)); + i += 3; + } else { + out.append(c); + } + } + return out.toString(); + } + + private static boolean visible(Path point, String relative) { + Path dir = point.resolve(relative).normalize(); + return dir.startsWith(point) && Files.isDirectory(dir); + } + + private static long v2Limit(Path root, String relative) { + Path dir = root.resolve(relative).normalize(); + if (!dir.startsWith(root) || !Files.isDirectory(dir)) { + // The process path is not visible from this mount (a cgroup namespace) + dir = root; + } + long limit = -1L; + for (Path d = dir; d != null && d.startsWith(root); d = d.getParent()) { + limit = lowest(limit, readLimit(d.resolve("memory.max"))); + } + return limit; + } + + private static long v1Limit(Path base, String relative) { + Path dir = base.resolve(relative).normalize(); + if (!dir.startsWith(base) || !Files.isDirectory(dir)) { + dir = base; + } + long limit = readLimit(dir.resolve("memory.limit_in_bytes")); + try { + for (String line : Files.readAllLines(dir.resolve("memory.stat"), + StandardCharsets.UTF_8)) { + if (line.startsWith("hierarchical_memory_limit ")) { + limit = lowest(limit, parseLimit(line.substring(line.indexOf(' ') + 1))); + } + } + } catch (IOException | RuntimeException e) { + // memory.stat is optional here, memory.limit_in_bytes already counted + } + return limit; + } + + private static long readLimit(Path file) { + try { + return parseLimit(new String(Files.readAllBytes(file), StandardCharsets.UTF_8)); + } catch (IOException | RuntimeException e) { + return -1L; + } + } + + // "max" (v2) and the near-Long.MAX_VALUE page-aligned v1 default mean no limit + private static long parseLimit(String value) { + String trimmed = value.trim(); + if (trimmed.isEmpty() || trimmed.equals("max")) { + return -1L; + } + try { + long limit = Long.parseLong(trimmed); + return limit <= 0 || limit >= Long.MAX_VALUE / 2 ? -1L : limit; + } catch (NumberFormatException e) { + return -1L; + } + } + + private static long lowest(long a, long b) { + if (a <= 0) { + return b; + } + return b <= 0 ? a : Math.min(a, b); + } +} diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RaftRocksdbOptions.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RaftRocksdbOptions.java index 2386a1f227..072eab9d16 100644 --- a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RaftRocksdbOptions.java +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RaftRocksdbOptions.java @@ -43,6 +43,11 @@ @Slf4j public class RaftRocksdbOptions { + // Block cache shared by all raft log DBs; RocksdbMemoryBudget reserves it outside the budget + static final long RAFT_LOG_CACHE_SIZE = 1024L * 1024L * 1024L; + // Memtables of all raft log DBs, charged to that cache + static final long RAFT_LOG_WRITE_BUFFER_SIZE = 256L * 1024L * 1024L; + private static RocksdbConfig rocksdbConfig = null; private static boolean raftRocksdbConfigRegistered = false; @@ -66,7 +71,7 @@ private static void registerRaftRocksdbConfig(HugeConfig options) { return; } - Cache blockCache = new LRUCache(SizeUnit.GB); + Cache blockCache = new LRUCache(RAFT_LOG_CACHE_SIZE); BlockBasedTableConfig tableConfig = new BlockBasedTableConfig() .setIndexType(IndexType.kTwoLevelIndexSearch) .setPartitionFilters(true) // @@ -82,12 +87,7 @@ private static void registerRaftRocksdbConfig(HugeConfig options) { StorageOptionsFactory.registerRocksDBTableFormatConfig(RocksDBLogStorage.class, tableConfig); - DBOptions dbOptions = StorageOptionsFactory.getDefaultRocksDBOptions(); - dbOptions.setEnv(rocksdbConfig.getEnv()); - - // raft rocksdb number is fixed, can be controlled by max_write_buffer_number - //dbOptions.setWriteBufferManager(rocksdbConfig.getBufferManager()); - dbOptions.setUnorderedWrite(true); + DBOptions dbOptions = raftLogDBOptions(rocksdbConfig.getEnv(), blockCache); StorageOptionsFactory.registerRocksDBOptions(RocksDBLogStorage.class, dbOptions); @@ -107,6 +107,23 @@ private static void registerRaftRocksdbConfig(HugeConfig options) { } } + /** + * Options for the raft log DBs. JRaft gives every log storage a copy of these, and a + * copy keeps the same WriteBufferManager, so all groups share one memtable cap. + */ + static DBOptions raftLogDBOptions(Env env, Cache blockCache) { + DBOptions dbOptions = StorageOptionsFactory.getDefaultRocksDBOptions(); + dbOptions.setEnv(env); + // Each partition's raft group opens its own log DB, so without a shared manager the + // log memtables grow with the partition count. Charging them to the log block cache + // makes them take room from it, and RocksdbMemoryBudget already reserves that cache, + // instead of adding to it + dbOptions.setWriteBufferManager(new WriteBufferManager(RAFT_LOG_WRITE_BUFFER_SIZE, + blockCache)); + dbOptions.setUnorderedWrite(true); + return dbOptions; + } + public static void initRocksdbGlobalConfig(Map config) { HugeConfig hugeConfig = BusinessHandlerImpl.initRocksdb(config, null); RocksdbConfig rocksdbConfig = getRocksdbConfig(hugeConfig); diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java new file mode 100644 index 0000000000..01fcbb4309 --- /dev/null +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudget.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.options; + +import java.lang.management.ManagementFactory; +import java.math.BigDecimal; +import java.math.RoundingMode; + +import com.sun.management.HotSpotDiagnosticMXBean; +import com.sun.management.OperatingSystemMXBean; + +import lombok.extern.slf4j.Slf4j; + +/** + * Resolves rocksdb.total_memory_size, the budget RaftRocksdbOptions splits into the + * write buffer manager's cache and the block cache. Both caches are off-heap, so "auto" + * sizes the budget from the memory left after the heap, direct memory and the reserve, + * instead of a fixed number that can exceed a container limit. + */ +@Slf4j +public final class RocksdbMemoryBudget { + + public static final String KEY = "total_memory_size"; + public static final String AUTO = "auto"; + + static final long MIB = 1024L * 1024L; + static final long GIB = 1024L * MIB; + // The raft log block cache, which the raft log memtables are charged to + static final long RAFT_LOG_RESERVE = RaftRocksdbOptions.RAFT_LOG_CACHE_SIZE; + // Thread stacks, metaspace and other native allocations outside the budget + static final long NATIVE_RESERVE = 512 * MIB; + static final long RESERVED = RAFT_LOG_RESERVE + NATIVE_RESERVE; + static final long MIN_AUTO = 256 * MIB; + // RaftRocksdbOptions gives the write buffer manager 2/3 of the budget, and a 0-byte + // write buffer manager means no memtable limit at all + static final long MIN_EXPLICIT = MIB; + // The value shipped before auto-sizing; a large host never gets more than it had + static final long MAX_AUTO = 32_000_000_000L; + + private RocksdbMemoryBudget() { + } + + /** + * Returns the configured budget in bytes, or an automatic one for "auto". A missing, + * blank or 0 value also means auto, as before "auto" existed. + */ + public static long resolve(String configured) { + long explicit = parse(configured); + long heap = Runtime.getRuntime().maxMemory(); + long direct = maxDirectMemory(heap); + long jvm = jvmMemory(); + CgroupMemoryLimit cgroup = CgroupMemoryLimit.detect(); + long physical = effectivePhysical(jvm, cgroup.bytes); + long left = physical - heap - direct - RESERVED; + long budget = explicit > 0 ? explicit : auto(physical, heap, direct); + + log.info("RocksDB memory budget: {} ({} bytes), mode={}, physical-limit={}, " + + "source={}, heap={}, direct={}, reserve={}, cgroup-view={}", + format(budget), budget, explicit > 0 ? "explicit" : AUTO, format(physical), + source(jvm, cgroup), format(heap), format(direct), format(RESERVED), + cgroup.incomplete ? "incomplete" : "complete"); + if (explicit > 0) { + if (explicit == MAX_AUTO) { + // Most likely an application-pd.yml kept from before auto-sizing + log.warn("rocksdb.total_memory_size={} is the fixed value shipped before " + + "auto-sizing; set it to auto to size the budget from the available " + + "memory", explicit); + } else if (explicit < MIN_AUTO) { + log.warn("rocksdb.total_memory_size={} is below the {} floor auto-sizing " + + "never goes under", format(explicit), format(MIN_AUTO)); + } + if (physical > 0 && explicit > left) { + log.warn("rocksdb.total_memory_size={} exceeds the {} left after the heap, " + + "direct memory and reserve; the process may be OOM-killed", + format(explicit), format(Math.max(left, 0L))); + } + return budget; + } + if (cgroup.incomplete) { + log.warn("This process can't see its whole cgroup hierarchy (a cgroup namespace " + + "or subtree-only cgroup mounts), so a tighter memory limit above it may " + + "be missed; set rocksdb.total_memory_size or " + + "HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE to a number of bytes if one applies"); + } + if (physical > 0 && left < 2 * MIN_AUTO) { + log.warn("Heap, direct memory and the {} reserve leave too little of the {} " + + "physical limit, RocksDB gets the {} floor; lower -Xmx or " + + "-XX:MaxDirectMemorySize", format(RESERVED), format(physical), + format(MIN_AUTO)); + } + return budget; + } + + /** + * The explicit budget in bytes, or 0 for auto: "auto", a missing or blank value, or 0. + */ + static long parse(String configured) { + String value = configured == null ? "" : configured.trim(); + if (value.isEmpty() || value.equalsIgnoreCase(AUTO)) { + return 0L; + } + long bytes; + try { + bytes = Long.parseLong(value); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(String.format( + "rocksdb.total_memory_size must be auto or a number of bytes, got '%s'", + configured), e); + } + if (bytes < 0) { + throw new IllegalArgumentException(String.format( + "rocksdb.total_memory_size must be auto or a number of bytes, got '%s'", + configured)); + } + if (bytes > 0 && bytes < MIN_EXPLICIT) { + throw new IllegalArgumentException(String.format( + "rocksdb.total_memory_size must be auto or at least %s bytes, got %s", + MIN_EXPLICIT, bytes)); + } + return bytes; + } + + /** + * Half of what is left after the heap, direct memory and the reserve, clamped to + * [MIN_AUTO, MAX_AUTO]. Falls back to the heap size, the previous default, when + * physical memory is unknown. + */ + static long auto(long physical, long heap, long direct) { + if (physical <= 0) { + return Math.min(Math.max(heap, MIN_AUTO), MAX_AUTO); + } + long available = physical - heap - direct - RESERVED; + return Math.min(Math.max(available / 2, MIN_AUTO), MAX_AUTO); + } + + /** + * The smaller of what the JVM reports and the cgroup limit, ignoring unknown (<= 0) + * values. + */ + static long effectivePhysical(long jvm, long cgroup) { + if (cgroup <= 0) { + return jvm; + } + return jvm <= 0 ? cgroup : Math.min(jvm, cgroup); + } + + // Where the physical limit used by effectivePhysical() came from + static String source(long jvm, CgroupMemoryLimit cgroup) { + if (cgroup.bytes > 0 && (jvm <= 0 || cgroup.bytes <= jvm)) { + return "cgroup-v" + cgroup.version; + } + return jvm > 0 ? "jvm" : "unknown"; + } + + /** + * The direct memory the budget leaves room for: -XX:MaxDirectMemorySize as given, or, + * when it is unset (0), the heap size, which is what the JVM then allows. + * bin/start-hugegraph-store.sh sets the flag to the heap it picks unless -j overrides + * it, so a large heap with a smaller cap should pass that cap explicitly. + */ + static long directMemory(long configured, long heap) { + return configured > 0 ? configured : heap; + } + + // Bytes in binary units for the logs, such as 1.5 GiB or 512 MiB + static String format(long bytes) { + if (bytes < 0) { + return "unknown"; + } + if (bytes >= GIB) { + return scaled(bytes, GIB) + " GiB"; + } + if (bytes >= MIB) { + return scaled(bytes, MIB) + " MiB"; + } + return bytes + " B"; + } + + private static String scaled(long bytes, long unit) { + return BigDecimal.valueOf(bytes).divide(BigDecimal.valueOf(unit), 2, RoundingMode.HALF_UP) + .stripTrailingZeros().toPlainString(); + } + + private static long jvmMemory() { + try { + // Reports the cgroup v1/v2 memory limit inside a container, but older JDK 17 + // updates may not see a limit set only on a parent cgroup + OperatingSystemMXBean os = ManagementFactory.getPlatformMXBean( + OperatingSystemMXBean.class); + return os == null ? -1L : os.getTotalMemorySize(); + } catch (RuntimeException | LinkageError e) { + log.warn("Failed to read physical memory size", e); + return -1L; + } + } + + private static long maxDirectMemory(long heap) { + try { + HotSpotDiagnosticMXBean hotspot = ManagementFactory.getPlatformMXBean( + HotSpotDiagnosticMXBean.class); + return directMemory(Long.parseLong( + hotspot.getVMOption("MaxDirectMemorySize").getValue()), heap); + } catch (RuntimeException | LinkageError e) { + log.warn("Failed to read MaxDirectMemorySize, assuming it is unset", e); + return directMemory(0L, heap); + } + } +} diff --git a/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh b/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh index 8ea9022a33..a31f341fdc 100755 --- a/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh +++ b/hugegraph-store/hg-store-dist/docker/docker-entrypoint.sh @@ -54,6 +54,22 @@ require_env "HG_STORE_RAFT_ADDRESS" : "${HG_STORE_GRPC_PORT:=8500}" : "${HG_STORE_REST_PORT:=8520}" : "${HG_STORE_DATA_PATH:=/hugegraph-store/storage}" +: "${HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE:=}" + +# auto or a number of bytes: the Store refuses to start on anything else +if [[ -n "${HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE}" && + ! "${HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE}" =~ ^([0-9]+|auto)$ ]]; then + echo "ERROR: HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE must be auto or a number of bytes," \ + "got '${HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE}'" >&2 + exit 2 +fi + +# Only emitted when set: SPRING_APPLICATION_JSON outranks conf/application-pd.yml +ROCKSDB_JSON="" +if [[ -n "${HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE}" ]]; then + ROCKSDB_JSON=", + \"rocksdb\": { \"total_memory_size\": \"${HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE}\" }" +fi # ── Build SPRING_APPLICATION_JSON ───────────────────────────────────── SPRING_APPLICATION_JSON="$(cat <}" ./bin/start-hugegraph-store.sh -d false -j "${JAVA_OPTS:-}" diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh index d1ccc651c7..41844e7e04 100755 --- a/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/start-hugegraph-store.sh @@ -169,7 +169,28 @@ if [ "$JAVA_OPTIONS" = "" ]; then >> ${OUTPUT} exit 1 fi - JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m -XX:MetaspaceSize=256M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${USER_OPTION} " + # Cap direct memory at the heap size picked here, the JVM's own unset default, so the + # RocksDB auto-sizing (RocksdbMemoryBudget) counts a limit the JVM actually enforces. + # Leave it out when -j sets the heap or the cap itself, or JAVA_TOOL_OPTIONS or + # JDK_JAVA_OPTIONS sets the cap: a fixed cap from the computed heap would no longer match + # a -j -Xmx, and the JVM then defaults it to that heap. + DIRECT_MEMORY_OPTION="-XX:MaxDirectMemorySize=${XMX}m" + # Split -j the way it reaches the JVM below (unquoted: spaces, tabs and newlines) + for opt in ${USER_OPTION}; do + case "$opt" in + -Xmx* | -XX:MaxHeapSize=* | -XX:MaxDirectMemorySize=*) DIRECT_MEMORY_OPTION="" ;; + esac + done + # The JVM reads JAVA_TOOL_OPTIONS and JDK_JAVA_OPTIONS before the command line, so a cap + # there would lose to this one; keep it instead. The JVM drops quotes anywhere in a token + # there, so drop them too. A false match, such as the text inside a quoted -D value, only + # leaves the cap to the JVM, whose default is this same heap. + ENV_JVM_OPTIONS=" ${JAVA_TOOL_OPTIONS:-} ${JDK_JAVA_OPTIONS:-} " + ENV_JVM_OPTIONS="${ENV_JVM_OPTIONS//[\"\']/}" + case "${ENV_JVM_OPTIONS//[$'\t\r\n']/ }" in + *" -XX:MaxDirectMemorySize="*) DIRECT_MEMORY_OPTION="" ;; + esac + JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m ${DIRECT_MEMORY_OPTION} -XX:MetaspaceSize=256M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${USER_OPTION} " # JAVA_OPTIONS="-Xms${MIN_MEM}m -Xmx${XMX}m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${LOGS} ${USER_OPTION}" # Rolling out detailed GC logs diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml index 56cbdd8e63..73d7498bf3 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml +++ b/hugegraph-store/hg-store-dist/src/assembly/static/conf/application-pd.yml @@ -28,17 +28,10 @@ management: rocksdb: # Match the explicit launcher opt-in when enabling Topling. provider: rocksdb - # rocksdb total memory usage, force flush to disk when reaching this value - # TODO: default this lower, or size it from the container memory limit, or map it from an - # HG_STORE_* variable in the Docker entrypoint. A default of 0 only makes AppConfig fall back - # to the JVM max heap, which is another heap-sized native budget, not a safe container total. - # It is the capacity of the RocksDB block and write LRU caches (RaftRocksdbOptions), not an - # upfront allocation, but pinned at 32 GB those native caches can grow past a smaller - # container memory limit under load and get the Store OOM-killed. Until then the only - # override is the JVM system property -Drocksdb.total_memory_size= (JAVA_OPTS, the - # Helm chart's store.javaOpts; helm/hugegraph). - # https://github.com/apache/hugegraph/issues/3254 - total_memory_size: 32000000000 + # Off-heap memory for rocksdb memtables and block cache: auto, or a number of bytes. + # auto sizes it from the memory left after heap, direct memory and a reserve + # (container limit aware, capped at 32 GB). + total_memory_size: auto # memtable size used by rocksdb write_buffer_size: 32000000 # For each rocksdb, the number of memtables reaches this value for writing to disk. diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java index e55bba090e..ad60d00780 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java @@ -23,6 +23,7 @@ import javax.annotation.PostConstruct; import org.apache.hugegraph.store.options.JobOptions; +import org.apache.hugegraph.store.options.RocksdbMemoryBudget; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -104,23 +105,37 @@ public void init() { } Map rocksdb = rocksdbConfig.rocksdb; - if (!rocksdb.containsKey("total_memory_size") - || "0".equals(rocksdb.get("total_memory_size"))) { - rocksdb.put("total_memory_size", Long.toString(rt.maxMemory())); - } - long totalMemory = Long.parseLong(rocksdbConfig.rocksdb.get("total_memory_size")); + long totalMemory = RocksdbMemoryBudget.resolve(rocksdb.get(RocksdbMemoryBudget.KEY)); + rocksdb.put(RocksdbMemoryBudget.KEY, Long.toString(totalMemory)); if (raft.getDisruptorBufferSize() == 0) { - int size = (int) (totalMemory / 1000 / 1000 / 1000); - size = (int) Math.pow(2, Math.round(Math.log(size) / Math.log(2))) * 32; - raft.setDisruptorBufferSize(size); // Increase one buffer every 32M + raft.setDisruptorBufferSize(disruptorBufferSize(totalMemory)); } if (!rocksdb.containsKey("write_buffer_size") || "0".equals(rocksdb.get("write_buffer_size"))) { - rocksdb.put("write_buffer_size", Long.toString(totalMemory / 1000)); + rocksdb.put("write_buffer_size", Long.toString(writeBufferSize(totalMemory))); } } + /** + * The raft disruptor buffer size when raft.disruptorBufferSize is 0: one buffer every + * 32M, from the RocksDB budget in GB, at least 1, rounded to a power of two. + */ + static int disruptorBufferSize(long totalMemory) { + // A budget under 1 GB would otherwise give log(0) and a zero-sized buffer + int size = (int) Math.max(1, totalMemory / 1000 / 1000 / 1000); + return (int) Math.pow(2, Math.round(Math.log(size) / Math.log(2))) * 32; + } + + /** + * The write_buffer_size when the key is unset or 0: a thousandth of the RocksDB budget, + * but never below the 1 MiB rocksdb.write_buffer_size accepts, which a budget under + * 1 GiB would otherwise fall under. + */ + static long writeBufferSize(long totalMemory) { + return Math.max(totalMemory / 1000, 1024L * 1024L); + } + @Override public String toString() { StringBuilder builder = new StringBuilder(); diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/PdConfigureListener.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/PdConfigureListener.java index 070d3c15ee..9ca6d0e043 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/PdConfigureListener.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/listener/PdConfigureListener.java @@ -39,6 +39,7 @@ import org.springframework.context.ConfigurableApplicationContext; import org.springframework.core.env.MutablePropertySources; import org.springframework.core.env.PropertiesPropertySource; +import org.springframework.core.env.PropertySource; import org.yaml.snakeyaml.LoaderOptions; import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.constructor.SafeConstructor; @@ -55,6 +56,12 @@ public class PdConfigureListener implements private static final String CONFIG_FIX_PREFIX = "S:FS"; private static final String TIMESTAMP_KEY = "S:Timestamp"; private static final String PD_CONFIG_FILE_NAME = "application-pd.yml"; + static final String MEMORY_KEY = "rocksdb.total_memory_size"; + // Where an operator sets one Store's RocksDB budget: the Docker entrypoint's + // SPRING_APPLICATION_JSON (HG_STORE_ROCKSDB_TOTAL_MEMORY_SIZE) or + // -Drocksdb.total_memory_size + private static final String[] MEMORY_OVERRIDE_SOURCES = {"spring.application.json", + "systemProperties"}; private final String workDir = System.getProperty("user.dir"); private final String fileSeparator = System.getProperty("file.separator"); private final String configFilePath = @@ -121,11 +128,29 @@ private void updatePdConfig(MutablePropertySources sources, KvClient client, if (!pdConfFile.exists() || pdConfFile.lastModified() <= pdLastModified) { log.info("update local application-pd.yml from pd....{}", pdConfig); writeYml(pdConfig); + dropOverriddenMemory(configs, sources); PropertiesPropertySource source = new PropertiesPropertySource("pd-config", configs); sources.addFirst(source); } } + /** + * The PD-managed configuration is shared by every Store and goes in front of all + * other property sources, so a rocksdb.total_memory_size in it would shadow the + * budget an operator set for this Store. When one is set, leave that key out of the + * PD source and let Spring's own precedence pick it; nothing is written back to PD. + */ + static void dropOverriddenMemory(Properties pdConfigs, MutablePropertySources sources) { + for (String name : MEMORY_OVERRIDE_SOURCES) { + PropertySource source = sources.get(name); + Object value = source == null ? null : source.getProperty(MEMORY_KEY); + if (value != null && !value.toString().trim().isEmpty()) { + pdConfigs.remove(MEMORY_KEY); + return; + } + } + } + private Properties getYmlConfig(String yml) { Yaml yaml = new Yaml(new SafeConstructor(new LoaderOptions())); Iterable load = yaml.loadAll(yml); diff --git a/hugegraph-store/hg-store-node/src/main/resources/application-pd.yml b/hugegraph-store/hg-store-node/src/main/resources/application-pd.yml index 7985e67da1..abd11a6be6 100644 --- a/hugegraph-store/hg-store-node/src/main/resources/application-pd.yml +++ b/hugegraph-store/hg-store-node/src/main/resources/application-pd.yml @@ -33,6 +33,8 @@ logging: level: root: info rocksdb: - # total memory size used by rocksdb - total_memory_size: 32000000000 + # Off-heap memory for rocksdb memtables and block cache: auto, or a number of bytes. + # auto sizes it from the memory left after heap, direct memory and a reserve + # (container limit aware, capped at 32 GB). + total_memory_size: auto write_buffer_size: 32000000 diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java index e2b9819bcb..3ff11f9af8 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java @@ -21,6 +21,11 @@ import org.apache.hugegraph.store.StoreIdChangeTest; import org.apache.hugegraph.store.core.snapshot.HgSnapshotHandlerTest; import org.apache.hugegraph.store.core.store.meta.PartitionMetaCodecTest; +import org.apache.hugegraph.store.node.AppConfigTuningTest; +import org.apache.hugegraph.store.node.listener.PdConfigureListenerMemoryOverrideTest; +import org.apache.hugegraph.store.options.CgroupMemoryLimitTest; +import org.apache.hugegraph.store.options.RaftRocksdbOptionsTest; +import org.apache.hugegraph.store.options.RocksdbMemoryBudgetTest; import org.apache.hugegraph.store.util.FileObjectIteratorTest; import org.apache.hugegraph.store.util.FilterIteratorCleanupTest; import org.apache.hugegraph.store.util.FilterIteratorColumnEqualityTest; @@ -65,7 +70,12 @@ MetricSessionTest.class, HgSnapshotHandlerTest.class, StoreIdChangeTest.class, - PartitionMetaCodecTest.class + PartitionMetaCodecTest.class, + RocksdbMemoryBudgetTest.class, + CgroupMemoryLimitTest.class, + AppConfigTuningTest.class, + RaftRocksdbOptionsTest.class, + PdConfigureListenerMemoryOverrideTest.class }) @Slf4j public class CoreSuiteTest { diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/node/AppConfigTuningTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/node/AppConfigTuningTest.java new file mode 100644 index 0000000000..3dad2d0c28 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/node/AppConfigTuningTest.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.hugegraph.config.ConfigException; +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.config.OptionSpace; +import org.apache.hugegraph.rocksdb.access.RocksDBOptions; +import org.junit.Test; + +/** + * The raft disruptor and write buffer sizes AppConfig derives from the RocksDB budget. + */ +public class AppConfigTuningTest { + + private static final long MIB = 1024L * 1024L; + + @Test + public void testDisruptorBufferSizeIsNeverZero() { + assertEquals(32, AppConfig.disruptorBufferSize(512 * MIB)); + assertEquals(32, AppConfig.disruptorBufferSize(1_000_000_000L)); + assertEquals(1024, AppConfig.disruptorBufferSize(32_000_000_000L)); + } + + @Test + public void testDerivedWriteBufferSizePassesRocksdbValidation() { + // The 4 GiB container from apache/hugegraph#3254 auto-sizes to 512 MiB, whose + // thousandth is under the 1 MiB rocksdb.write_buffer_size minimum + long budget = 512 * MIB; + assertEquals(MIB, AppConfig.writeBufferSize(budget)); + assertEquals(MIB, AppConfig.writeBufferSize(MIB)); + assertEquals(32_000_000L, AppConfig.writeBufferSize(32_000_000_000L)); + + // The same validation BusinessHandlerImpl.initRocksdb() runs at Store startup + OptionSpace.register("rocksdb", RocksDBOptions.class.getName()); + RocksDBOptions.instance(); + HugeConfig config = new HugeConfig(rocksdbConfig(budget, + AppConfig.writeBufferSize(budget))); + assertEquals(MIB, (long) config.get(RocksDBOptions.MEMTABLE_SIZE)); + try { + new HugeConfig(rocksdbConfig(budget, budget / 1000)); + fail("accepted a write buffer under 1 MiB"); + } catch (ConfigException e) { + assertTrue(e.getMessage().contains("write_buffer_size")); + } + } + + private static Map rocksdbConfig(long totalMemory, long writeBufferSize) { + Map config = new HashMap<>(); + config.put("rocksdb.total_memory_size", String.valueOf(totalMemory)); + config.put("rocksdb.write_buffer_size", String.valueOf(writeBufferSize)); + return config; + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/node/listener/PdConfigureListenerMemoryOverrideTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/node/listener/PdConfigureListenerMemoryOverrideTest.java new file mode 100644 index 0000000000..d6c37a91fc --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/node/listener/PdConfigureListenerMemoryOverrideTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node.listener; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import org.junit.Test; +import org.springframework.boot.context.properties.bind.Bindable; +import org.springframework.boot.context.properties.bind.Binder; +import org.springframework.core.env.MapPropertySource; +import org.springframework.core.env.MutablePropertySources; +import org.springframework.core.env.PropertiesPropertySource; +import org.springframework.core.env.StandardEnvironment; + +/** + * PdConfigureListener puts the PD-managed application-pd.yml in front of every other + * property source. An explicit per-Store rocksdb.total_memory_size, from the Docker + * entrypoint's SPRING_APPLICATION_JSON or -D, must still win when RocksdbConfig binds, + * while the rest of the PD configuration keeps its precedence. + */ +public class PdConfigureListenerMemoryOverrideTest { + + private static final String PD_BUDGET = "32000000000"; + private static final String PD_WRITE_BUFFER = "64000000"; + + @Test + public void testDockerOverrideWinsOverPdConfig() { + Map rocksdb = bind(environment(json("268435456"), null)); + assertEquals("268435456", rocksdb.get("total_memory_size")); + // Other PD-managed keys keep their precedence + assertEquals(PD_WRITE_BUFFER, rocksdb.get("write_buffer_size")); + } + + @Test + public void testDockerAutoOverrideWinsOverPdConfig() { + assertEquals("auto", bind(environment(json("auto"), null)).get("total_memory_size")); + } + + @Test + public void testSystemPropertyOverrideWinsOverPdConfig() { + assertEquals("536870912", + bind(environment(null, "536870912")).get("total_memory_size")); + // SPRING_APPLICATION_JSON outranks -D, as without PD configuration + assertEquals("268435456", bind(environment(json("268435456"), "536870912")) + .get("total_memory_size")); + } + + @Test + public void testWithoutOverridePdConfigStillApplies() { + Map rocksdb = bind(environment(null, null)); + assertEquals(PD_BUDGET, rocksdb.get("total_memory_size")); + assertEquals(PD_WRITE_BUFFER, rocksdb.get("write_buffer_size")); + } + + @Test + public void testOnlyTheMemoryKeyIsDropped() { + StandardEnvironment env = environment(json("268435456"), null); + Properties pd = pdConfig(); + PdConfigureListener.dropOverriddenMemory(pd, env.getPropertySources()); + assertFalse(pd.containsKey(PdConfigureListener.MEMORY_KEY)); + assertEquals(PD_WRITE_BUFFER, pd.getProperty("rocksdb.write_buffer_size")); + } + + @Test + public void testReapplyingKeepsOnePdSource() { + StandardEnvironment env = environment(json("268435456"), null); + applyPdConfig(env); + int size = env.getPropertySources().size(); + // A PD config change re-runs updatePdConfig on the same sources, and the new + // pd-config replaces the old one + applyPdConfig(env); + assertEquals(size, env.getPropertySources().size()); + assertEquals("268435456", env.getProperty(PdConfigureListener.MEMORY_KEY)); + } + + private static Map bind(StandardEnvironment env) { + applyPdConfig(env); + return Binder.get(env).bind("rocksdb", Bindable.mapOf(String.class, String.class)) + .orElseGet(HashMap::new); + } + + // What updatePdConfig() does with the PD copy + private static void applyPdConfig(StandardEnvironment env) { + MutablePropertySources sources = env.getPropertySources(); + Properties pd = pdConfig(); + PdConfigureListener.dropOverriddenMemory(pd, sources); + sources.addFirst(new PropertiesPropertySource("pd-config", pd)); + } + + /** + * The sources as the listener sees them: SPRING_APPLICATION_JSON, system properties and + * the local yml, in Spring Boot's order. + */ + private static StandardEnvironment environment(MapPropertySource json, + String systemProperty) { + StandardEnvironment env = new StandardEnvironment(); + MutablePropertySources sources = env.getPropertySources(); + // Keep the real JVM and process environment out of the test + sources.remove(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME); + sources.remove(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME); + Map local = new HashMap<>(); + local.put(PdConfigureListener.MEMORY_KEY, "auto"); + local.put("rocksdb.write_buffer_size", "32000000"); + sources.addLast(new MapPropertySource("applicationConfig: [application-pd.yml]", local)); + if (systemProperty != null) { + Map props = new HashMap<>(); + props.put(PdConfigureListener.MEMORY_KEY, systemProperty); + sources.addFirst(new MapPropertySource( + StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME, props)); + } + if (json != null) { + sources.addFirst(json); + } + return env; + } + + private static Properties pdConfig() { + Properties pd = new Properties(); + pd.put(PdConfigureListener.MEMORY_KEY, PD_BUDGET); + pd.put("rocksdb.write_buffer_size", PD_WRITE_BUFFER); + return pd; + } + + private static MapPropertySource json(String budget) { + Map json = new HashMap<>(); + json.put(PdConfigureListener.MEMORY_KEY, budget); + json.put("app.data-path", "/hugegraph-store/storage"); + return new MapPropertySource("spring.application.json", json); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/CgroupMemoryLimitTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/CgroupMemoryLimitTest.java new file mode 100644 index 0000000000..79f508a737 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/CgroupMemoryLimitTest.java @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.options; + +import static org.apache.hugegraph.store.options.RocksdbMemoryBudget.GIB; +import static org.apache.hugegraph.store.options.RocksdbMemoryBudget.MIB; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.commons.io.FileUtils; +import org.junit.Test; + +public class CgroupMemoryLimitTest { + + private static final Path NO_MOUNTINFO = Path.of("/nonexistent/proc/self/mountinfo"); + + @Test + public void testAncestorCgroupV2LimitIsUsed() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2"); + try { + // A 4 GiB limit on the pod, none on the container or above the pod + write(root, "kubepods/memory.max", "max"); + write(root, "kubepods/pod1/memory.max", String.valueOf(4 * GIB)); + write(root, "kubepods/pod1/ctr/memory.max", "max\n"); + Path proc = write(root, "proc-self-cgroup", "0::/kubepods/pod1/ctr\n"); + long limit = CgroupMemoryLimit.read(proc, NO_MOUNTINFO, root).bytes; + assertEquals(4 * GIB, limit); + assertEquals(2, CgroupMemoryLimit.read(proc, NO_MOUNTINFO, root).version); + + // The case from review: a 64 GiB host reported by the JVM, -Xmx1g and + // -XX:MaxDirectMemorySize=512m get the 4 GiB limit's budget, not host RAM's + long physical = RocksdbMemoryBudget.effectivePhysical(64 * GIB, limit); + assertEquals(512 * MIB, RocksdbMemoryBudget.auto(physical, GIB, 512 * MIB)); + + // The container's own lower limit wins over the pod's + write(root, "kubepods/pod1/ctr/memory.max", String.valueOf(2 * GIB)); + assertEquals(2 * GIB, CgroupMemoryLimit.read(proc, NO_MOUNTINFO, root).bytes); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testCgroupV2NamespaceRootAndNoLimit() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2"); + try { + // Inside a cgroup namespace the process sees "/" and the mount is its cgroup + Path proc = write(root, "proc-self-cgroup", "0::/\n"); + write(root, "memory.max", "max"); + assertEquals(-1L, CgroupMemoryLimit.read(proc, NO_MOUNTINFO, root).bytes); + write(root, "memory.max", String.valueOf(3 * GIB)); + assertEquals(3 * GIB, CgroupMemoryLimit.read(proc, NO_MOUNTINFO, root).bytes); + + // A path the mount doesn't show falls back to the mount root + Path hidden = write(root, "proc-hidden", "0::/system.slice/hidden.service\n"); + assertEquals(3 * GIB, CgroupMemoryLimit.read(hidden, NO_MOUNTINFO, root).bytes); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testCgroupV1HierarchicalLimitIsUsed() throws IOException { + Path root = Files.createTempDirectory("cgroup-v1"); + try { + Path proc = write(root, "proc-self-cgroup", + "12:cpu,cpuacct:/docker/abc\n4:memory:/docker/abc\n"); + // Docker mounts the container's own memory cgroup, so /docker/abc isn't there + write(root, "memory/memory.limit_in_bytes", "9223372036854771712"); + write(root, "memory/memory.stat", + "cache 0\nhierarchical_memory_limit " + (4 * GIB) + "\n"); + assertEquals(4 * GIB, CgroupMemoryLimit.read(proc, NO_MOUNTINFO, root).bytes); + assertEquals(1, CgroupMemoryLimit.read(proc, NO_MOUNTINFO, root).version); + + write(root, "memory/memory.stat", "hierarchical_memory_limit 9223372036854771712\n"); + assertEquals(-1L, CgroupMemoryLimit.read(proc, NO_MOUNTINFO, root).bytes); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testCgroupV2SubtreeMountIsRebased() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2-subtree"); + try { + // cgroupfs mounted from /kubepods: the pod's limit is at /pod1, not at + // /kubepods/pod1, and the mount root itself is unlimited + Path mount = root.resolve("sys-fs-cgroup"); + write(mount, "memory.max", "max"); + write(mount, "pod1/memory.max", String.valueOf(4 * GIB)); + write(mount, "pod1/ctr/memory.max", "max"); + Path proc = write(root, "proc-self-cgroup", "0::/kubepods/pod1/ctr\n"); + Path mountinfo = write(root, "mountinfo", + "25 1 8:1 / / rw - ext4 /dev/sda1 rw\n" + + "30 25 0:26 /kubepods " + mount + + " rw,nosuid shared:4 - cgroup2 cgroup2 rw\n"); + long limit = CgroupMemoryLimit.read(proc, mountinfo, root).bytes; + assertEquals(4 * GIB, limit); + long physical = RocksdbMemoryBudget.effectivePhysical(64 * GIB, limit); + assertEquals(512 * MIB, RocksdbMemoryBudget.auto(physical, GIB, 512 * MIB)); + // Only a subtree is mounted, so anything above /kubepods is invisible + assertTrue(CgroupMemoryLimit.read(proc, mountinfo, root).incomplete); + + // Without the mount root the path is looked up under kubepods/ and missed + assertEquals(-1L, CgroupMemoryLimit.read(proc, NO_MOUNTINFO, mount).bytes); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testStricterLimitAboveCgroupV2SubtreeMount() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2-above-subtree"); + try { + // /outer is limited to 2 GiB, the pod to 4 GiB, kubepods and the container not + Path full = root.resolve("full"); + write(full, "outer/memory.max", String.valueOf(2 * GIB)); + write(full, "outer/kubepods/memory.max", "max"); + write(full, "outer/kubepods/pod1/memory.max", String.valueOf(4 * GIB)); + write(full, "outer/kubepods/pod1/ctr/memory.max", "max"); + // The same hierarchy mounted from /outer/kubepods hides the 2 GiB on /outer + Path sub = root.resolve("sub"); + write(sub, "memory.max", "max"); + write(sub, "pod1/memory.max", String.valueOf(4 * GIB)); + write(sub, "pod1/ctr/memory.max", "max"); + Path proc = write(root, "proc-self-cgroup", "0::/outer/kubepods/pod1/ctr\n"); + String fullLine = "31 25 0:26 / " + full + " rw - cgroup2 cgroup2 rw\n"; + String subLine = "32 25 0:26 /outer/kubepods " + sub + + " rw - cgroup2 cgroup2 rw\n"; + + // Subtree only: the 2 GiB limit above the mount is missed, and that is flagged + Path subOnly = write(root, "mountinfo-sub", subLine); + CgroupMemoryLimit limit = + CgroupMemoryLimit.read(proc, subOnly, root); + assertEquals(4 * GIB, limit.bytes); + assertTrue(limit.incomplete); + + // With a readable full-hierarchy view in either order the 2 GiB is found + for (String order : new String[]{subLine + fullLine, fullLine + subLine}) { + Path mountinfo = write(root, "mountinfo", order); + limit = CgroupMemoryLimit.read(proc, mountinfo, root); + assertEquals(order, 2 * GIB, limit.bytes); + assertFalse(order, limit.incomplete); + } + + // A "/" mount that doesn't show the process's path doesn't count as a full view + Path empty = root.resolve("empty-full"); + write(empty, "memory.max", "max"); + Path unreadable = write(root, "mountinfo-unreadable", subLine + + "33 25 0:26 / " + empty + " rw - cgroup2 cgroup2 rw\n"); + limit = CgroupMemoryLimit.read(proc, unreadable, root); + assertEquals(4 * GIB, limit.bytes); + assertTrue(limit.incomplete); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testCgroupV1SubtreeMountAndEscapedMountPoint() throws IOException { + Path root = Files.createTempDirectory("cgroup-v1-subtree"); + try { + Path mount = root.resolve("memory dir"); + write(mount, "memory.limit_in_bytes", "9223372036854771712"); + write(mount, "memory.stat", "hierarchical_memory_limit " + (4 * GIB) + "\n"); + Path proc = write(root, "proc-self-cgroup", "4:memory:/docker/abc\n"); + // mountinfo writes the space in the mount point as \040 + Path mountinfo = write(root, "mountinfo", + "40 25 0:30 /docker/abc " + + mount.toString().replace(" ", "\\040") + + " rw,nosuid - cgroup cgroup rw,memory\n"); + assertEquals(4 * GIB, CgroupMemoryLimit.read(proc, mountinfo, root).bytes); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testCgroupV2MountOrderDoesNotHideAncestorLimit() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2-mounts"); + try { + // A full view of the hierarchy with the 4 GiB limit on the pod + Path full = root.resolve("full"); + write(full, "kubepods/pod1/memory.max", String.valueOf(4 * GIB)); + write(full, "kubepods/pod1/ctr/memory.max", "max"); + // A leaf-only view of the container and an unrelated subtree, both unlimited + Path leaf = root.resolve("leaf"); + write(leaf, "memory.max", "max"); + Path other = root.resolve("other"); + write(other, "memory.max", "max"); + Path proc = write(root, "proc-self-cgroup", "0::/kubepods/pod1/ctr\n"); + String fullLine = "31 25 0:26 / " + full + " rw - cgroup2 cgroup2 rw\n"; + String leafLine = "32 25 0:26 /kubepods/pod1/ctr " + leaf + " rw - cgroup2 cgroup2 rw\n"; + String otherLine = "33 25 0:26 /system.slice " + other + " rw - cgroup2 cgroup2 rw\n"; + for (String order : new String[]{otherLine + leafLine + fullLine, + fullLine + leafLine + otherLine, + leafLine + otherLine + fullLine}) { + Path mountinfo = write(root, "mountinfo", order); + long limit = CgroupMemoryLimit.read(proc, mountinfo, root).bytes; + assertEquals(order, 4 * GIB, limit); + long physical = RocksdbMemoryBudget.effectivePhysical(64 * GIB, limit); + assertEquals(512 * MIB, RocksdbMemoryBudget.auto(physical, GIB, 512 * MIB)); + } + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testCgroupV1MountOrderDoesNotHideAncestorLimit() throws IOException { + Path root = Files.createTempDirectory("cgroup-v1-mounts"); + try { + Path full = root.resolve("full"); + write(full, "kubepods/pod1/ctr/memory.limit_in_bytes", "9223372036854771712"); + write(full, "kubepods/pod1/ctr/memory.stat", + "hierarchical_memory_limit " + (4 * GIB) + "\n"); + Path other = root.resolve("other"); + write(other, "memory.limit_in_bytes", "9223372036854771712"); + Path proc = write(root, "proc-self-cgroup", "4:memory:/kubepods/pod1/ctr\n"); + String fullLine = "41 25 0:30 / " + full + " rw - cgroup cgroup rw,memory\n"; + String otherLine = "42 25 0:30 /system.slice " + other + " rw - cgroup cgroup rw,memory\n"; + // A cpu-only mount is not the memory controller and is ignored + String cpuLine = "43 25 0:31 / " + other + " rw - cgroup cgroup rw,cpu,cpuacct\n"; + for (String order : new String[]{otherLine + cpuLine + fullLine, + fullLine + otherLine + cpuLine}) { + Path mountinfo = write(root, "mountinfo", order); + assertEquals(order, 4 * GIB, + CgroupMemoryLimit.read(proc, mountinfo, root).bytes); + } + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testInheritedNamespaceMountIsFlaggedNotTreatedAsUnlimited() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2-namespace"); + try { + // cgroup_namespaces(7): /proc/self/cgroup says "/" while a cgroup2 mount made + // outside the namespace has root "/..". The namespace root is a child with a + // 2 GiB limit whose name can't be learned from inside; its parent is unlimited + Path mount = root.resolve("sys-fs-cgroup"); + write(mount, "memory.max", "max"); + write(mount, "ns-child/memory.max", String.valueOf(2 * GIB)); + Path proc = write(root, "proc-self-cgroup", "0::/\n"); + Path mountinfo = write(root, "mountinfo", + "35 25 0:26 /.. " + mount + " rw - cgroup2 cgroup2 rw\n"); + CgroupMemoryLimit limit = + CgroupMemoryLimit.read(proc, mountinfo, root); + assertTrue(limit.incomplete); + assertEquals(-1L, limit.bytes); + + // A limit at the visible mount root still bounds the result + write(mount, "memory.max", String.valueOf(8 * GIB)); + limit = CgroupMemoryLimit.read(proc, mountinfo, root); + assertTrue(limit.incomplete); + assertEquals(8 * GIB, limit.bytes); + + // v1 with the same layout + Path v1 = root.resolve("memory"); + write(v1, "memory.limit_in_bytes", "9223372036854771712"); + write(v1, "memory.stat", "hierarchical_memory_limit " + (8 * GIB) + "\n"); + Path proc1 = write(root, "proc-v1", "4:memory:/\n"); + Path mountinfo1 = write(root, "mountinfo-v1", + "45 25 0:30 /.. " + v1 + " rw - cgroup cgroup rw,memory\n"); + limit = CgroupMemoryLimit.read(proc1, mountinfo1, root); + assertTrue(limit.incomplete); + assertEquals(8 * GIB, limit.bytes); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testNamespaceRootViewIsFlagged() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2-ns-root"); + try { + // A private cgroup namespace: the process is at "/", the cgroup2 mount made + // inside the namespace is rooted at "/", and its own memory.max is "max". A + // stricter limit on the parent of the namespace root can't be seen from here + Path mount = root.resolve("sys-fs-cgroup"); + write(mount, "memory.max", "max"); + Path proc = write(root, "proc-self-cgroup", "0::/\n"); + Path mountinfo = write(root, "mountinfo", + "30 25 0:26 / " + mount + " rw - cgroup2 cgroup2 rw\n"); + CgroupMemoryLimit limit = + CgroupMemoryLimit.read(proc, mountinfo, root); + assertEquals(-1L, limit.bytes); + assertTrue(limit.incomplete); + + // A finite limit at the namespace root still applies, but a stricter one on a + // hidden ancestor (say 1 GiB above this 2 GiB) can't be ruled out + write(mount, "memory.max", String.valueOf(2 * GIB)); + limit = CgroupMemoryLimit.read(proc, mountinfo, root); + assertEquals(2 * GIB, limit.bytes); + assertTrue(limit.incomplete); + + // v1 reports the whole hierarchy in hierarchical_memory_limit, so it isn't flagged + Path v1 = root.resolve("memory"); + write(v1, "memory.limit_in_bytes", "9223372036854771712"); + write(v1, "memory.stat", "hierarchical_memory_limit 9223372036854771712\n"); + Path proc1 = write(root, "proc-v1", "4:memory:/\n"); + Path mountinfo1 = write(root, "mountinfo-v1", + "45 25 0:30 / " + v1 + " rw - cgroup cgroup rw,memory\n"); + assertFalse(CgroupMemoryLimit.read(proc1, mountinfo1, root).incomplete); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testMatchedOrMissingMountsAreNotFlagged() throws IOException { + Path root = Files.createTempDirectory("cgroup-v2-flag"); + try { + // A host service: the full hierarchy is mounted and every ancestor is readable + Path mount = root.resolve("sys-fs-cgroup"); + write(mount, "memory.max", "max"); + write(mount.resolve("system.slice"), "memory.max", "max"); + write(mount.resolve("system.slice/hg.service"), "memory.max", + String.valueOf(4 * GIB)); + Path proc = write(root, "proc-self-cgroup", "0::/system.slice/hg.service\n"); + Path mountinfo = write(root, "mountinfo", + "30 25 0:26 / " + mount + " rw - cgroup2 cgroup2 rw\n"); + CgroupMemoryLimit limit = + CgroupMemoryLimit.read(proc, mountinfo, root); + assertFalse(limit.incomplete); + assertEquals(4 * GIB, limit.bytes); + // Without any mountinfo entry the default layout is used and nothing is flagged + assertFalse(CgroupMemoryLimit.read(proc, NO_MOUNTINFO, mount).incomplete); + } finally { + FileUtils.deleteQuietly(root.toFile()); + } + } + + @Test + public void testMountRootContainsMembership() { + assertTrue(CgroupMemoryLimit.contains("/", "/kubepods/pod1/ctr")); + assertTrue(CgroupMemoryLimit.contains("/kubepods", "/kubepods/pod1/ctr")); + assertTrue(CgroupMemoryLimit.contains("/kubepods/", "/kubepods/pod1/ctr")); + assertTrue(CgroupMemoryLimit.contains("/kubepods/pod1/ctr", "/kubepods/pod1/ctr")); + assertFalse(CgroupMemoryLimit.contains("/system.slice", "/kubepods/pod1/ctr")); + assertFalse(CgroupMemoryLimit.contains("/kubepods", "/kubepodsx/pod1")); + } + + @Test + public void testMembershipIsRebasedOntoMountRoot() { + assertEquals("pod1/ctr", CgroupMemoryLimit.rebase("/kubepods/pod1/ctr", "/kubepods")); + assertEquals("pod1/ctr", CgroupMemoryLimit.rebase("/kubepods/pod1/ctr", "/kubepods/")); + assertEquals("", CgroupMemoryLimit.rebase("/kubepods", "/kubepods")); + assertEquals("a/b", CgroupMemoryLimit.rebase("/a/b", "/")); + assertEquals("", CgroupMemoryLimit.rebase("/", "/")); + // A path outside the mount root is kept and then falls back to the mount + assertEquals("other/x", CgroupMemoryLimit.rebase("/other/x", "/kubepods")); + assertEquals("kubepodsx/y", CgroupMemoryLimit.rebase("/kubepodsx/y", "/kubepods")); + } + + private static Path write(Path root, String relative, String content) throws IOException { + Path file = root.resolve(relative); + Files.createDirectories(file.getParent()); + Files.write(file, content.getBytes(StandardCharsets.UTF_8)); + return file; + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RaftRocksdbOptionsTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RaftRocksdbOptionsTest.java new file mode 100644 index 0000000000..7fa7491962 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RaftRocksdbOptionsTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.options; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.commons.io.FileUtils; +import org.junit.Test; +import org.rocksdb.Cache; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.DBOptions; +import org.rocksdb.Env; +import org.rocksdb.LRUCache; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.WriteBufferManager; +import org.rocksdb.util.SizeUnit; + +public class RaftRocksdbOptionsTest { + + @Test + public void testRaftLogMemtablesShareOneManager() throws IOException, RocksDBException { + RocksDB.loadLibrary(); + Path dir = Files.createTempDirectory("raft-log-wbm"); + try (Cache cache = new LRUCache(64 * SizeUnit.MB); + DBOptions registered = RaftRocksdbOptions.raftLogDBOptions(Env.getDefault(), + cache); + // StorageOptionsFactory.getRocksDBOptions hands each log storage a copy like this + DBOptions first = new DBOptions(registered); + DBOptions second = new DBOptions(registered)) { + WriteBufferManager manager = registered.writeBufferManager(); + assertNotNull(manager); + assertSame(manager, first.writeBufferManager()); + assertSame(manager, second.writeBufferManager()); + + // Nothing else uses the cache, so it starts empty + assertEquals(0L, cache.getUsage()); + // A copy opens a working log DB, and its memtable is charged to the cache + try (ColumnFamilyOptions cfOptions = new ColumnFamilyOptions(); + Options options = new Options(first, cfOptions).setCreateIfMissing(true); + RocksDB db = RocksDB.open(options, dir.toString())) { + byte[] key = "k".getBytes(StandardCharsets.UTF_8); + byte[] value = "v".getBytes(StandardCharsets.UTF_8); + db.put(key, value); + assertArrayEquals(value, db.get(key)); + assertTrue(cache.getUsage() > 0L); + } + } finally { + FileUtils.deleteQuietly(dir.toFile()); + } + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java new file mode 100644 index 0000000000..582a9d7784 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/options/RocksdbMemoryBudgetTest.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.options; + +import static org.apache.hugegraph.store.options.RocksdbMemoryBudget.GIB; +import static org.apache.hugegraph.store.options.RocksdbMemoryBudget.MAX_AUTO; +import static org.apache.hugegraph.store.options.RocksdbMemoryBudget.MIB; +import static org.apache.hugegraph.store.options.RocksdbMemoryBudget.MIN_AUTO; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.nio.file.Path; + +import org.junit.Test; + +public class RocksdbMemoryBudgetTest { + + private static final Path NO_MOUNTINFO = Path.of("/nonexistent/proc/self/mountinfo"); + + @Test + public void testExplicitValueIsKept() { + assertEquals(32_000_000_000L, RocksdbMemoryBudget.resolve("32000000000")); + assertEquals(4_000_000_000L, RocksdbMemoryBudget.resolve(" 4000000000 ")); + assertEquals(MIB, RocksdbMemoryBudget.resolve(String.valueOf(MIB))); + } + + @Test + public void testExplicitValueBelowOneMibIsRefused() { + for (String value : new String[]{"1", "2", String.valueOf(MIB - 1)}) { + try { + RocksdbMemoryBudget.resolve(value); + fail("accepted " + value); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("at least")); + } + } + } + + @Test + public void testAutoValuesResolveWithinBounds() { + for (String value : new String[]{null, "", " ", "auto", "0", " 0 ", "00"}) { + long budget = RocksdbMemoryBudget.resolve(value); + assertTrue(String.valueOf(value), budget >= MIN_AUTO && budget <= MAX_AUTO); + } + } + + @Test + public void testNonNumericValueIsRefused() { + for (String value : new String[]{"4GB", "4g", "-1", "1.5e9", "99999999999999999999", "automatic"}) { + try { + RocksdbMemoryBudget.resolve(value); + fail("accepted " + value); + } catch (IllegalArgumentException e) { + assertTrue(e.getMessage().contains("number of bytes")); + } + } + } + + @Test + public void testAutoFitsContainerLimit() { + // The setup from apache/hugegraph#3254: 4 GiB limit, 1 GiB heap, 512 MiB direct + assertEquals(512 * MIB, RocksdbMemoryBudget.auto(4 * GIB, GIB, 512 * MIB)); + assertEquals(2560 * MIB, RocksdbMemoryBudget.auto(8 * GIB, GIB, 512 * MIB)); + } + + @Test + public void testUnsetDirectMemoryCountsAsTheHeap() { + assertEquals(512 * MIB, RocksdbMemoryBudget.directMemory(512 * MIB, 8 * GIB)); + assertEquals(4 * GIB, RocksdbMemoryBudget.directMemory(4 * GIB, 8 * GIB)); + // Unset: the JVM allows as much direct memory as heap, so the budget counts that + assertEquals(8 * GIB, RocksdbMemoryBudget.directMemory(0L, 8 * GIB)); + assertEquals(512 * MIB, RocksdbMemoryBudget.directMemory(0L, 512 * MIB)); + } + + @Test + public void testBareMetalBudgetsWithTheLauncherDirectCap() { + // The default launcher picks a heap of at most 2 GiB and caps direct memory at it + assertEquals(5376 * MIB, RocksdbMemoryBudget.auto(16 * GIB, 2 * GIB, 2 * GIB)); + // A raised heap with a smaller explicit -XX:MaxDirectMemorySize keeps a usable budget + assertEquals(2304 * MIB, RocksdbMemoryBudget.auto(16 * GIB, 8 * GIB, 2 * GIB)); + assertEquals(12544 * MIB, RocksdbMemoryBudget.auto(64 * GIB, 36 * GIB, 2 * GIB)); + // Without one, direct memory counts as a second heap and the floor applies + assertEquals(MIN_AUTO, RocksdbMemoryBudget.auto( + 64 * GIB, 36 * GIB, RocksdbMemoryBudget.directMemory(0L, 36 * GIB))); + } + + @Test + public void testUnreadableCgroupKeepsJvmMemory() { + Path missing = Path.of("/nonexistent/proc/self/cgroup"); + assertEquals(-1L, CgroupMemoryLimit.read(missing, NO_MOUNTINFO, + Path.of("/nonexistent")).bytes); + assertEquals(16 * GIB, RocksdbMemoryBudget.effectivePhysical(16 * GIB, -1L)); + assertEquals(4 * GIB, RocksdbMemoryBudget.effectivePhysical(-1L, 4 * GIB)); + assertEquals(4 * GIB, RocksdbMemoryBudget.effectivePhysical(4 * GIB, 8 * GIB)); + } + + @Test + public void testAutoIsClamped() { + assertEquals(MIN_AUTO, RocksdbMemoryBudget.auto(2 * GIB, GIB, GIB)); + assertEquals(MAX_AUTO, RocksdbMemoryBudget.auto(1024 * GIB, 8 * GIB, 8 * GIB)); + } + + @Test + public void testUnknownPhysicalMemoryFallsBackToHeap() { + assertEquals(2 * GIB, RocksdbMemoryBudget.auto(-1L, 2 * GIB, 2 * GIB)); + assertEquals(MIN_AUTO, RocksdbMemoryBudget.auto(0L, 64 * MIB, 64 * MIB)); + } + + @Test + public void testAutoAliases() { + for (String value : new String[]{null, "", " ", "auto", "AUTO", " auto ", "0", "00"}) { + assertEquals(String.valueOf(value), 0L, RocksdbMemoryBudget.parse(value)); + } + assertEquals(4_000_000_000L, RocksdbMemoryBudget.parse(" 4000000000 ")); + } + + @Test + public void testLogUnits() { + assertEquals("1.5 GiB", RocksdbMemoryBudget.format(RocksdbMemoryBudget.RESERVED)); + assertEquals("512 MiB", RocksdbMemoryBudget.format(512 * MIB)); + assertEquals("4 GiB", RocksdbMemoryBudget.format(4 * GIB)); + assertEquals("29.8 GiB", RocksdbMemoryBudget.format(MAX_AUTO)); + assertEquals("1000 B", RocksdbMemoryBudget.format(1000)); + assertEquals("0 B", RocksdbMemoryBudget.format(0)); + assertEquals("unknown", RocksdbMemoryBudget.format(-1L)); + } + + @Test + public void testReserveIsTheRaftLogCachePlusNativeMemory() { + assertEquals(RaftRocksdbOptions.RAFT_LOG_CACHE_SIZE, RocksdbMemoryBudget.RAFT_LOG_RESERVE); + assertEquals(1536 * MIB, RocksdbMemoryBudget.RESERVED); + } + + @Test + public void testPhysicalLimitSource() { + CgroupMemoryLimit v2 = new CgroupMemoryLimit(4 * GIB, 2, false); + assertEquals("cgroup-v2", RocksdbMemoryBudget.source(64 * GIB, v2)); + assertEquals("cgroup-v2", RocksdbMemoryBudget.source(-1L, v2)); + assertEquals("jvm", RocksdbMemoryBudget.source(2 * GIB, v2)); + assertEquals("jvm", RocksdbMemoryBudget.source(16 * GIB, CgroupMemoryLimit.NONE)); + assertEquals("unknown", RocksdbMemoryBudget.source(-1L, CgroupMemoryLimit.NONE)); + } +}