diff --git a/.dockerignore b/.dockerignore index 28d48eb939..4116b3140f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -59,3 +59,7 @@ apache-hugegraph-*/ **/docker-compose*.yaml **/*.md docs/ +# The Store assembly packages this upgrade guide. +!docs/ +docs/* +!docs/graphid-upgrade.md diff --git a/.github/workflows/pd-store-ci.yml b/.github/workflows/pd-store-ci.yml index e4dd1809ea..7e9a7147d4 100644 --- a/.github/workflows/pd-store-ci.yml +++ b/.github/workflows/pd-store-ci.yml @@ -364,6 +364,9 @@ jobs: run: | mvn editorconfig:check -pl hugegraph-store/hg-store-test -am -ntp + - name: Check Store GraphID checker launcher + run: bash hugegraph-store/hg-store-dist/src/assembly/travis/test-store-check-launcher.sh + - name: Check startup test prerequisites (Store) id: store-preflight run: | diff --git a/docs/graphid-upgrade.md b/docs/graphid-upgrade.md new file mode 100644 index 0000000000..b7dc08840e --- /dev/null +++ b/docs/graphid-upgrade.md @@ -0,0 +1,260 @@ +# HStore GraphID checks before upgrading from 1.7 + +Some 1.7 batch PUT/MERGE paths wrote business keys with the reserved GraphID +`0xFFFE` when no mapping existed. Current Store allocates a real GraphID for +these operations. The allocation can also happen during Raft replay, before a +client sends a request. Existing `0xFFFE` rows then become invisible to reads +through the new mapping. Replacing the Store JAR with 1.7 does not undo a mapping +that has already been persisted. + +This is an HStore upgrade issue. A 1.7 database using only valid allocated +GraphIDs does not need its keys converted for this issue. + +## Check each Store before the first 1.8 startup + +Keep a complete, matching backup of the 1.7 PD, every Store (data **and** Raft +paths), Server configuration, and the original input where available. Quiesce +all writers, take snapshots with 1.7, and stop the Stores before copying their +files. A stopped process alone does not prove that WAL-disabled writes were +flushed; retain the Raft logs together with the data and snapshots. + +From the 1.8 Store distribution, using that Store's effective local configuration: + +```bash +bin/hugegraph-store-check.sh --check +``` + +An explicit configuration directory and multiple data roots are supported: + +```bash +bin/hugegraph-store-check.sh --check \ + --config /srv/hugegraph-store/conf \ + --data-path /srv/store-data-a,/srv/store-data-b +``` + +Run the command with Java 17, selected through `JAVA_HOME` or `java` on `PATH`. +It uses the RocksDB dependencies already in the Store distribution, needs no +running PD, Store or Server, and creates no database. It reads `application.yml` +followed by `application-pd.yml` when present. It does not start Spring or resolve +its environment, JSON or profile overrides. If the deployment uses those +overrides, pass the actual data roots with `--data-path` and point `--config` at +local configuration files containing the resolved effective RocksDB settings. +For example, a Docker deployment's `HG_STORE_DATA_PATH` must be supplied as +`--data-path`; the command does not read the entrypoint's `SPRING_APPLICATION_JSON`. +Relative paths are resolved from the Store distribution, as with its startup scripts. +The checker uses the same verified runtime selector as the Store launcher. For a +Topling deployment, prepare its runtime first, set +`TOPLINGDB_ROCKSDB_PROVIDER=topling`, and retain `rocksdb.provider=topling` in the +effective configuration. Its selected JNI JAR takes precedence over the bundled +standard RocksDB classes; missing or unverified runtime assets fail the check. +Do not point the command at a running Store: it refuses locked active databases, +and a separate read-only database cannot observe another process's memtables. + +| Result | Meaning | Next step | +| --- | --- | --- | +| Exit 0 | No reserved-prefix rows or invalid GraphID mappings were found in the persisted databases and latest snapshots inspected | Keep the backups and follow the normal release upgrade procedure; this check does not certify the entire upgrade | +| Exit 2 with `Legacy GraphID 0xFFFE` | The reported database/column family contains incompatible historical rows | Use the recovery path below before starting 1.8 on this data | +| Exit 2 with an invalid mapping, lock, configuration or I/O error | Inspection could not establish a clean persisted state | Resolve the reported problem and run the check again; do not treat this as a clean result | + +A launcher or JVM failure can return another nonzero status. Treat every nonzero +status as an unsuccessful check. + +Run the check on **every Store**. Users do not need to supply numeric GraphIDs or +partition IDs. The check visits configured data roots, retained DB versions and +the latest local snapshot for each partition. It also considers stored locations +that are not remapped from the configured roots. It seeks the reserved prefix +once per column family and reads GraphID metadata; it does not scan every +business row. Retained versions are inspected without invoking the normal +session opener, which can delete older versions. + +Store startup performs the same read-only inspection before the Store engine, +RocksDB factory, Raft groups and Store registration initialize. On failure the +process exits unsuccessfully instead of leaving a partially initialized Store +advertising healthy partitions. The earlier configuration listener can still +contact PD or synchronize local configuration. The guarantee here concerns the +Store databases and registration, not all configuration activity. + +Incoming snapshots are checked before opening or replacing the local partition +database. A rejected snapshot does not advance the state machine's committed +index. Local snapshots are also checked at startup: a `should_not_load` marker +only skips local installation and does not stop a checkpoint reaching another +replica. + +### Incomplete database directories + +`Incomplete RocksDB directory: ` means the reported directory has no +regular `CURRENT` file. This is distinct from finding a reserved GraphID row. +For example, interruption while the normal opener removes an old `db/00000` +can leave that directory incomplete beside a valid `db/00000_42`. The read-only +checker cannot certify the leftover directory and does not delete or skip it. + +An existing `hgstore-metadata` path must also have a regular `CURRENT` file. +This database records partition locations, so an incomplete metadata directory +or a dangling link cannot be treated as a new Store. Preserve it and recover a +complete matching backup; do not move it aside to make the check pass. A new +Store with no metadata path is still allowed. + +Keep the Store stopped and preserve a complete backup before changing any +directory. Check the exact reported path against the partition's stored location, +the previous Store logs and all sibling versions. A larger numeric suffix alone +does not establish that another directory is the correct live database. + +If an operator has independently confirmed that the reported path is only an +obsolete, partly deleted version and has verified the selected database, move +that one leftover directory to a backed-up quarantine location outside **every** +configured Store data and Raft root. Keep its files; renaming it within a data +root is not a recovery check. Run the offline command again on every Store with +the effective configuration and data roots before attempting startup. + +Do not use that procedure for a selected/latest database, a current snapshot or +a directory whose role is uncertain. Do not fabricate a `CURRENT` file. Preserve +the evidence and recover a complete matching backup or follow the independently +verified logical recovery route below. This manual procedure does not make the +checker automatically distinguish harmless cleanup remnants from damaged data. + +## Choose the recovery source first + +Do not infer that every `0xFFFE` row belongs to the graph named in the Server +configuration. Different graphs, including the automatically created +`DEFAULT/~sys_graph/g`, could share the same physical namespace. System variables +can occupy vertex and index tables too. Reassigning the entire prefix to one +graph can silently assign data to the wrong owner. + +| Historical state | Supported action | +| --- | --- | +| Valid GraphID mappings and no incompatible rows | No GraphID-specific data conversion | +| A trusted 1.7 source still returns all expected application data | Validate a logical export and restore it into a **new, empty** 1.8 cluster | +| New and old prefixes coexist, including after replay, writes or truncate | Preserve the affected cluster. Recover a matching trusted pre-upgrade source or original input first; do not union the prefixes automatically | +| Several graphs shared the prefix | Validate each graph's ownership and complete application data against independent input/backups before exporting. Refuse an automatic reassignment when ownership is uncertain | +| A physical key was overwritten or deleted | No mapping or prefix conversion can recover the lost value without another copy | + +An empty new prefix, or equal values at currently colliding keys, does not prove +that combining prefixes is safe: an earlier delete or truncate can make a merge +resurrect data the application already removed. A disk inspection cannot prove +historical ownership or the absence of those operations. + +If 1.8 has already opened the data, do not use JAR rollback as a recovery +procedure. Bring up a **copy** of the matching 1.7 backup with its original +configuration in an isolated recovery environment. Verify the source through +fresh requests, not a Server cache retained across the upgrade. Keep the +original backup immutable. + +## Logical migration with existing HugeGraph Tools + +For a validated, readable application graph, the existing Tools `backup` and +`restore` commands avoid editing RocksDB keys, allocator metadata, Raft logs or +snapshots. Restore uses the normal Server/Store API and Raft replication. The +target cluster creates its own mappings, replicas and snapshots. Use independent +PD and Store data paths and endpoints for the empty target; never attach an empty +target to the source PD or reuse the source Store identity. + +The example below is for a graph in the **DEFAULT GraphSpace**, without special +state outside the export scope described below. Use a HugeGraph Tools +release compatible with both source and target. +Keep application writers stopped until export and validation finish. Keep target +application traffic disabled until all checks pass. + +Use new, empty output and log directories for **each attempt**. Reusing a backup +directory can overwrite an earlier export and retain stale shard files from it. +`restore --clean` deletes the local backup; it is deliberately absent below. + +```bash +# Run from the HugeGraph Tools distribution. Set these for the isolated clusters. +SOURCE_URL=http://source-server:8080 +TARGET_URL=http://target-server:8080 +GRAPH=hugegraph +BACKUP=/srv/upgrade/backup-attempt-1 +BACKUP_LOG=/srv/upgrade/backup-log-attempt-1 +RESTORE_LOG=/srv/upgrade/restore-log-attempt-1 +VERIFY_BACKUP=/srv/upgrade/verify-attempt-1 +VERIFY_LOG=/srv/upgrade/verify-log-attempt-1 + +bin/hugegraph --url "$SOURCE_URL" --graph "$GRAPH" backup \ + --directory "$BACKUP" --log "$BACKUP_LOG" --huge-types all \ + --format json --compress false --thread-num 1 + +# Verify the source export and failure logs before continuing (see below). +bin/hugegraph --url "$TARGET_URL" --graph "$GRAPH" graph-mode-set \ + --graph-mode RESTORING + +bin/hugegraph --url "$TARGET_URL" --graph "$GRAPH" restore \ + --directory "$BACKUP" --log "$RESTORE_LOG" --huge-types all --thread-num 1 + +bin/hugegraph --url "$TARGET_URL" --graph "$GRAPH" backup \ + --directory "$VERIFY_BACKUP" --log "$VERIFY_LOG" --huge-types all \ + --format json --compress false --thread-num 1 + +# Only after complete comparison, query checks and restart verification: +bin/hugegraph --url "$TARGET_URL" --graph "$GRAPH" graph-mode-set \ + --graph-mode NONE +``` + +Adjust the worker count for the deployment after establishing correctness. +The example uses a single worker to make a small validation run +straightforward; it is not a throughput recommendation. + +### Validate completeness before cutover + +A zero exit status is necessary but insufficient. The tested Tools version can +log a failed backup shard and still exit successfully. Check its separate +vertices/edges `failed_shards`, `timeout_shards` and `limit_exceed_shards` files; +any nonempty such file means the export is incomplete. Preserve the logs and +start a new attempt in new directories after resolving the error. + +Compare the source export with independent expected data, then compare source +and target exports. Include all four schema types and their IDs; vertex IDs, +labels and properties; edge IDs, labels, endpoints, direction and properties; +record counts and ID uniqueness. Ignore transport-only schema controls and +normalize unordered sets, but preserve ordered list properties. `RESTORING` +retains schema IDs; the tool can recompute primary-key vertex IDs from the label +ID and key properties, so verify IDs explicitly. + +Run application queries that exercise indexes, adjacency, filters and graph +isolation. Repeat the read checks after a cold target restart and verify a new +write. On replicated deployments, verify failover and snapshot installation on a +recovering replica before accepting the target. Counts alone are not an adequate +migration oracle. + +If restore fails or is interrupted, leave target application traffic disabled. +For this upgrade, use another empty target or recreate only the disposable target +from its initial empty state and rerun the complete restore from the retained, +verified export. Do not merge an uncertain partial restore into the source. A +failed command can leave the target in `RESTORING`; inspect its mode explicitly. +Keep source backups through the acceptance period. Once applications write to the +target, returning to the source requires reconciling those writes; it is no +longer a file or JAR rollback. + +### Export scope and release limits + +Tools exports property keys, vertex labels, edge labels, index labels, vertices +and edges. This is not a full Store/PD/system-state backup. Authentication data +has separate Tools commands. Graph variables, system KV, task state, OLAP state +and operational configuration need a separate inventory and recovery procedure. +Do not restore the shared historical system namespace by assigning it to the +user graph. + +The inspected Tools CLI addresses DEFAULT GraphSpace and has no `--graphspace` +option. Other GraphSpaces require an explicitly GraphSpace-aware API/client +export and a separately verified restore. Do not substitute a graphspace-prefixed +graph name in these commands. TTL expiration timestamps are another boundary: +restoring a TTL label without `ttl_start_time` can restart its lifetime. A plain +logical export must not be presented as preserving those original timestamps. + +Deployments needing these features must validate their additional recovery path +before upgrading. Shared-prefix ownership that cannot be established and lost +rows without backups remain recovery blockers. The preflight intentionally has +no override that silently assigns ambiguous data. + +## Background and compatibility + +The legacy allocation issue is documented in +[apache/hugegraph#3224](https://github.com/apache/hugegraph/issues/3224) +and [#3095](https://github.com/apache/hugegraph/issues/3095). + +The issue author's [reference data](https://github.com/SebastianGruza/hugegraph-validation) +is available with the original report. + +This fix adds no second read/write format, GraphID allocator rule, Raft command +or recurring scan to normal requests. It does not establish general rolling +upgrade/rollback compatibility for the release; see +[apache/hugegraph#3223](https://github.com/apache/hugegraph/issues/3223). diff --git a/hugegraph-store/README.md b/hugegraph-store/README.md index 531ff49079..679ffc88b0 100644 --- a/hugegraph-store/README.md +++ b/hugegraph-store/README.md @@ -85,6 +85,10 @@ For detailed architecture, Raft consensus mechanisms, and partition management, --- +## Upgrading from 1.7 + +Before starting 1.8 on existing HStore data, follow the [GraphID upgrade checks and recovery guide](../docs/graphid-upgrade.md). The Store distribution includes a read-only `bin/hugegraph-store-check.sh --check` command. + ## Quick Start ### Prerequisites diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/BusinessHandlerImpl.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/BusinessHandlerImpl.java index 65e97b5ab4..1a42d5c0e7 100644 --- a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/BusinessHandlerImpl.java +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/business/BusinessHandlerImpl.java @@ -26,6 +26,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -115,6 +116,7 @@ import org.apache.hugegraph.store.schema.SchemaGraph; import org.apache.hugegraph.store.term.Bits; import org.apache.hugegraph.store.term.HgPair; +import org.apache.hugegraph.store.upgrade.LegacyGraphIdChecker; import org.apache.hugegraph.store.util.DefaultThreadFactory; import org.apache.hugegraph.store.util.ExecutorUtil; import org.apache.hugegraph.store.util.HgStoreException; @@ -122,6 +124,7 @@ import org.apache.hugegraph.util.Bytes; import org.rocksdb.Cache; import org.rocksdb.MemoryUsageType; +import org.rocksdb.RocksDBException; import com.alipay.sofa.jraft.util.Utils; import com.google.protobuf.ByteString; @@ -1194,6 +1197,17 @@ public void saveSnapshot(String snapshotPath, String graph, int partId) throws @Override public void loadSnapshot(String snapshotPath, String graph, int partId, long v1) throws HgStoreException { + // Validate incoming data before getSession() can create or prune local DB versions. + try { + LegacyGraphIdChecker.checkDatabase(Path.of(snapshotPath), factory.getHugeConfig()); + } catch (IOException | RocksDBException | IllegalStateException e) { + HgStoreException failure = new HgStoreException( + HgStoreException.EC_RKDB_IMPORT_SNAPSHOT_FAIL, + "GraphID upgrade preflight rejected snapshot %s for partition %d: %s", + snapshotPath, partId, e.getMessage()); + failure.initCause(e); + throw failure; + } try (RocksDBSession dbSession = getSession(graph, partId)) { dbSession.loadSnapshot(snapshotPath, v1); keyCreator.clearCache(partId); diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/PartitionStateMachine.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/PartitionStateMachine.java index cf74810aec..1ba2e7399c 100644 --- a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/PartitionStateMachine.java +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/raft/PartitionStateMachine.java @@ -222,11 +222,11 @@ public void onSnapshotSave(final SnapshotWriter writer, final Closure done) { @Override public boolean onSnapshotLoad(final SnapshotReader reader) { + final long snapshotIndex; try { RaftOutter.SnapshotMeta meta = reader.load(); if (meta != null) { - this.committedIndex = meta.getLastIncludedIndex(); - log.info("onSnapshotLoad committedIndex = {}", this.committedIndex); + snapshotIndex = meta.getLastIncludedIndex(); } else { log.error("onSnapshotLoad failed to get SnapshotMeta"); return false; @@ -241,8 +241,9 @@ public boolean onSnapshotLoad(final SnapshotReader reader) { return false; } try { - snapshotHandler.onSnapshotLoad(reader, this.committedIndex); - log.info("Raft {} onSnapshotLoad success", groupId); + snapshotHandler.onSnapshotLoad(reader, snapshotIndex); + this.committedIndex = snapshotIndex; + log.info("Raft {} onSnapshotLoad success, committedIndex = {}", groupId, snapshotIndex); return true; } catch (HgStoreException e) { log.error(String.format("Raft %s onSnapshotLoad failed. ", groupId), e); diff --git a/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/upgrade/LegacyGraphIdChecker.java b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/upgrade/LegacyGraphIdChecker.java new file mode 100644 index 0000000000..25fe8f3721 --- /dev/null +++ b/hugegraph-store/hg-store-core/src/main/java/org/apache/hugegraph/store/upgrade/LegacyGraphIdChecker.java @@ -0,0 +1,328 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.upgrade; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.config.OptionSpace; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.rocksdb.access.RocksDBOptions; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.meta.GraphIdManager; +import org.apache.hugegraph.store.meta.MetadataKeyHelper; +import org.apache.hugegraph.store.util.HgStoreException; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.DBOptions; +import org.rocksdb.InfoLogLevel; +import org.rocksdb.Logger; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; + +import com.google.protobuf.Int64Value; + +/** Read-only inspection before Store registration or an incoming snapshot is opened for writing. */ +public final class LegacyGraphIdChecker { + + private static final byte[] LEGACY = {(byte) 0xFF, (byte) 0xFE}; + private static final Pattern DB_NAME = Pattern.compile("([0-9]{5,})(?:_[0-9]+){0,2}"); + private static final Pattern SNAPSHOT_NAME = Pattern.compile("snapshot_([0-9]+)"); + + private LegacyGraphIdChecker() { + } + + public static HugeConfig configuration(Map options) { + OptionSpace.register("rocksdb", RocksDBOptions.class.getName()); + return new HugeConfig(options); + } + + public static void checkStore(String dataPaths, HugeConfig config) { + try { + inspect(dataPaths, config, false); + } catch (IOException | RocksDBException | RuntimeException e) { + throw new StartupPreflightException("GraphID upgrade preflight refused Store startup: " + + e.getMessage() + ". Preserve the complete 1.7 backup. Run " + + "bin/hugegraph-store-check.sh --check while the Store is stopped; " + + "see docs/graphid-upgrade.md", e); + } + } + + /** Identifies the startup refusal across Spring's initialization wrappers. */ + public static final class StartupPreflightException extends HgStoreException { + + private static final long serialVersionUID = 1L; + + private StartupPreflightException(String message, Throwable cause) { + super(message, cause); + } + } + + /** A running Store may have unflushed writes with WAL disabled: never certify its disk view. */ + public static void checkOffline(String dataPaths, HugeConfig config) throws IOException, RocksDBException { + inspect(dataPaths, config, true); + } + + public static void checkDatabase(Path path, HugeConfig config) throws IOException, RocksDBException { + try (Database database = new Database(path, config)) { + for (int i = 0; i < database.names.size(); i++) { + try (RocksIterator iterator = database.db.newIterator(database.handles.get(i))) { + iterator.seek(LEGACY); + if (iterator.isValid() && startsWith(iterator.key(), LEGACY)) { + throw new IllegalStateException("Legacy GraphID 0xFFFE in " + path + + ", column family " + new String(database.names.get(i), StandardCharsets.UTF_8)); + } + iterator.status(); + } + } + // Also reject a historical manual FFFE mapping whose business rows have been removed. + Map ids = new TreeMap<>(); + byte[] prefix = MetadataKeyHelper.getGraphIDKey(""); + try (RocksIterator iterator = database.db.newIterator()) { + for (iterator.seek(prefix); iterator.isValid() && startsWith(iterator.key(), prefix); + iterator.next()) { + byte[] key = iterator.key(); + byte[] name = Arrays.copyOfRange(key, prefix.length, key.length); + String graph = new String(name, StandardCharsets.UTF_8); + // Replacement characters can collapse distinct damaged keys and hide an invalid ID. + if (!Arrays.equals(name, graph.getBytes(StandardCharsets.UTF_8))) { + throw new IllegalStateException("Invalid UTF-8 GraphID metadata in " + path); + } + ids.put(graph, Int64Value.parseFrom(iterator.value()).getValue()); + } + iterator.status(); + } + try { + GraphIdManager.checkGraphIds(ids, -1); + } catch (HgStoreException e) { + throw new IllegalStateException("Invalid GraphID metadata in " + path + ": " + e.getMessage(), e); + } + } + } + + private static void inspect(String dataPaths, HugeConfig config, boolean offline) + throws IOException, RocksDBException { + List roots = new ArrayList<>(); + for (String value : dataPaths.split(",", -1)) { + if (value.isBlank()) { + throw new IOException("Empty Store data path"); + } + Path root = Path.of(value).toAbsolutePath().normalize(); + if (offline && !Files.isDirectory(root)) { + throw new IOException("Expected an existing stopped Store data directory: " + root); + } + roots.add(root); + } + try (DatabaseLocks locks = new DatabaseLocks()) { + Path metadata = roots.get(0).resolve("hgstore-metadata"); + boolean metadataExists = Files.exists(metadata, LinkOption.NOFOLLOW_LINKS); + if (metadataExists) { + requireCurrent(metadata); + locks.add(metadata); + } + Set databases = new LinkedHashSet<>(); + Set snapshots = new LinkedHashSet<>(); + Set discovered = new HashSet<>(); + for (Path root : roots) { + discover(root, null, databases, snapshots, discovered); + } + if (metadataExists) { + try (Database database = new Database(metadata, config); + RocksIterator iterator = database.db.newIterator()) { + byte[] prefix = MetadataKeyHelper.getPartitionStorePrefix(); + for (iterator.seek(prefix); iterator.isValid() && startsWith(iterator.key(), prefix); + iterator.next()) { + Metapb.PartitionStore location = Metapb.PartitionStore.parseFrom(iterator.value()); + // Normal moved roots are remapped by PartitionManager.resetPartitionPath(). + if (!discovered.contains(location.getPartitionId())) { + discover(Path.of(location.getStoreLocation()), location.getPartitionId(), + databases, snapshots, new HashSet<>()); + } + } + iterator.status(); + } + } + for (Path database : databases) { + requireCurrent(database); + locks.add(database); + checkDatabase(database, config); + } + for (Path snapshot : snapshots) { + checkDatabase(snapshot, config); + } + } + } + + private static void discover(Path root, Integer onlyPartition, Set databases, + Set snapshots, Set discovered) throws IOException { + for (Path path : children(root.resolve("db"))) { + Matcher match = DB_NAME.matcher(path.getFileName().toString()); + if (Files.isDirectory(path) && match.matches()) { + int partition = Integer.parseInt(match.group(1)); + if (onlyPartition == null || onlyPartition == partition) { + // Inspect retained versions without invoking findLatestDBPath(), which deletes older DBs. + databases.add(path.toRealPath()); + discovered.add(partition); + } + } + } + // Snapshots live on the data disk even when Raft logs use a separate disk. + for (Path partition : children(root.resolve("raft"))) { + if (!partition.getFileName().toString().matches("[0-9]{5,}") || + onlyPartition != null && Integer.parseInt(partition.getFileName().toString()) != onlyPartition) { + continue; + } + long latestIndex = -1L; + Path latest = null; + for (Path candidate : children(partition.resolve("snapshot"))) { + Matcher match = SNAPSHOT_NAME.matcher(candidate.getFileName().toString()); + if (match.matches()) { + long index = Long.parseLong(match.group(1)); + if (index > latestIndex) { + latest = candidate; + latestIndex = index; + } + } + } + if (latest != null) { + // Local should_not_load does not prevent this checkpoint being sent to a new replica. + snapshots.add(latest.resolve("data").toAbsolutePath().normalize()); + } + } + } + + private static List children(Path parent) throws IOException { + if (!Files.exists(parent)) { + return List.of(); + } + try (var stream = Files.list(parent)) { + return stream.sorted().toList(); + } + } + + private static void requireCurrent(Path path) throws IOException { + if (!Files.isRegularFile(path.resolve("CURRENT"))) { + throw new IOException("Incomplete RocksDB directory: " + path + + ". CURRENT is missing or not a regular file. Preserve this directory; see " + + "docs/graphid-upgrade.md#incomplete-database-directories"); + } + } + + private static boolean startsWith(byte[] value, byte[] prefix) { + return value.length >= prefix.length && Arrays.equals(value, 0, prefix.length, prefix, 0, prefix.length); + } + + private static final class Database implements AutoCloseable { + + private final DBOptions options = new DBOptions(); + private final ColumnFamilyOptions familyOptions = new ColumnFamilyOptions(); + private final List handles = new ArrayList<>(); + private final Logger logger = new Logger(this.options) { + @Override + protected void log(InfoLogLevel level, String message) { + // Read-only open must not create or rotate LOG files in the inspected database. + } + }; + private List names; + private RocksDB db; + + private Database(Path path, HugeConfig config) throws IOException, RocksDBException { + try { + requireCurrent(path); + RocksDBSession.initOptions(config, null, null, this.familyOptions, this.familyOptions); + this.options.setCreateIfMissing(false).setCreateMissingColumnFamilies(false).setLogger(this.logger); + try (Options listing = new Options().setLogger(this.logger)) { + this.names = RocksDB.listColumnFamilies(listing, path.toString()); + } + List descriptors = new ArrayList<>(); + for (byte[] name : this.names) { + descriptors.add(new ColumnFamilyDescriptor(name, this.familyOptions)); + } + this.db = RocksDB.openReadOnly(this.options, path.toString(), descriptors, this.handles); + } catch (IOException | RocksDBException | RuntimeException | Error e) { + this.close(); + throw e; + } + } + + @Override + public void close() { + this.handles.forEach(ColumnFamilyHandle::close); + if (this.db != null) { + this.db.close(); + } + this.familyOptions.close(); + this.logger.close(); + this.options.close(); + } + } + + private static final class DatabaseLocks implements AutoCloseable { + + private final List channels = new ArrayList<>(); + private final Set paths = new HashSet<>(); + + private void add(Path database) throws IOException { + Path path = database.resolve("LOCK").toRealPath(); + if (!this.paths.add(path)) { + return; + } + FileChannel channel = FileChannel.open(path, StandardOpenOption.READ); + this.channels.add(channel); + FileLock lock = channel.tryLock(0L, Long.MAX_VALUE, true); + if (lock == null) { + throw new IOException("Store is running or database is locked: " + database); + } + } + + @Override + public void close() throws IOException { + IOException failure = null; + for (int i = this.channels.size() - 1; i >= 0; i--) { + try { + this.channels.get(i).close(); + } catch (IOException e) { + failure = e; + } + } + if (failure != null) { + throw failure; + } + } + } +} diff --git a/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml b/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml index ee2b6af9bf..65d7687c31 100644 --- a/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml +++ b/hugegraph-store/hg-store-dist/src/assembly/descriptor/server-assembly.xml @@ -50,6 +50,11 @@ + + ${project.basedir}/../../docs/graphid-upgrade.md + docs + 644 + ${project.basedir}/../../hugegraph-server/hugegraph-dist/src/assembly/static/bin/ArthasAttach.java bin diff --git a/hugegraph-store/hg-store-dist/src/assembly/static/bin/hugegraph-store-check.sh b/hugegraph-store/hg-store-dist/src/assembly/static/bin/hugegraph-store-check.sh new file mode 100755 index 0000000000..a9e41c8917 --- /dev/null +++ b/hugegraph-store/hg-store-dist/src/assembly/static/bin/hugegraph-store-check.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail + +BIN=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +TOP=$(cd -- "$BIN/.." && pwd) +JAVA=java +if [[ -n "${JAVA_HOME:-}" ]]; then + JAVA="$JAVA_HOME/bin/java" +fi +shopt -s nullglob +JARS=("$TOP"/lib/hg-store-node-*.jar) +if [[ ${#JARS[@]} -ne 1 ]]; then + echo "Expected exactly one hg-store-node JAR in $TOP/lib" >&2 + exit 2 +fi +MODULE_OPTIONS=() +while IFS= read -r option || [[ -n "$option" ]]; do + [[ -z "$option" || "$option" == \#* ]] || MODULE_OPTIONS+=("$option") +done < "$BIN/jvm-module.options" +source "$BIN/preload-topling.sh" +CHECKER_CLASSPATH="${JARS[0]}" +if [[ -n "${TOPLING_RUNTIME_CLASSPATH:-}" ]]; then + CHECKER_CLASSPATH="${TOPLING_RUNTIME_CLASSPATH}:$CHECKER_CLASSPATH" +fi +cd -- "$TOP" +exec "$JAVA" "${MODULE_OPTIONS[@]}" \ + -Dloader.main=org.apache.hugegraph.store.node.GraphIdCheckCommand \ + -cp "$CHECKER_CLASSPATH" org.springframework.boot.loader.PropertiesLauncher "$@" diff --git a/hugegraph-store/hg-store-dist/src/assembly/travis/test-store-check-launcher.sh b/hugegraph-store/hg-store-dist/src/assembly/travis/test-store-check-launcher.sh new file mode 100644 index 0000000000..a06a5c8215 --- /dev/null +++ b/hugegraph-store/hg-store-dist/src/assembly/travis/test-store-check-launcher.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Launcher-selection fixtures; native database inspection is tested separately. +set -Eeuo pipefail +SOURCE_BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")/../static/bin" && pwd)" +REPO_ROOT="$(cd "$SOURCE_BIN/../../../../../.." && pwd)" +SHARED_BIN="$REPO_ROOT/hugegraph-server/hugegraph-dist/src/assembly/static/bin" +TEST_ROOT=$(mktemp -d) +TEST_ROOT=$(cd "$TEST_ROOT" && pwd -P) +trap 'rm -rf "$TEST_ROOT"' EXIT +TOP="$TEST_ROOT/checker distribution" +mkdir -p "$TOP/bin" "$TOP/lib" "$TOP/conf" "$TEST_ROOT/java/bin" "$TEST_ROOT/fake-bin" +cp "$SOURCE_BIN/hugegraph-store-check.sh" "$SHARED_BIN/preload-topling.sh" "$TOP/bin/" +printf '%s\n' '# module options' '--add-opens=java.base/java.lang=ALL-UNNAMED' > "$TOP/bin/jvm-module.options" +touch "$TOP/lib/hg-store-node-test.jar" +cat > "$TEST_ROOT/java/bin/java" <<'JAVA' +#!/bin/bash +printf '%s\0' "$@" > "$CAPTURE.args" +printf '%s\n' "${TOPLING_RUNTIME_CLASSPATH:-}" > "$CAPTURE.runtime" +JAVA +cat > "$TEST_ROOT/fake-bin/uname" <<'UNAME' +#!/bin/bash +case "$1" in -m) echo x86_64 ;; *) echo Linux ;; esac +UNAME +chmod +x "$TEST_ROOT/java/bin/java" "$TEST_ROOT/fake-bin/uname" + +run_checker() { + env -u LD_PRELOAD -u TOPLING_ACTIVE_NATIVE -u TOPLING_RUNTIME_CLASSPATH \ + -u TOPLINGDB_EASY_MIGRATE_CONF -u TOPLING_AUTO_MIGRATE_CONF \ + JAVA_HOME="$TEST_ROOT/java" PATH="$TEST_ROOT/fake-bin:$PATH" \ + TOPLINGDB_ROCKSDB_PROVIDER="$1" CAPTURE="$TEST_ROOT/capture" \ + bash "$TOP/bin/hugegraph-store-check.sh" --check \ + --config "$TOP/conf" --data-path "$TOP/data with spaces" > "$TEST_ROOT/output" 2>&1 +} + +for provider in rocksdb topling; do + expected="$TOP/lib/hg-store-node-test.jar" + if [[ "$provider" == topling ]]; then + mkdir -p "$TOP/topling" + touch "$TOP/topling/rocksdbjni.jar" "$TOP/topling/librocksdbjni-linux64.so" + (cd "$TOP/topling" && sha256sum rocksdbjni.jar librocksdbjni-linux64.so > easy-migrate.sha256) + printf '%s\n' 'http: {auto_start_http: false}' > "$TOP/conf/toplingdb.yaml" + expected="$TOP/topling/rocksdbjni.jar:$expected" + fi + run_checker "$provider" || { cat "$TEST_ROOT/output"; exit 1; } + mapfile -d '' -t args < "$TEST_ROOT/capture.args" + [[ ${#args[@]} -eq 10 ]] + [[ "${args[0]}" == --add-opens=java.base/java.lang=ALL-UNNAMED ]] + [[ "${args[1]}" == -Dloader.main=org.apache.hugegraph.store.node.GraphIdCheckCommand ]] + [[ "${args[2]}" == -cp && "${args[3]}" == "$expected" ]] + [[ "${args[4]}" == org.springframework.boot.loader.PropertiesLauncher ]] + [[ "${args[5]}" == --check && "${args[6]}" == --config && "${args[7]}" == "$TOP/conf" ]] + [[ "${args[8]}" == --data-path && "${args[9]}" == "$TOP/data with spaces" ]] + if [[ "$provider" == topling ]]; then + [[ "$(cat "$TEST_ROOT/capture.runtime")" == "$TOP/topling/rocksdbjni.jar" ]] + else + [[ ! -s "$TEST_ROOT/capture.runtime" || "$(cat "$TEST_ROOT/capture.runtime")" == '' ]] + fi +done + +# Do not start Java with a modified or incompletely prepared provider runtime. +printf 'modified' >> "$TOP/topling/rocksdbjni.jar" +rm "$TEST_ROOT/capture.args" +if run_checker topling; then + echo 'FAIL: unverified checker runtime was accepted' >&2 + exit 1 +fi +grep -q 'unverified EasyMigrate runtime' "$TEST_ROOT/output" +[[ ! -e "$TEST_ROOT/capture.args" ]] +echo 'PASS: checker standard/Topling classpaths, quoted argv, module flags and unverified-runtime refusal' diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/GraphIdCheckCommand.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/GraphIdCheckCommand.java new file mode 100644 index 0000000000..64b0096115 --- /dev/null +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/GraphIdCheckCommand.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import org.apache.hugegraph.store.upgrade.LegacyGraphIdChecker; +import org.apache.hugegraph.util.RocksDBRuntime; +import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; +import org.springframework.core.io.FileSystemResource; + +/** Read-only command in the Store executable JAR. Does not start Spring, PD or a Raft node. */ +public final class GraphIdCheckCommand { + + private static final Set VALUES = Set.of("--config", "--data-path"); + + private GraphIdCheckCommand() { + } + + public static void main(String[] args) { + System.exit(run(args)); + } + + static int run(String... args) { + try { + if (args.length == 1 && "--help".equals(args[0])) { + help(); + return 0; + } + if (args.length == 0) { + throw new IllegalArgumentException( + "Use --check to inspect a stopped Store, or --help"); + } + Map arguments = new HashMap<>(); + for (int i = 0; i < args.length; i++) { + String name = args[i]; + String value = "true"; + if (VALUES.contains(name)) { + if (++i == args.length || args[i].startsWith("--") || args[i].isBlank()) { + throw new IllegalArgumentException("Missing value for " + name); + } + value = args[i]; + } else if (!"--check".equals(name)) { + throw new IllegalArgumentException("Unknown option " + name); + } + if (arguments.putIfAbsent(name, value) != null) { + throw new IllegalArgumentException("Repeated option " + name); + } + } + if (!arguments.containsKey("--check")) { + throw new IllegalArgumentException("Use --check to inspect a stopped Store, or --help"); + } + Properties properties = configuration(Path.of(arguments.getOrDefault("--config", "conf"))); + String data = arguments.getOrDefault("--data-path", System.getProperty("app.data-path", + properties.getProperty("app.data-path"))); + if (data == null || data.isBlank()) { + throw new IllegalArgumentException("Missing app.data-path; supply the effective --data-path"); + } + if (data.contains("${")) { + throw new IllegalArgumentException("Unresolved app.data-path; supply the effective --data-path"); + } + Map options = new HashMap<>(); + properties.forEach((key, value) -> { + if (key.toString().startsWith("rocksdb.")) { + options.put(key.toString(), value.toString()); + } + }); + System.getProperties().forEach((key, value) -> { + if (key.toString().startsWith("rocksdb.")) { + options.put(key.toString(), value.toString()); + } + }); + RocksDBRuntime.verify(String.valueOf(options.getOrDefault("rocksdb.provider", "rocksdb"))); + LegacyGraphIdChecker.checkOffline(data, LegacyGraphIdChecker.configuration(options)); + System.out.println("No legacy GraphID prefix or invalid GraphID mapping found in persisted " + + "databases and latest snapshots: " + data); + System.out.println("This read-only check does not certify historical ownership or backup completeness. " + + "See docs/graphid-upgrade.md before upgrading."); + return 0; + } catch (Exception e) { + System.err.println("GraphID check failed: " + e.getMessage()); + for (Throwable cause = e.getCause(); cause != null; cause = cause.getCause()) { + System.err.println("Caused by: " + cause.getMessage()); + } + System.err.println("No data was converted. Preserve the complete 1.7 backup; see docs/graphid-upgrade.md."); + return 2; + } + } + + private static Properties configuration(Path directory) { + Path application = directory.resolve("application.yml"); + if (!Files.isRegularFile(application)) { + throw new IllegalArgumentException("Missing Store configuration " + application); + } + YamlPropertiesFactoryBean yaml = new YamlPropertiesFactoryBean(); + Path pd = directory.resolve("application-pd.yml"); + yaml.setResources(Files.isRegularFile(pd) ? + new FileSystemResource[]{new FileSystemResource(application), new FileSystemResource(pd)} : + new FileSystemResource[]{new FileSystemResource(application)}); + Properties properties = yaml.getObject(); + return properties == null ? new Properties() : properties; + } + + private static void help() { + System.out.println(""" + Usage: bin/hugegraph-store-check.sh --check [options] + --check Inspect a stopped Store without changing its databases + --config DIR Local application.yml/application-pd.yml (default: conf) + --data-path PATHS Effective app.data-path; comma-separated roots are supported + --help Show this help + + Run once on every stopped Store with its effective configuration, including RocksDB + options. No PD, Server, or running Store is required. Exit 0 means no incompatible + persisted GraphID was found; exit 2 means legacy data, a locked/unreadable database, + or a configuration error. The diagnostic includes the affected path and table. + + The check cannot see another process's unflushed writes or prove that a backup is + complete. Preserve matching 1.7 Store, PD and Server backups before upgrading. + A failed check needs the recovery/upgrade procedure in docs/graphid-upgrade.md. + Do not assign FFFE data to a graph or modify RocksDB keys by hand. + """); + } +} diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/StoreNodeApplication.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/StoreNodeApplication.java index 2f96a34d0c..bc5525b97f 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/StoreNodeApplication.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/StoreNodeApplication.java @@ -17,8 +17,14 @@ package org.apache.hugegraph.store.node; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.Set; + import org.apache.hugegraph.store.node.listener.PdConfigureListener; +import org.apache.hugegraph.store.upgrade.LegacyGraphIdChecker; import org.apache.hugegraph.util.RocksDBRuntime; +import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.ConfigurableApplicationContext; @@ -32,7 +38,27 @@ public class StoreNodeApplication { public static void main(String[] args) { - start(args); + try { + start(args); + } catch (RuntimeException e) { + if (!isGraphIdPreflightFailure(e)) { + throw e; + } + LoggerFactory.getLogger(StoreNodeApplication.class) + .error("GraphID preflight refused Store startup; exiting with status 1", e); + // An incomplete Spring bootstrap can leave non-daemon native/Raft executors alive. + System.exit(1); + } + } + + static boolean isGraphIdPreflightFailure(Throwable failure) { + Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); + for (Throwable cause = failure; cause != null && visited.add(cause); cause = cause.getCause()) { + if (cause instanceof LegacyGraphIdChecker.StartupPreflightException) { + return true; + } + } + return false; } public static void start() { diff --git a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java index 93268cd64f..a4ae5df3c0 100644 --- a/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java +++ b/hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/grpc/HgStoreNodeService.java @@ -39,6 +39,7 @@ import org.apache.hugegraph.store.raft.RaftClosure; import org.apache.hugegraph.store.raft.RaftOperation; import org.apache.hugegraph.store.raft.RaftTaskHandler; +import org.apache.hugegraph.store.upgrade.LegacyGraphIdChecker; import org.apache.hugegraph.store.util.HgRaftError; import org.apache.hugegraph.store.util.HgStoreException; import org.springframework.beans.factory.annotation.Autowired; @@ -122,6 +123,8 @@ public void init() { }}; RocksDBRuntime.verify(String.valueOf(options.getRocksdbConfig().getOrDefault("rocksdb.provider", "rocksdb"))); + LegacyGraphIdChecker.checkStore(options.getDataPath(), + LegacyGraphIdChecker.configuration(options.getRocksdbConfig())); RaftRocksdbOptions.initRocksdbGlobalConfig(options.getRocksdbConfig()); options.getLabels().put("rest.port", Integer.toString(appConfig.getRestPort())); @@ -233,7 +236,9 @@ public boolean invoke(int partId, byte methodId, Object req, RaftClosure respons @PreDestroy public void destroy() { - storeEngine.shutdown(); + if (storeEngine != null) { + storeEngine.shutdown(); + } } private String getSerializingExceptionMessage(String target) { diff --git a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java index c206cb88ed..21c722b828 100644 --- a/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java +++ b/hugegraph-store/hg-store-rocksdb/src/main/java/org/apache/hugegraph/rocksdb/access/RocksDBSession.java @@ -697,6 +697,7 @@ private boolean verifySnapshot(String snapshotPath) { try { try (final Options options = new Options(); final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions()) { + RocksDBSession.initOptions(this.hugeConfig, null, null, cfOptions, cfOptions); List columnFamilyDescriptorList = new ArrayList<>(); List columnFamilyBytes = RocksDB.listColumnFamilies(options, snapshotPath); @@ -711,15 +712,27 @@ private boolean verifySnapshot(String snapshotPath) { } List columnFamilyHandleList = new ArrayList<>(); - try (final DBOptions dbOptions = new DBOptions(); - final RocksDB db = RocksDB.openReadOnly(dbOptions, snapshotPath, - columnFamilyDescriptorList, - columnFamilyHandleList)) { - for (ColumnFamilyHandle handle : columnFamilyHandleList) { - if (handle == null) { - log.error("verifySnapshot some ColumnFamilyHandle is null in {}", - snapshotPath); - return false; + try (final DBOptions dbOptions = new DBOptions()) { + RocksDB db = null; + try { + db = RocksDB.openReadOnly(dbOptions, snapshotPath, + columnFamilyDescriptorList, + columnFamilyHandleList); + for (ColumnFamilyHandle handle : columnFamilyHandleList) { + if (handle == null) { + log.error("verifySnapshot some ColumnFamilyHandle is null in {}", + snapshotPath); + return false; + } + } + } finally { + for (ColumnFamilyHandle handle : columnFamilyHandleList) { + if (handle != null) { + handle.close(); + } + } + if (db != null) { + db.close(); } } } diff --git a/hugegraph-store/hg-store-test/pom.xml b/hugegraph-store/hg-store-test/pom.xml index 213db1d38c..1924294726 100644 --- a/hugegraph-store/hg-store-test/pom.xml +++ b/hugegraph-store/hg-store-test/pom.xml @@ -252,6 +252,10 @@ **/RecoveryPdLookupTest.java **/BatchGraphIsolationTest.java **/GraphIDManagerTest.java + **/GraphIdSnapshotGuardTest.java + **/GraphIdSnapshotLevelsTest.java + **/LegacyGraphIdCheckerTest.java + **/GraphIdCheckCommandTest.java **/SchemaDriverTest.java diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java index e2b9819bcb..a2afc1ee2c 100644 --- a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/CoreSuiteTest.java @@ -65,7 +65,8 @@ MetricSessionTest.class, HgSnapshotHandlerTest.class, StoreIdChangeTest.class, - PartitionMetaCodecTest.class + PartitionMetaCodecTest.class, + LegacyGraphIdStartupLockTest.class }) @Slf4j public class CoreSuiteTest { diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/GraphIdSnapshotGuardTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/GraphIdSnapshotGuardTest.java new file mode 100644 index 0000000000..6d57fe3423 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/GraphIdSnapshotGuardTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Map; + +import org.apache.hugegraph.rocksdb.access.RocksDBFactory; +import org.apache.hugegraph.store.business.BusinessHandlerImpl; +import org.apache.hugegraph.store.constant.HugeServerTables; +import org.apache.hugegraph.store.raft.PartitionStateMachine; +import org.apache.hugegraph.store.snapshot.SnapshotHandler; +import org.apache.hugegraph.store.upgrade.LegacyGraphIdChecker; +import org.apache.hugegraph.store.util.HgStoreException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; + +import com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; + +public class GraphIdSnapshotGuardTest { + + @Rule + public TemporaryFolder temporary = new TemporaryFolder(); + + @Test + public void testRejectedIncomingSnapshotNeverOpensLocalDatabase() throws Exception { + Path incoming = databaseWithPrefix(0xFFFE); + BusinessHandlerImpl handler = mock(BusinessHandlerImpl.class, CALLS_REAL_METHODS); + HgStoreException failure = assertThrows(HgStoreException.class, + () -> handler.loadSnapshot(incoming.toString(), "g", 7, 100)); + assertEquals(HgStoreException.EC_RKDB_IMPORT_SNAPSHOT_FAIL, failure.getCode()); + assertTrue(failure.getMessage().contains("0xFFFE")); + assertTrue(failure.getMessage().contains(incoming.toString())); + assertTrue(failure.getCause() instanceof IllegalStateException); + verify(handler, never()).getSession(7); + } + + @Test + public void testLegalHighGraphIdReachesExistingSnapshotLoader() throws Exception { + Path incoming = databaseWithPrefix(0xFFFD); + BusinessHandlerImpl handler = mock(BusinessHandlerImpl.class, CALLS_REAL_METHODS); + HgStoreException reachedLoader = new HgStoreException("Reached existing loader"); + doThrow(reachedLoader).when(handler).getSession(7); + assertSame(reachedLoader, assertThrows(HgStoreException.class, + () -> handler.loadSnapshot(incoming.toString(), "g", 7, 100))); + verify(handler).getSession(7); + } + + @Test + public void testSnapshotIndexIsPublishedOnlyAfterSuccessfulLoad() { + SnapshotHandler handler = mock(SnapshotHandler.class); + PartitionStateMachine machine = new PartitionStateMachine(7, handler); + SnapshotReader reader = reader(42); + doAnswer(invocation -> { + assertEquals(0, machine.getCommittedIndex()); + assertEquals(42L, (long) invocation.getArgument(1)); + return null; + }).when(handler).onSnapshotLoad(any(), anyLong()); + assertTrue(machine.onSnapshotLoad(reader)); + assertEquals(42, machine.getCommittedIndex()); + } + + @Test + public void testRejectedSnapshotPreservesPreviousCommittedIndex() { + SnapshotHandler handler = mock(SnapshotHandler.class); + PartitionStateMachine machine = new PartitionStateMachine(7, handler); + assertTrue(machine.onSnapshotLoad(reader(42))); + doThrow(new HgStoreException("Legacy GraphID snapshot")).when(handler) + .onSnapshotLoad(any(), anyLong()); + assertFalse(machine.onSnapshotLoad(reader(100))); + assertEquals(42, machine.getCommittedIndex()); + } + + @Test + public void testUnreadableSnapshotMetadataPreservesCommittedIndex() { + SnapshotHandler handler = mock(SnapshotHandler.class); + PartitionStateMachine machine = new PartitionStateMachine(7, handler); + assertTrue(machine.onSnapshotLoad(reader(42))); + SnapshotReader broken = mock(SnapshotReader.class); + when(broken.load()).thenThrow(new IllegalStateException("Unreadable metadata")); + assertFalse(machine.onSnapshotLoad(broken)); + assertEquals(42, machine.getCommittedIndex()); + } + + private Path databaseWithPrefix(int id) throws Exception { + RocksDB.loadLibrary(); + RocksDBFactory.getInstance().setHugeConfig(LegacyGraphIdChecker.configuration(Map.of())); + Path path = this.temporary.newFolder().toPath(); + try (Options options = new Options().setCreateIfMissing(true); + ColumnFamilyOptions familyOptions = new ColumnFamilyOptions(); + RocksDB db = RocksDB.open(options, path.toString()); + ColumnFamilyHandle table = db.createColumnFamily(new ColumnFamilyDescriptor( + HugeServerTables.VERTEX_TABLE.getBytes(StandardCharsets.UTF_8), familyOptions))) { + db.put(table, new byte[]{(byte) (id >>> 8), (byte) id, 0, 0, 1}, new byte[]{1, 2, 3}); + } + return path; + } + + private static SnapshotReader reader(long index) { + SnapshotReader reader = mock(SnapshotReader.class); + when(reader.load()).thenReturn(SnapshotMeta.newBuilder().setLastIncludedIndex(index) + .setLastIncludedTerm(3).build()); + return reader; + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/GraphIdSnapshotLevelsTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/GraphIdSnapshotLevelsTest.java new file mode 100644 index 0000000000..d1d5badb8b --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/GraphIdSnapshotLevelsTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.HexFormat; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.config.OptionSpace; +import org.apache.hugegraph.rocksdb.access.RocksDBOptions; +import org.apache.hugegraph.rocksdb.access.RocksDBSession; +import org.apache.hugegraph.store.constant.HugeServerTables; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.Checkpoint; +import org.rocksdb.CompactRangeOptions; +import org.rocksdb.RocksDB; + +public class GraphIdSnapshotLevelsTest { + + private static final String TABLE = HugeServerTables.VERTEX_TABLE; + private static final byte[] KEY = {(byte) 0xFF, (byte) 0xFD, 1, 0, 0}; + private static final byte[] VALUE = {31, 32, 33}; + + @Rule + public TemporaryFolder temporary = new TemporaryFolder(); + + @BeforeClass + public static void init() { + RocksDB.loadLibrary(); + OptionSpace.register("rocksdb", RocksDBOptions.class.getName()); + } + + @Test + public void testConfiguredLevelSevenSnapshotActivatesAndReopens() throws Exception { + HugeConfig config = new HugeConfig(Map.of( + "rocksdb.optimize_mode", false, + "rocksdb.write_buffer_size", 1048576L, + "rocksdb.num_levels", 8, + "rocksdb.compression_per_level", "[none,none,none,none,none,none,none,none]")); + Path producer = this.temporary.newFolder("producer").toPath(); + Path snapshot = producer.resolve("snapshot"); + try (RocksDBSession source = new RocksDBSession(config, producer.toString(), "seed", 0L)) { + source.checkTable(TABLE); + var table = source.getTables().get(TABLE); + source.getDB().put(table, KEY, VALUE); + source.flush(true); + try (CompactRangeOptions compact = new CompactRangeOptions() + .setChangeLevel(true).setTargetLevel(7)) { + source.getDB().compactRange(table, null, null, compact); + } + assertFalse("The regression requires a real SST in configured level 7", + source.getDB().getColumnFamilyMetaData(table).levels().get(7).files().isEmpty()); + try (Checkpoint checkpoint = Checkpoint.create(source.getDB())) { + checkpoint.createCheckpoint(snapshot.toString()); + } + } + Map incoming = hashes(snapshot); + Path destination = this.temporary.newFolder("destination").toPath(); + try (RocksDBSession active = new RocksDBSession(config, destination.toString(), "00000", 0L)) { + active.loadSnapshot(snapshot.toString(), 5L); + } + assertTrue(Files.isRegularFile(destination.resolve("00000_5/CURRENT"))); + assertFalse(Files.exists(destination.resolve("00000_5_temp_"))); + try (RocksDBSession restored = new RocksDBSession(config, destination.toString(), "00000", 5L)) { + assertEquals(destination.resolve("00000_5").toString(), restored.getDbPath()); + assertArrayEquals(VALUE, restored.sessionOp().get(TABLE, KEY)); + } + assertEquals(incoming, hashes(snapshot)); + } + + private static Map hashes(Path root) throws Exception { + Map result = new TreeMap<>(); + try (var paths = Files.walk(root)) { + for (Path path : paths.filter(Files::isRegularFile).toList()) { + result.put(root.relativize(path).toString(), HexFormat.of().formatHex( + MessageDigest.getInstance("SHA-256").digest(Files.readAllBytes(path)))); + } + } + return result; + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/LegacyGraphIdCheckerTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/LegacyGraphIdCheckerTest.java new file mode 100644 index 0000000000..f305d34ede --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/LegacyGraphIdCheckerTest.java @@ -0,0 +1,415 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.Arrays; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; + +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.pd.grpc.Metapb; +import org.apache.hugegraph.store.meta.MetadataKeyHelper; +import org.apache.hugegraph.store.upgrade.LegacyGraphIdChecker; +import org.apache.hugegraph.store.util.HgStoreException; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; + +import com.google.protobuf.Int64Value; + +public class LegacyGraphIdCheckerTest { + + private static final String GRAPH = "DEFAULT/hugegraph/g"; + private static final byte[] LEGACY_KEY = {(byte) 0xFF, (byte) 0xFE, 1, 0, 0}; + private static final byte[] LEGAL_KEY = {(byte) 0xFF, (byte) 0xFD, 1, 0, 0}; + private static final byte[] VALUE = {1, 2, 3}; + private static HugeConfig config; + + @Rule + public TemporaryFolder temporary = new TemporaryFolder(); + + @BeforeClass + public static void init() { + RocksDB.loadLibrary(); + config = LegacyGraphIdChecker.configuration(Map.of( + "rocksdb.optimize_mode", false, + "rocksdb.write_buffer_size", 1048576L)); + } + + @Test + public void testLegalHighIdAndMappingLeaveDatabaseUnchanged() throws Exception { + Path database = this.temporary.newFolder().toPath(); + write(database, "default", MetadataKeyHelper.getGraphIDKey(GRAPH), + Int64Value.of(65533).toByteArray()); + write(database, "g+v", LEGAL_KEY, VALUE); + Map before = hashes(database); + LegacyGraphIdChecker.checkDatabase(database, config); + assertEquals(before, hashes(database)); + } + + @Test + public void testLegacyPrefixInEveryColumnFamilyLeavesDatabaseUnchanged() throws Exception { + for (String table : List.of("default", "g+v", "g+oe", "g+ie", "g+index", + "g+olap", "g+task", "g+server", "custom-table")) { + Path database = this.temporary.newFolder().toPath(); + write(database, table, LEGACY_KEY, VALUE); + Map before = hashes(database); + IllegalStateException failure = assertThrows(IllegalStateException.class, + () -> LegacyGraphIdChecker.checkDatabase(database, config)); + assertTrue(failure.getMessage(), failure.getMessage().contains("0xFFFE")); + assertTrue(failure.getMessage(), failure.getMessage().contains(table)); + assertEquals(before, hashes(database)); + } + } + + @Test + public void testInvalidMappingWithoutBusinessRowsIsRejected() throws Exception { + for (long id : new long[]{-1L, 65534L, 65535L, Long.MAX_VALUE}) { + Path database = this.temporary.newFolder().toPath(); + write(database, "default", MetadataKeyHelper.getGraphIDKey(GRAPH), + Int64Value.of(id).toByteArray()); + Map before = hashes(database); + IllegalStateException failure = assertThrows(IllegalStateException.class, + () -> LegacyGraphIdChecker.checkDatabase(database, config)); + assertTrue(failure.getMessage(), failure.getMessage().contains("Invalid GraphID metadata")); + assertEquals(before, hashes(database)); + } + } + + @Test + public void testDuplicateMappingsWithoutBusinessRowsAreRejected() throws Exception { + Path database = this.temporary.newFolder().toPath(); + write(database, "default", MetadataKeyHelper.getGraphIDKey(GRAPH), Int64Value.of(7).toByteArray()); + write(database, "default", MetadataKeyHelper.getGraphIDKey("DEFAULT/other/g"), + Int64Value.of(7).toByteArray()); + IllegalStateException failure = assertThrows(IllegalStateException.class, + () -> LegacyGraphIdChecker.checkDatabase(database, config)); + assertTrue(failure.getMessage(), failure.getMessage().contains("multiple graphs")); + } + + @Test + public void testMalformedMappingValueIsRejectedWithoutModification() throws Exception { + Path database = this.temporary.newFolder().toPath(); + write(database, "default", MetadataKeyHelper.getGraphIDKey(GRAPH), new byte[]{8}); + Map before = hashes(database); + assertThrows(IOException.class, () -> LegacyGraphIdChecker.checkDatabase(database, config)); + assertEquals(before, hashes(database)); + } + + @Test + public void testInvalidUtf8GraphNamesCannotHideInvalidMapping() throws Exception { + Path database = this.temporary.newFolder().toPath(); + byte[] prefix = MetadataKeyHelper.getGraphIDKey(""); + byte[] first = Arrays.copyOf(prefix, prefix.length + 1); + byte[] second = Arrays.copyOf(first, first.length); + first[prefix.length] = (byte) 0x80; + second[prefix.length] = (byte) 0x81; + // Both suffixes decode to the replacement character; a String-keyed map loses the first value. + write(database, "default", first, Int64Value.of(65534).toByteArray()); + write(database, "default", second, Int64Value.of(1).toByteArray()); + Map before = hashes(database); + assertThrows(IllegalStateException.class, () -> LegacyGraphIdChecker.checkDatabase(database, config)); + assertEquals(before, hashes(database)); + } + + @Test + public void testRetainedDatabaseVersionIsCheckedWithoutPruning() throws Exception { + Path root = this.temporary.newFolder().toPath(); + Path retained = root.resolve("db/00000"); + write(retained, "g+v", LEGACY_KEY, VALUE); + write(root.resolve("db/00000_42"), "g+v", LEGAL_KEY, VALUE); + Map before = hashes(root); + HgStoreException failure = assertThrows(HgStoreException.class, + () -> LegacyGraphIdChecker.checkStore(root.toString(), config)); + assertTrue(failure.getMessage(), failure.getMessage().contains(retained.toString())); + assertEquals(before, hashes(root)); + } + + @Test + public void testIncompleteRetainedDirectoryNeedsExplicitRecoveryWithoutPruning() throws Exception { + Path root = this.temporary.newFolder().toPath(); + Path retained = root.resolve("db/00000"); + write(retained, "g+v", LEGAL_KEY, VALUE); + write(root.resolve("db/00000_42"), "g+v", LEGAL_KEY, VALUE); + Files.delete(retained.resolve("CURRENT")); + Map before = hashes(root); + + HgStoreException failure = assertThrows(HgStoreException.class, + () -> LegacyGraphIdChecker.checkStore(root.toString(), config)); + assertTrue(failure.getMessage(), failure.getMessage().contains(retained.toString())); + assertTrue(failure.getMessage(), failure.getMessage().contains("CURRENT is missing")); + assertTrue(failure.getMessage(), failure.getMessage().contains("#incomplete-database-directories")); + assertEquals(before, hashes(root)); + } + + @Test + public void testIncompleteLatestDirectoryIsNeverIgnored() throws Exception { + Path root = this.temporary.newFolder().toPath(); + write(root.resolve("db/00000"), "g+v", LEGAL_KEY, VALUE); + Path latest = root.resolve("db/00000_42"); + write(latest, "g+v", LEGAL_KEY, VALUE); + Files.delete(latest.resolve("CURRENT")); + Map before = hashes(root); + + HgStoreException failure = assertThrows(HgStoreException.class, + () -> LegacyGraphIdChecker.checkStore(root.toString(), config)); + assertTrue(failure.getMessage(), failure.getMessage().contains(latest.toString())); + assertEquals(before, hashes(root)); + } + + @Test + public void testIncompleteMetadataDirectoryIsNeverIgnored() throws Exception { + Path root = this.temporary.newFolder().toPath(); + Path stored = this.temporary.newFolder().toPath(); + write(stored.resolve("db/00007"), "g+oe", LEGACY_KEY, VALUE); + storeLocation(root, stored, 7); + IllegalStateException legacy = assertThrows(IllegalStateException.class, + () -> LegacyGraphIdChecker.checkOffline(root.toString(), config)); + assertTrue(legacy.getMessage(), legacy.getMessage().contains(stored.toString())); + + Path metadata = root.resolve("hgstore-metadata"); + // Check missing CURRENT alone, then CURRENT and LOCK together: diagnose before locking. + for (String file : List.of("CURRENT", "LOCK")) { + Files.delete(metadata.resolve(file)); + Map before = hashes(this.temporary.getRoot().toPath()); + IOException offline = assertThrows(IOException.class, + () -> LegacyGraphIdChecker.checkOffline(root.toString(), config)); + LegacyGraphIdChecker.StartupPreflightException startup = assertThrows( + LegacyGraphIdChecker.StartupPreflightException.class, + () -> LegacyGraphIdChecker.checkStore(root.toString(), config)); + for (Exception failure : List.of(offline, startup)) { + assertTrue(failure.getMessage(), failure.getMessage().contains(metadata.toString())); + assertTrue(failure.getMessage(), failure.getMessage().contains("CURRENT is missing")); + assertTrue(failure.getMessage(), failure.getMessage().contains("#incomplete-database-directories")); + } + assertEquals(before, hashes(this.temporary.getRoot().toPath())); + } + } + + @Test + public void testFreshDataRootWithoutMetadataIsAccepted() throws Exception { + Path root = this.temporary.newFolder().toPath(); + LegacyGraphIdChecker.checkOffline(root.toString(), config); + LegacyGraphIdChecker.checkStore(root.toString(), config); + assertFalse(Files.exists(root.resolve("hgstore-metadata"))); + assertTrue(hashes(root).isEmpty()); + } + + @Test + public void testDanglingMetadataLinkIsRejected() throws Exception { + Path root = this.temporary.newFolder().toPath(); + Path metadata = root.resolve("hgstore-metadata"); + Path missing = root.resolve("missing-metadata"); + Files.createSymbolicLink(metadata, missing); + IOException failure = assertThrows(IOException.class, + () -> LegacyGraphIdChecker.checkOffline(root.toString(), config)); + assertTrue(failure.getMessage(), failure.getMessage().contains(metadata.toString())); + assertTrue(failure.getMessage(), failure.getMessage().contains("CURRENT is missing")); + assertThrows(LegacyGraphIdChecker.StartupPreflightException.class, + () -> LegacyGraphIdChecker.checkStore(root.toString(), config)); + assertEquals(missing, Files.readSymbolicLink(metadata)); + assertFalse(Files.exists(missing)); + } + + @Test + public void testLatestSnapshotIsCheckedDespiteShouldNotLoadMarker() throws Exception { + Path root = this.temporary.newFolder().toPath(); + write(root.resolve("db/00000"), "g+v", LEGAL_KEY, VALUE); + Path snapshot = root.resolve("raft/00000/snapshot/snapshot_00000000000000000042"); + write(snapshot.resolve("data"), "g+v", LEGACY_KEY, VALUE); + Files.writeString(snapshot.resolve("should_not_load"), "saved snapshot"); + Map before = hashes(root); + HgStoreException failure = assertThrows(HgStoreException.class, + () -> LegacyGraphIdChecker.checkStore(root.toString(), config)); + assertTrue(failure.getMessage(), failure.getMessage().contains(snapshot.toString())); + assertEquals(before, hashes(root)); + } + + @Test + public void testSupersededSnapshotIsNotSelected() throws Exception { + Path root = this.temporary.newFolder().toPath(); + write(root.resolve("raft/00000/snapshot/snapshot_2/data"), "g+v", LEGACY_KEY, VALUE); + write(root.resolve("raft/00000/snapshot/snapshot_10/data"), "g+v", LEGAL_KEY, VALUE); + Map before = hashes(root); + LegacyGraphIdChecker.checkStore(root.toString(), config); + assertEquals(before, hashes(root)); + } + + @Test + public void testWhitespaceInConfiguredDirectoryIsNotTrimmed() throws Exception { + Path root = this.temporary.newFolder(" store ").toPath(); + write(root.resolve("db/00000"), "g+v", LEGACY_KEY, VALUE); + Map before = hashes(root); + HgStoreException failure = assertThrows(HgStoreException.class, + () -> LegacyGraphIdChecker.checkStore(root.toString(), config)); + assertTrue(failure.getMessage(), failure.getMessage().contains(root.toString())); + assertEquals(before, hashes(root)); + } + + @Test + public void testAllConfiguredDataRootsAreChecked() throws Exception { + Path first = this.temporary.newFolder().toPath(); + Path second = this.temporary.newFolder().toPath(); + write(first.resolve("db/00000"), "g+v", LEGAL_KEY, VALUE); + write(second.resolve("db/00001"), "g+ie", LEGACY_KEY, VALUE); + Map before = hashes(this.temporary.getRoot().toPath()); + HgStoreException failure = assertThrows(HgStoreException.class, + () -> LegacyGraphIdChecker.checkStore(first + "," + second, config)); + assertTrue(failure.getMessage(), failure.getMessage().contains(second.toString())); + assertEquals(before, hashes(this.temporary.getRoot().toPath())); + } + + @Test + public void testUndiscoveredStoredLocationIsChecked() throws Exception { + Path root = this.temporary.newFolder().toPath(); + Path stored = this.temporary.newFolder().toPath(); + write(stored.resolve("db/00007"), "g+oe", LEGACY_KEY, VALUE); + storeLocation(root, stored, 7); + Map before = hashes(this.temporary.getRoot().toPath()); + HgStoreException failure = assertThrows(HgStoreException.class, + () -> LegacyGraphIdChecker.checkStore(root.toString(), config)); + assertTrue(failure.getMessage(), failure.getMessage().contains(stored.toString())); + assertEquals(before, hashes(this.temporary.getRoot().toPath())); + } + + @Test + public void testMovedConfiguredPartitionSupersedesStoredLocation() throws Exception { + Path root = this.temporary.newFolder().toPath(); + Path stored = this.temporary.newFolder().toPath(); + write(root.resolve("db/00007"), "g+v", LEGAL_KEY, VALUE); + write(stored.resolve("db/00007"), "g+v", LEGACY_KEY, VALUE); + storeLocation(root, stored, 7); + Map before = hashes(this.temporary.getRoot().toPath()); + LegacyGraphIdChecker.checkStore(root.toString(), config); + assertEquals(before, hashes(this.temporary.getRoot().toPath())); + } + + @Test + public void testOfflineCheckRejectsAnotherProcessesNativeLock() throws Exception { + Path root = this.temporary.newFolder().toPath(); + Path metadata = root.resolve("hgstore-metadata"); + write(metadata, "default", new byte[]{1}, VALUE); + Path signals = this.temporary.newFolder().toPath(); + Path ready = signals.resolve("ready"); + Path output = signals.resolve("writer.log"); + String java = Path.of(System.getProperty("java.home"), "bin", "java").toString(); + String classpath = System.getProperty("surefire.test.class.path", System.getProperty("java.class.path")); + Process writer = new ProcessBuilder(java, "-cp", classpath, NativeLockHolder.class.getName(), + metadata.toString(), ready.toString()) + .redirectErrorStream(true).redirectOutput(output.toFile()).start(); + try { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!Files.exists(ready) && writer.isAlive() && System.nanoTime() < deadline) { + Thread.sleep(10L); + } + assertTrue(Files.readString(output), Files.exists(ready)); + Map before = hashes(root); + IOException failure = assertThrows(IOException.class, + () -> LegacyGraphIdChecker.checkOffline(root.toString(), config)); + assertTrue(failure.getMessage(), failure.getMessage().contains("locked")); + assertEquals(before, hashes(root)); + } finally { + writer.getOutputStream().close(); + if (!writer.waitFor(10, TimeUnit.SECONDS)) { + writer.destroyForcibly(); + writer.waitFor(10, TimeUnit.SECONDS); + } + } + assertEquals(Files.readString(output), 0, writer.exitValue()); + LegacyGraphIdChecker.checkOffline(root.toString(), config); + } + + @Test + public void testOfflineCheckRequiresExistingRootsAndLocks() throws Exception { + Path root = this.temporary.newFolder().toPath(); + Path missing = root.resolve("missing"); + assertThrows(IOException.class, () -> LegacyGraphIdChecker.checkOffline(missing.toString(), config)); + assertFalse(Files.exists(missing)); + Path database = root.resolve("db/00000"); + write(database, "g+v", LEGAL_KEY, VALUE); + Files.delete(database.resolve("LOCK")); + Map before = hashes(root); + assertThrows(IOException.class, () -> LegacyGraphIdChecker.checkOffline(root.toString(), config)); + assertEquals(before, hashes(root)); + } + + private static void storeLocation(Path root, Path location, int partition) throws Exception { + write(root.resolve("hgstore-metadata"), "default", MetadataKeyHelper.getPartitionStoreKey(partition), + Metapb.PartitionStore.newBuilder().setPartitionId(partition) + .setStoreLocation(location.toString()).build().toByteArray()); + } + + private static void write(Path path, String table, byte[] key, byte[] value) throws Exception { + Files.createDirectories(path); + try (Options options = new Options().setCreateIfMissing(true); + RocksDB database = RocksDB.open(options, path.toString())) { + if ("default".equals(table)) { + database.put(key, value); + } else { + try (ColumnFamilyOptions familyOptions = new ColumnFamilyOptions(); + ColumnFamilyHandle handle = database.createColumnFamily(new ColumnFamilyDescriptor( + table.getBytes(StandardCharsets.UTF_8), familyOptions))) { + database.put(handle, key, value); + } + } + } + } + + private static Map hashes(Path root) throws Exception { + Map result = new TreeMap<>(); + try (var paths = Files.walk(root)) { + for (Path path : paths.filter(Files::isRegularFile).toList()) { + result.put(root.relativize(path).toString(), Files.getLastModifiedTime(path) + ":" + + HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(Files.readAllBytes(path)))); + } + } + return result; + } + + public static final class NativeLockHolder { + + public static void main(String[] args) throws Exception { + RocksDB.loadLibrary(); + try (Options options = new Options(); + RocksDB ignored = RocksDB.open(options, args[0])) { + Files.writeString(Path.of(args[1]), "locked"); + System.in.read(); + } + } + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/LegacyGraphIdStartupLockTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/LegacyGraphIdStartupLockTest.java new file mode 100644 index 0000000000..2defddf828 --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/core/LegacyGraphIdStartupLockTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.core; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.hugegraph.config.HugeConfig; +import org.apache.hugegraph.store.upgrade.LegacyGraphIdChecker; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; + +public class LegacyGraphIdStartupLockTest { + + private static HugeConfig config; + + @Rule + public TemporaryFolder temporary = new TemporaryFolder(); + + @BeforeClass + public static void init() { + RocksDB.loadLibrary(); + config = LegacyGraphIdChecker.configuration(Map.of( + "rocksdb.optimize_mode", false, + "rocksdb.write_buffer_size", 1048576L)); + } + + @Test + public void testStartupCheckRejectsRunningStoreNativeLock() throws Exception { + Path root = this.temporary.newFolder().toPath(); + Path metadata = root.resolve("hgstore-metadata"); + Files.createDirectories(metadata); + try (Options options = new Options().setCreateIfMissing(true); + RocksDB database = RocksDB.open(options, metadata.toString())) { + database.put(new byte[]{1}, new byte[]{1}); + } + + Path signals = this.temporary.newFolder().toPath(); + Path ready = signals.resolve("ready"); + Path output = signals.resolve("writer.log"); + String java = Path.of(System.getProperty("java.home"), "bin", "java").toString(); + String classpath = System.getProperty("surefire.test.class.path", + System.getProperty("java.class.path")); + Process writer = new ProcessBuilder( + java, "-cp", classpath, + LegacyGraphIdCheckerTest.NativeLockHolder.class.getName(), + metadata.toString(), ready.toString()) + .redirectErrorStream(true).redirectOutput(output.toFile()).start(); + try { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!Files.exists(ready) && writer.isAlive() && + System.nanoTime() < deadline) { + Thread.sleep(10L); + } + String log = Files.exists(output) ? Files.readString(output) : ""; + assertTrue(log, Files.exists(ready)); + LegacyGraphIdChecker.StartupPreflightException failure = assertThrows( + LegacyGraphIdChecker.StartupPreflightException.class, + () -> LegacyGraphIdChecker.checkStore(root.toString(), config)); + assertTrue(failure.getMessage(), failure.getMessage().contains("locked")); + } finally { + writer.getOutputStream().close(); + if (!writer.waitFor(10, TimeUnit.SECONDS)) { + writer.destroyForcibly(); + writer.waitFor(10, TimeUnit.SECONDS); + } + } + assertEquals(Files.readString(output), 0, writer.exitValue()); + LegacyGraphIdChecker.checkStore(root.toString(), config); + } +} diff --git a/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/node/GraphIdCheckCommandTest.java b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/node/GraphIdCheckCommandTest.java new file mode 100644 index 0000000000..8d57d96b5b --- /dev/null +++ b/hugegraph-store/hg-store-test/src/main/java/org/apache/hugegraph/store/node/GraphIdCheckCommandTest.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.store.node; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.hugegraph.store.upgrade.LegacyGraphIdChecker; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; + +public class GraphIdCheckCommandTest { + + @Rule + public TemporaryFolder temporary = new TemporaryFolder(); + + @Test + public void testLocalYamlAndExplicitDataPathWithoutServices() throws Exception { + Path config = this.temporary.newFolder("conf").toPath(); + Path root = this.temporary.newFolder("store").toPath(); + Files.writeString(config.resolve("application.yml"), """ + app: + data-path: ${MISSING_DATA_PATH} + rocksdb: + num_levels: 8 + compression_per_level: '[none,none,none,none,none,none,none,none]' + """); + assertEquals(0, GraphIdCheckCommand.run("--check", "--config", config.toString(), + "--data-path", root.toString())); + assertFalse(Files.exists(root.resolve("hgstore-metadata"))); + assertEquals(2, GraphIdCheckCommand.run("--check", "--config", config.toString())); + } + + @Test + public void testLegacyDataReturnsFailureWithoutChangingKey() throws Exception { + RocksDB.loadLibrary(); + Path config = this.temporary.newFolder("conf").toPath(); + Path root = this.temporary.newFolder("store").toPath(); + Path database = root.resolve("db/00000"); + Files.createDirectories(database); + Files.writeString(config.resolve("application.yml"), "app:\n data-path: '" + root + "'\n"); + byte[] key = {(byte) 0xFF, (byte) 0xFE, 1, 0, 0}; + try (Options options = new Options().setCreateIfMissing(true); + ColumnFamilyOptions cfOptions = new ColumnFamilyOptions(); + RocksDB db = RocksDB.open(options, database.toString()); + ColumnFamilyHandle table = db.createColumnFamily(new ColumnFamilyDescriptor( + new byte[]{'g', '+', 'v'}, cfOptions))) { + db.put(table, key, new byte[]{1}); + } + assertEquals(2, GraphIdCheckCommand.run("--check", "--config", config.toString())); + assertEquals(2, GraphIdCheckCommand.run("--check", "--config", config.toString())); + RuntimeException refusal = assertThrows(LegacyGraphIdChecker.StartupPreflightException.class, + () -> LegacyGraphIdChecker.checkStore(root.toString(), + LegacyGraphIdChecker.configuration(Map.of()))); + assertTrue(StoreNodeApplication.isGraphIdPreflightFailure(refusal)); + assertTrue(StoreNodeApplication.isGraphIdPreflightFailure(new RuntimeException(refusal))); + assertFalse(Files.exists(root.resolve("hgstore-metadata"))); + } + + @Test + public void testUnrelatedStartupErrorsKeepTheirExistingLifecycle() { + assertFalse(StoreNodeApplication.isGraphIdPreflightFailure( + new RuntimeException("PD unreachable"))); + assertFalse(StoreNodeApplication.isGraphIdPreflightFailure( + new RuntimeException("GraphID upgrade preflight refused Store startup"))); + RuntimeException first = new RuntimeException(); + RuntimeException second = new RuntimeException(first); + first.initCause(second); + assertFalse(StoreNodeApplication.isGraphIdPreflightFailure(first)); + } + + @Test + public void testMissingDataPathDoesNotGuessAnEmptyDirectory() throws Exception { + Path directory = this.temporary.newFolder("distribution").toPath(); + Path config = Files.createDirectory(directory.resolve("conf")); + Files.createDirectory(directory.resolve("storage")); + Files.writeString(config.resolve("application.yml"), "rocksdb:\n num_levels: 7\n"); + Process process = new ProcessBuilder( + Path.of(System.getProperty("java.home"), "bin", "java").toString(), + "-cp", System.getProperty("java.class.path"), GraphIdCheckCommand.class.getName(), + "--check", "--config", config.toString()) + .directory(directory.toFile()).redirectErrorStream(true).start(); + try { + assertTrue("Checker process timed out", process.waitFor(10, TimeUnit.SECONDS)); + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + assertEquals(output, 2, process.exitValue()); + assertTrue(output, output.contains("Missing app.data-path")); + } finally { + process.destroyForcibly(); + } + } + + @Test + public void testInvalidArgumentsDoNotRunStore() { + assertEquals(0, GraphIdCheckCommand.run("--help")); + assertEquals(2, GraphIdCheckCommand.run()); + assertEquals(2, GraphIdCheckCommand.run("--execute")); + assertEquals(2, GraphIdCheckCommand.run("--check", "--check")); + assertEquals(2, GraphIdCheckCommand.run("--check", "--data-path")); + } +}