From 7a0db2d283be32f7ed061f3e08bef203282d30a6 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Mon, 9 Nov 2026 16:06:12 -0700 Subject: [PATCH 1/7] Remove test workflow for prepare-incremental-build action The dedicated test workflow is no longer needed. Delete it along with the status badge in the action's README, which linked to it. Signed-off-by: Ihor Solodrai --- .../test-prepare-incremental-build-action.yml | 69 ------------------- prepare-incremental-build/README.md | 2 - 2 files changed, 71 deletions(-) delete mode 100644 .github/workflows/test-prepare-incremental-build-action.yml diff --git a/.github/workflows/test-prepare-incremental-build-action.yml b/.github/workflows/test-prepare-incremental-build-action.yml deleted file mode 100644 index 8137d117..00000000 --- a/.github/workflows/test-prepare-incremental-build-action.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Workflow to test prepare-incremental-build action - -on: - push: - paths: - - 'prepare-incremental-build/**' - - '.github/**' - workflow_dispatch: - inputs: - arch: - required: true - type: string - default: x86_64 - toolchain_full: - required: true - type: string - default: llvm-17 - runs_on: - required: true - type: string - default: ubuntu-24.04 - kernel-repo: - required: true - type: string - default: kernel-patches/bpf - kernel-branch: - required: true - type: string - default: bpf-next - -jobs: - test-job: - name: Test prepare-incremental-build action - runs-on: ${{ inputs.runs_on || 'ubuntu-24.04' }} - timeout-minutes: 100 - env: - ARCH: ${{ inputs.arch || 'x86_64' }} - TOOLCHAIN_FULL: ${{ inputs.toolchain_full || 'llvm-17' }} - KERNEL_REPO: ${{ inputs.kernel-repo || 'kernel-patches/bpf' }} - KERNEL_BRANCH: ${{ inputs.kernel-branch || 'bpf-next' }} - REPO_ROOT: ${{ github.workspace }}/linux - KBUILD_OUTPUT: ${{ github.workspace }}/kbuild-output - steps: - - - uses: actions/checkout@v4 - name: Checkout libbpf/ci - - - uses: actions/checkout@v4 - name: Checkout ${{ env.KERNEL_REPO }} - with: - repository: ${{ env.KERNEL_REPO }} - path: ${{ env.REPO_ROOT }} - - - name: Run prepare-incremental-build action - uses: ./prepare-incremental-build - with: - repo-root: ${{ env.REPO_ROOT }} - base-branch: ${{ env.KERNEL_BRANCH }} - arch: ${{ env.ARCH }} - toolchain_full: ${{ env.TOOLCHAIN_FULL }} - kbuild-output: ${{ env.KBUILD_OUTPUT }} - cache-key-prefix: 'test-prepare-incremental-build-action' - - - name: Put something in KBUILD_OUTPUT to trigger caching - shell: bash - run: | - mkdir -p $KBUILD_OUTPUT/path/to - echo "$ARCH $TOOLCHAIN_FULL $KERNEL" > $KBUILD_OUTPUT/path/to/output.txt - diff --git a/prepare-incremental-build/README.md b/prepare-incremental-build/README.md index 91a91a8a..bb2f6b6f 100644 --- a/prepare-incremental-build/README.md +++ b/prepare-incremental-build/README.md @@ -1,5 +1,3 @@ -[![Action test](https://github.com/libbpf/ci/actions/workflows/test-prepare-incremental-build-action.yml/badge.svg)](https://github.com/libbpf/ci/actions/workflows/test-prepare-incremental-build-action.yml) - # Prepare incremental build This action uses [actions/cache](https://github.com/actions/cache) in combination with custom scripts to save kernel build output from previous workflow runs to faciliate incremental builds. From 55d5a7fb62605ef3643520c3f9187e29884cc596 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Mon, 9 Nov 2026 15:56:00 -0700 Subject: [PATCH 2/7] SPLAT_ALLOWLIST: ignore KASAN-for-JIT selftests splats Port the block added by kernel-patches/vmtest#520. libbpf/ci carries its own copy of these configs and never received it, so the expected KASAN reports from the `kasan` selftest failed the splat check on libbpf/ci runs (e.g. run 42701280551) even though the test itself passed. Signed-off-by: Ihor Solodrai --- ci/vmtest/configs/SPLAT_ALLOWLIST | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ci/vmtest/configs/SPLAT_ALLOWLIST b/ci/vmtest/configs/SPLAT_ALLOWLIST index 0c23bfb3..ef22209a 100644 --- a/ci/vmtest/configs/SPLAT_ALLOWLIST +++ b/ci/vmtest/configs/SPLAT_ALLOWLIST @@ -19,3 +19,15 @@ WARNING: Unprivileged eBPF is enabled # https://lore.kernel.org/all/20260812-hello_world-v1-1-c3c2ddcb362d@meta.com/ # Delete this entry once that fix reaches the CI kernel. BUG: KASAN: stack-out-of-bounds in (stack_trace_consume_entry|filter_irq_stacks)\+ + +# Ignore the KASAN splats generated by KASAN-for-JIT selftests: those are +# expected to trigger many splats to validate that the KASAN +# instrumentation is properly inserted in JITed programs. +BUG: KASAN: slab-(out-of-bounds|use-after-free) in bpf_prog_([a-f0-9]){16}_(st|ld)(x)?(_not)?_on_stack\+ +BUG: KASAN: slab-(out-of-bounds|use-after-free) in bpf_prog_([a-f0-9]){16}_simple_atomic(_fetch)?(_not)?_on_stack\+ +BUG: KASAN: slab-(out-of-bounds|use-after-free) in bpf_prog_([a-f0-9]){16}_ldx_patched(_not)?_on_stack\+ +BUG: KASAN: slab-(out-of-bounds|use-after-free) in bpf_prog_([a-f0-9]){16}_(load_acquire|store_release)(_not)?_on_stack\+ +BUG: KASAN: slab-(out-of-bounds|use-after-free) in bpf_prog_([a-f0-9]){16}_verifier_paths_stack_and_non_stack\+ +BUG: KASAN: slab-(out-of-bounds|use-after-free) in bpf_prog_([a-f0-9]){16}_ldx_oob\+ +BUG: KASAN: slab-(out-of-bounds|use-after-free) in bpf_prog_([a-f0-9]){16}_st_blinded\+ +BUG: KASAN: slab-(out-of-bounds|use-after-free) in bpf_prog_([a-f0-9]){16}_ldx_self_alias_on_stack\+ From 5086acb61743dc5903fb40305983b236714f3e34 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Mon, 9 Nov 2026 09:28:00 -0800 Subject: [PATCH 3/7] run-veristat: add a portable corpus runner Running several corpora in one guest needs measurement independent of the selftest directory and GitHub environment. Accept prepared object paths and checksums, and keep each corpus result in its own directory. Require valid, nonempty CSV before marking a measurement complete. Keep program rejection as data and continue after corpus errors so completed results remain available to the caller. Log input and failure context for CI diagnosis while keeping tool output in files. Signed-off-by: Ihor Solodrai --- run-veristat/README.md | 48 +++++++++++ run-veristat/run_veristat.py | 158 +++++++++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+) create mode 100644 run-veristat/README.md create mode 100644 run-veristat/run_veristat.py diff --git a/run-veristat/README.md b/run-veristat/README.md new file mode 100644 index 00000000..771d497e --- /dev/null +++ b/run-veristat/README.md @@ -0,0 +1,48 @@ +# Run veristat + +`run_veristat.py` measures prepared BPF corpora against the running kernel. +It requires Python 3 and a veristat executable. The caller prepares the objects, +boots the kernel, and handles baselines and publication. + +```sh +python3 run_veristat.py --veristat /tools/veristat \ + --manifest /inputs/corpora.json --output /results/current +``` + +The manifest selects exact objects and optional veristat filter files. Paths +are relative to the manifest directory; corpus names must be unique directory +names and object basenames must be unique within each corpus. + +```json +{ + "version": 1, + "corpora": [ + { + "name": "kernel", + "objects": {"kernel/example.bpf.o": ""}, + "filter": "kernel/veristat.cfg" + } + ] +} +``` + +The output directory must be new. The runner checks object SHA256s and runs +one veristat process per corpus with `-o csv -q -e file,prog,verdict,states`. +Each corpus gets `results.csv`, `stderr.log`, and an atomic `status.json`. +Status is `running`, `measured`, or `error`; a missing status means the corpus +was not started. Interrupted and partial results must not be cached as measured. + +`measured` requires a zero tool exit status and nonempty, valid program records. +Rejected programs remain CSV data for the caller's comparison policy. +The runner continues after a corpus error and returns nonzero if any corpus +could not be measured. The VM launcher handles VM failure and timeouts. + +Status includes the corpus digest: SHA256 of the sorted SHA256SUMS-style +` \n` listing. Source/build details and directory locations +do not participate in object identity. + +Diagnostics go to stderr: run inputs, corpus progress, and completion or failure +details. Messages are flushed so an interrupted run retains its last operation. +Tool output stays in files. The caller controls visibility of the result files +and VM console. Meta objects and raw verifier logs must remain private; OSS +callers can publish their results. diff --git a/run-veristat/run_veristat.py b/run-veristat/run_veristat.py new file mode 100644 index 00000000..04056851 --- /dev/null +++ b/run-veristat/run_veristat.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Measure prepared BPF corpora against the running kernel.""" + +import argparse +import csv +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys + + +CSV_FIELDS = ["file_name", "prog_name", "verdict", "total_states"] + + +def log(message): + print(f"run-veristat: {message}", file=sys.stderr, flush=True) + + +def file_sha256(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def corpus_sha256(objects): + names = {Path(path).name: digest for path, digest in objects.items()} + if len(names) != len(objects): + raise ValueError("object basenames must be unique within a corpus") + listing = "".join(f"{names[name]} {name}\n" for name in sorted(names)) + return hashlib.sha256(listing.encode()).hexdigest() + + +def write_status(directory, status): + temporary = directory / "status.tmp" + temporary.write_text(json.dumps(status, indent=2) + "\n") + temporary.replace(directory / "status.json") + + +def validate_results(path): + with path.open(newline="") as stream: + rows = csv.reader(stream) + programs = 0 + try: + if next(rows, None) != CSV_FIELDS: + raise ValueError("unexpected veristat CSV header") + for row in rows: + if len(row) != len(CSV_FIELDS): + raise ValueError("incomplete veristat CSV record") + filename, program, verdict, states = row + if not filename or not program or verdict not in ("success", "failure"): + raise ValueError("invalid veristat program result") + if states != "N/A": + try: + int(states) + except ValueError: + raise ValueError("invalid total_states value") from None + programs += 1 + if not programs: + raise ValueError("veristat produced no program results") + except (ValueError, csv.Error) as error: + raise ValueError(f"{path}:{max(rows.line_num, 1)}: {error}") from None + return programs + + +def measure(veristat, manifest_path, output): + manifest_path = Path(manifest_path).absolute() + veristat = Path(veristat).absolute() + output = Path(output).absolute() + log( + f"kernel={os.uname().release} binary={veristat} " + f"manifest={manifest_path} output={output}" + ) + manifest = json.loads(manifest_path.read_text()) + if manifest["version"] != 1 or not manifest["corpora"]: + raise ValueError("expected a version 1 manifest with at least one corpus") + output.mkdir(parents=True) + statuses = {} + for corpus in manifest["corpora"]: + name = corpus["name"] + directory = output / name + directory.mkdir() + status = {"state": "running"} + write_status(directory, status) + try: + objects = corpus["objects"] + config = ( + manifest_path.parent / corpus["filter"] + if corpus.get("filter") + else None + ) + log(f"{name}: checking {len(objects)} objects; filter={config or 'none'}") + if not objects: + raise ValueError("corpus contains no objects") + status["corpus_sha256"] = corpus_sha256(objects) + paths = [] + for relative, expected in sorted(objects.items()): + path = manifest_path.parent / relative + actual = file_sha256(path) + if actual != expected: + raise ValueError( + f"object checksum mismatch: {path}; expected={expected} actual={actual}" + ) + paths.append(str(path)) + command = [ + str(veristat), + "-o", + "csv", + "-q", + "-e", + "file,prog,verdict,states", + ] + if config: + command.append(f"-f@{config}") + with (directory / "results.csv").open("w") as stdout, ( + directory / "stderr.log" + ).open("w") as stderr: + log( + f"{name}: running veristat; corpus_sha256={status['corpus_sha256']}" + ) + result = subprocess.run(command + paths, stdout=stdout, stderr=stderr) + status["returncode"] = result.returncode + if result.returncode: + raise ValueError( + f"veristat exited with status {result.returncode}; " + f"stderr={directory / 'stderr.log'}" + ) + programs = validate_results(directory / "results.csv") + status["state"] = "measured" + message = f"{name}: measured {programs} programs" + except (OSError, ValueError, csv.Error) as error: + status.update(state="error", error=str(error)) + message = f"{name}: error: {error}" + write_status(directory, status) + log(message) + statuses[name] = status + return statuses + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--veristat", required=True) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + try: + statuses = measure(args.veristat, args.manifest, args.output) + except (OSError, ValueError) as error: + log(f"error: {error}") + return 1 + return int(any(status["state"] != "measured" for status in statuses.values())) + + +if __name__ == "__main__": + sys.exit(main()) From 93956eeff11046bc8ac1e1beb98c31d16dee422b Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Mon, 9 Nov 2026 09:29:24 -0800 Subject: [PATCH 4/7] run-veristat: prepare manifests from local objects Callers select objects by directory and glob. Provide a small helper that records that selection and its checksums, so single-corpus and batch callers can prepare the same input format without duplicating hashing. Signed-off-by: Ihor Solodrai --- run-veristat/README.md | 13 ++++++++++ run-veristat/create_manifest.py | 44 +++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 run-veristat/create_manifest.py diff --git a/run-veristat/README.md b/run-veristat/README.md index 771d497e..9d0ed507 100644 --- a/run-veristat/README.md +++ b/run-veristat/README.md @@ -46,3 +46,16 @@ details. Messages are flushed so an interrupted run retains its last operation. Tool output stays in files. The caller controls visibility of the result files and VM console. Meta objects and raw verifier logs must remain private; OSS callers can publish their results. + +## Prepare a corpus + +`create_manifest.py` selects local files by glob and records their checksums: + +```sh +python3 create_manifest.py --name kernel --directory /inputs/kernel \ + --pattern '*.bpf.o' --filter /inputs/kernel/veristat.cfg \ + --output /inputs/corpora.json +``` + +Omit `--filter` when none is needed. A batch caller can use `make_corpus()` +to build the manifest's `corpora` list. Downloads and builds belong to the caller. diff --git a/run-veristat/create_manifest.py b/run-veristat/create_manifest.py new file mode 100644 index 00000000..26b96917 --- /dev/null +++ b/run-veristat/create_manifest.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Describe selected local objects for a veristat measurement.""" + +import argparse +import json +import os +from pathlib import Path + +from run_veristat import file_sha256 + + +def make_corpus(name, directory, pattern, config, manifest_path): + parent = Path(manifest_path).absolute().parent + corpus = { + "name": name, + "objects": { + os.path.relpath(path, parent): file_sha256(path) + for path in sorted(Path(directory).absolute().glob(pattern)) + if path.is_file() + }, + } + if config: + corpus["filter"] = os.path.relpath(Path(config).absolute(), parent) + return corpus + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--name", required=True) + parser.add_argument("--directory", required=True) + parser.add_argument("--pattern", required=True) + parser.add_argument("--filter") + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + corpus = make_corpus( + args.name, args.directory, args.pattern, args.filter, args.output + ) + args.output.write_text( + json.dumps({"version": 1, "corpora": [corpus]}, indent=2) + "\n" + ) + + +if __name__ == "__main__": + main() From f2b24fe8f50d27e4f4f1aad8b1f61b14750b0234 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Mon, 9 Nov 2026 11:07:48 -0800 Subject: [PATCH 5/7] run-vmtest: scope selftest setup to its guest Custom VM scripts currently inherit the LATEST bpftool check and selftest JSON reporting. This makes unrelated workloads require the kernel selftest source and can report stale selftest results. Select the guest before preparing BPF selftests and use that selection to gate bpftool checks and summaries. Keep the config hook ability to override the bpftool default. Signed-off-by: Ihor Solodrai --- run-vmtest/README.md | 6 ++++++ run-vmtest/run.sh | 26 +++++++++++++++++--------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/run-vmtest/README.md b/run-vmtest/README.md index 2617526d..78b6ab84 100644 --- a/run-vmtest/README.md +++ b/run-vmtest/README.md @@ -19,6 +19,12 @@ In summary the action performs the following: Note that behavior of the running scripts is tunable mostly by the environment variables. +`VMTEST_SCRIPT` selects a custom guest entrypoint. Configuration from +`run-vmtest.env` is loaded before selecting the script. BPF selftest preparation, +bpftool checks, and JSON summaries run only for `run-bpf-selftests.sh`. Custom +scripts share VM setup and exitstatus collection, and provide their own workload +setup and results. + ## Required inputs * `arch` - Target architecture diff --git a/run-vmtest/run.sh b/run-vmtest/run.sh index 3fc30b8e..7f84d2da 100755 --- a/run-vmtest/run.sh +++ b/run-vmtest/run.sh @@ -33,6 +33,7 @@ else exit 2 fi +# Set this default before the config hook so its overrides take precedence. RUN_BPFTOOL_CHECKS=${RUN_BPFTOOL_CHECKS:-} if [[ -z "${RUN_BPFTOOL_CHECKS}" \ && "${KERNEL}" = 'LATEST' \ @@ -53,18 +54,23 @@ then VMTEST_SCRIPT="${GITHUB_ACTION_PATH}/run-scx-selftests.sh" elif [[ -z "$VMTEST_SCRIPT" ]]; then - ${GITHUB_ACTION_PATH}/prepare-bpf-selftests.sh VMTEST_SCRIPT="${GITHUB_ACTION_PATH}/run-bpf-selftests.sh" fi +bpf_selftests=false +if [[ "$VMTEST_SCRIPT" == "${GITHUB_ACTION_PATH}/run-bpf-selftests.sh" ]]; then + bpf_selftests=true + ${GITHUB_ACTION_PATH}/prepare-bpf-selftests.sh +fi + # clear exitstatus file echo -n > exitstatus foldable start bpftool_checks "Running bpftool checks..." # bpftool checks are aimed at checking type names, documentation, shell -# completion etc. against the current kernel, so only run on LATEST. -if [[ -n "${RUN_BPFTOOL_CHECKS}" ]]; then +# completion etc. against the current kernel, so default to LATEST. +if "$bpf_selftests" && [[ -n "${RUN_BPFTOOL_CHECKS:-}" ]]; then bpftool_exitstatus=0 # "&& true" does not change the return code (it is not executed if the # Python script fails), but it prevents the trap on ERR set at the top @@ -143,12 +149,14 @@ fi foldable end collect_status -SUMMARIES=$(find . -maxdepth 1 -name "test_*.json") -for summary in ${SUMMARIES}; do - if [ -f "${summary}" ]; then - "${GITHUB_ACTION_PATH}/print_test_summary.py" -s "${GITHUB_STEP_SUMMARY}" -j "${summary}" - fi -done +if "$bpf_selftests"; then + SUMMARIES=$(find . -maxdepth 1 -name "test_*.json") + for summary in ${SUMMARIES}; do + if [ -f "${summary}" ]; then + "${GITHUB_ACTION_PATH}/print_test_summary.py" -s "${GITHUB_STEP_SUMMARY}" -j "${summary}" + fi + done +fi # Final summary - Don't use a fold, keep it visible echo -e "\033[1;33mTest Results:\033[0m" From e6cb471c9b18835bda91c8a2a4baceea180bac86 Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Mon, 9 Nov 2026 12:39:52 -0800 Subject: [PATCH 6/7] run-vmtest: give veristat its own guest entrypoint Veristat currently enters through the BPF selftest script and its per-corpus shell configs. Prepared manifests let it run independently of that test-only setup. Add a guest entrypoint that invokes the portable worker and records its aggregate exit status for the host. Retire the selftest function and selector configs; callers upgrading from v4 must use the new entrypoint. Signed-off-by: Ihor Solodrai --- ci/vmtest/configs/run_veristat.cilium.cfg | 3 -- ci/vmtest/configs/run_veristat.kernel.cfg | 4 --- ci/vmtest/configs/run_veristat.meta.cfg | 4 --- ci/vmtest/configs/run_veristat.scx.cfg | 3 -- run-veristat/README.md | 22 +++++++++++-- run-vmtest/README.md | 6 +++- run-vmtest/run-bpf-selftests.sh | 40 ----------------------- run-vmtest/run-veristat.sh | 14 ++++++++ 8 files changed, 39 insertions(+), 57 deletions(-) delete mode 100644 ci/vmtest/configs/run_veristat.cilium.cfg delete mode 100644 ci/vmtest/configs/run_veristat.kernel.cfg delete mode 100644 ci/vmtest/configs/run_veristat.meta.cfg delete mode 100644 ci/vmtest/configs/run_veristat.scx.cfg create mode 100755 run-vmtest/run-veristat.sh diff --git a/ci/vmtest/configs/run_veristat.cilium.cfg b/ci/vmtest/configs/run_veristat.cilium.cfg deleted file mode 100644 index a9e78676..00000000 --- a/ci/vmtest/configs/run_veristat.cilium.cfg +++ /dev/null @@ -1,3 +0,0 @@ -VERISTAT_OBJECTS_DIR="${CILIUM_BUILD_OUTPUT}/bpf" -VERISTAT_OBJECTS_GLOB="*.o" -VERISTAT_OUTPUT="veristat-cilium" diff --git a/ci/vmtest/configs/run_veristat.kernel.cfg b/ci/vmtest/configs/run_veristat.kernel.cfg deleted file mode 100644 index 807efc25..00000000 --- a/ci/vmtest/configs/run_veristat.kernel.cfg +++ /dev/null @@ -1,4 +0,0 @@ -VERISTAT_OBJECTS_DIR="${SELFTESTS_BPF}" -VERISTAT_OBJECTS_GLOB="*.bpf.o" -VERISTAT_CFG_FILE="${SELFTESTS_BPF}/veristat.cfg" -VERISTAT_OUTPUT="veristat-kernel" diff --git a/ci/vmtest/configs/run_veristat.meta.cfg b/ci/vmtest/configs/run_veristat.meta.cfg deleted file mode 100644 index 14f08d24..00000000 --- a/ci/vmtest/configs/run_veristat.meta.cfg +++ /dev/null @@ -1,4 +0,0 @@ -VERISTAT_OBJECTS_DIR="${WORKING_DIR}/bpf_objects" -VERISTAT_OBJECTS_GLOB="*.o" -VERISTAT_OUTPUT="veristat-meta" -VERISTAT_CFG_FILE="${VERISTAT_CONFIGS}/veristat_meta.cfg" diff --git a/ci/vmtest/configs/run_veristat.scx.cfg b/ci/vmtest/configs/run_veristat.scx.cfg deleted file mode 100644 index bf289c00..00000000 --- a/ci/vmtest/configs/run_veristat.scx.cfg +++ /dev/null @@ -1,3 +0,0 @@ -VERISTAT_OBJECTS_DIR="${SCX_BUILD_OUTPUT}/bpf" -VERISTAT_OBJECTS_GLOB="*.bpf.o" -VERISTAT_OUTPUT="veristat-scx" diff --git a/run-veristat/README.md b/run-veristat/README.md index 9d0ed507..dab58224 100644 --- a/run-veristat/README.md +++ b/run-veristat/README.md @@ -6,7 +6,7 @@ boots the kernel, and handles baselines and publication. ```sh python3 run_veristat.py --veristat /tools/veristat \ - --manifest /inputs/corpora.json --output /results/current + --manifest /inputs/manifest.json --output /results/current ``` The manifest selects exact objects and optional veristat filter files. Paths @@ -54,8 +54,26 @@ callers can publish their results. ```sh python3 create_manifest.py --name kernel --directory /inputs/kernel \ --pattern '*.bpf.o' --filter /inputs/kernel/veristat.cfg \ - --output /inputs/corpora.json + --output /inputs/manifest.json ``` Omit `--filter` when none is needed. A batch caller can use `make_corpus()` to build the manifest's `corpora` list. Downloads and builds belong to the caller. + +## Run with run-vmtest + +Select `run-vmtest/run-veristat.sh` through `VMTEST_SCRIPT` and supply guest paths: + +```sh +VERISTAT_BINARY=/mnt/vmtest/selftests/bpf/veristat +VERISTAT_MANIFEST=/mnt/vmtest/manifest.json +VERISTAT_RESULTS=/mnt/vmtest/veristat-results +``` + +The entrypoint records `veristat:` in `STATUS_FILE` (default +`/mnt/vmtest/exitstatus`) and runs the kernel-splat check. The host `run.sh` +collects those statuses after the guest exits and fails the action on errors. +Per-corpus outcomes remain in the result JSON files. + +When upgrading from v4, migrate `kernel-test: run_veristat` callers to this +entrypoint and prepare their manifests before booting the VM. diff --git a/run-vmtest/README.md b/run-vmtest/README.md index 78b6ab84..6dd31c6c 100644 --- a/run-vmtest/README.md +++ b/run-vmtest/README.md @@ -1,6 +1,7 @@ # Run vmtest -This action is designed to run Linux Kernel BPF selftests. +This action boots a supplied Linux kernel and runs a guest workload. The default +guest script runs Linux kernel BPF selftests. It expects kernel binaries as well as test runner binaries as input, and executes the test runners with a given kernel using the [vmtest @@ -25,6 +26,9 @@ bpftool checks, and JSON summaries run only for `run-bpf-selftests.sh`. Custom scripts share VM setup and exitstatus collection, and provide their own workload setup and results. +For veristat, select `run-veristat.sh` and provide prepared corpora as described +in [run-veristat](../run-veristat/README.md#run-with-run-vmtest). + ## Required inputs * `arch` - Target architecture diff --git a/run-vmtest/run-bpf-selftests.sh b/run-vmtest/run-bpf-selftests.sh index 151e62c6..d29584d5 100755 --- a/run-vmtest/run-bpf-selftests.sh +++ b/run-vmtest/run-bpf-selftests.sh @@ -94,46 +94,6 @@ test_progs-bpf_gcc() { test_progs_helper "-bpf_gcc" "" } -export VERISTAT_TARGET=${VERISTAT_TARGET:-kernel} -export VERISTAT_CONFIGS=${VERISTAT_CONFIGS:-/mnt/vmtest/ci/vmtest/configs} -export WORKING_DIR=$(pwd) # veristat config expects this variable - -run_veristat() { - foldable start run_veristat_${VERISTAT_TARGET} "Running veristat ${VERISTAT_TARGET}" - # Make veristat commands visible in the log - if [ -o xtrace ]; then - xtrace_was_on="1" - else - xtrace_was_on="" - set -x - fi - - ( - # shellcheck source=ci/vmtest/configs/run_veristat.default.cfg - # shellcheck source=ci/vmtest/configs/run_veristat.meta.cfg - source "${VERISTAT_CONFIGS}/run_veristat.${VERISTAT_TARGET}.cfg" - pushd "${VERISTAT_OBJECTS_DIR}" - - args=() - args+=(-o csv) - args+=(-q) - args+=(-e file,prog,verdict,states) - args+=(${VERISTAT_CFG_FILE:+-f@$VERISTAT_CFG_FILE}) - - "${SELFTESTS_BPF}/veristat" "${args[@]}" ${VERISTAT_OBJECTS_GLOB} > \ - "${OUTPUT_DIR}/${VERISTAT_OUTPUT}" - - echo "run_veristat_${VERISTAT_TARGET}:$?" >> ${STATUS_FILE} - popd - ) - - # Hide commands again - if [ -z "$xtrace_was_on" ]; then - set +x - fi - foldable end run_veristat_${VERISTAT_TARGET} -} - foldable end vm_init foldable start kernel_config "Kconfig" diff --git a/run-vmtest/run-veristat.sh b/run-vmtest/run-veristat.sh new file mode 100755 index 00000000..cb831aa5 --- /dev/null +++ b/run-vmtest/run-veristat.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +status=0 +python3 "${SCRIPT_DIR}/../run-veristat/run_veristat.py" \ + --veristat "${VERISTAT_BINARY}" \ + --manifest "${VERISTAT_MANIFEST}" \ + --output "${VERISTAT_RESULTS}" || status=$? + +printf 'veristat:%s\n' "$status" >> "${STATUS_FILE:-/mnt/vmtest/exitstatus}" +"${SCRIPT_DIR}/check-kernel-splats.sh" From 2b211c576ee50767f4d8bfff0139b5d57a6a103c Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Mon, 9 Nov 2026 12:59:22 -0800 Subject: [PATCH 7/7] run-veristat: cover silent measurement failures Successful tool exits can hide empty measurements, changed inputs, or missing completion receipts. Those cases can leave CI green while its comparison inputs are unusable or have the wrong object identity. Cover the CLI and receipt boundaries, and verify that the shell entrypoint records an incomplete measurement as a failing status. Run the existing script-test workflow on main pushes, matching the repository default branch. Its repository-wide discovery includes the new veristat suite. Signed-off-by: Ihor Solodrai --- .github/workflows/scripts_tests.yml | 2 +- run-veristat/tests/run-test.sh | 3 + run-veristat/tests/test_runner.py | 92 +++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) create mode 100755 run-veristat/tests/run-test.sh create mode 100644 run-veristat/tests/test_runner.py diff --git a/.github/workflows/scripts_tests.yml b/.github/workflows/scripts_tests.yml index 4cac0a1e..9caad9e5 100644 --- a/.github/workflows/scripts_tests.yml +++ b/.github/workflows/scripts_tests.yml @@ -5,7 +5,7 @@ on: pull_request: push: branches: - - master + - main jobs: test_run_vmtest: diff --git a/run-veristat/tests/run-test.sh b/run-veristat/tests/run-test.sh new file mode 100755 index 00000000..1c3d2736 --- /dev/null +++ b/run-veristat/tests/run-test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=.. python3 -m unittest -bv test_runner diff --git a/run-veristat/tests/test_runner.py b/run-veristat/tests/test_runner.py new file mode 100644 index 00000000..d52e0084 --- /dev/null +++ b/run-veristat/tests/test_runner.py @@ -0,0 +1,92 @@ +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +import run_veristat as runner + + +class RunnerTest(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.results = self.root / "results" + (self.root / "object.bpf.o").write_bytes(b"BPF object") + self.corpus = { + "name": "test", + "objects": {"object.bpf.o": hashlib.sha256(b"BPF object").hexdigest()}, + } + self.manifest = self.root / "manifest.json" + self.manifest.write_text(json.dumps({"version": 1, "corpora": [self.corpus]})) + self.csv = ( + "file_name,prog_name,verdict,total_states\nobject.bpf.o,prog,failure,0\n" + ) + + def tool_output(self, command, stdout, stderr, **kwargs): + stdout.write(self.csv) + return subprocess.CompletedProcess(command, 0) + + def invoke(self): + args = [ + "run_veristat", + "--veristat=/usr/bin/true", + f"--manifest={self.manifest}", + f"--output={self.results}", + ] + with patch.object(sys, "argv", args), patch.object( + runner.subprocess, "run", side_effect=self.tool_output + ): + return runner.main() + + def test_empty_measurement_does_not_pass(self): + self.csv = self.csv.splitlines()[0] + "\n" + self.assertNotEqual(self.invoke(), 0) + + def test_changed_object_does_not_pass(self): + (self.root / "object.bpf.o").write_bytes(b"changed") + self.assertNotEqual(self.invoke(), 0) + + def test_completed_receipt_uses_object_basename(self): + (self.root / "moved").mkdir() + (self.root / "object.bpf.o").rename(self.root / "moved/object.bpf.o") + self.corpus["objects"] = { + "moved/object.bpf.o": self.corpus["objects"]["object.bpf.o"] + } + self.manifest.write_text(json.dumps({"version": 1, "corpora": [self.corpus]})) + self.assertEqual(self.invoke(), 0) + status = json.loads((self.results / "test/status.json").read_text()) + self.assertEqual(status["state"], "measured") + self.assertEqual( + status["corpus_sha256"], + "730cbbe4991552fc640e8bb57604945c63e4afa3f5a57cbcce831e1ca10d6a40", + ) + + def test_guest_records_incomplete_measurement_as_failure(self): + script = Path(runner.__file__).parent.parent / "run-vmtest/run-veristat.sh" + result = subprocess.run( + [str(script)], + env=dict( + os.environ, + VERISTAT_BINARY="/usr/bin/true", + VERISTAT_MANIFEST=str(self.manifest), + VERISTAT_RESULTS=str(self.results), + STATUS_FILE=str(self.root / "exitstatus"), + OUTPUT_DIR=str(self.root), + SPLAT_DENYLIST_FILE="", + SPLAT_ALLOWLIST_FILE="", + SPLAT_LOG_FILE=str(self.root / "kernel_splats.log"), + ), + capture_output=True, + text=True, + ) + self.assertIn( + "veristat:1", + (self.root / "exitstatus").read_text().splitlines(), + result.stderr, + )