From 8102bd8f6e59fc4550eb2ac1cd501ccbc15b0d1d Mon Sep 17 00:00:00 2001 From: Martin Velay Date: Mon, 5 Oct 2026 13:43:52 +0200 Subject: [PATCH] feat: let a regression include other regressions A regression's tests key only accepts test names, so a regression made of two smaller ones had to repeat both lists. The new regressions key names other regressions whose tests it also runs. Only their tests are taken, and the modes and options of the including regression apply to all of them. Signed-off-by: Martin Velay --- doc/glossary.md | 16 +++++ src/dvsim/regression.py | 58 ++++++++++++++- tests/test_regression.py | 149 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 222 insertions(+), 1 deletion(-) create mode 100644 tests/test_regression.py diff --git a/doc/glossary.md b/doc/glossary.md index c5baca6a..d74b7425 100644 --- a/doc/glossary.md +++ b/doc/glossary.md @@ -107,6 +107,22 @@ DVSim provides these standard regression targets for all DUTs: - `all` (runs all tests with the preset reseeds, without coverage) - `all_once` (run all tests with only a single randomly chosen seed) +A regression can include other regressions by name with the `regressions` key, and then runs their tests along with its own: + +```hjson +regressions: [ + { + name: regression_a_b + regressions: ["regression_a", "regression_b"] + } +] +``` + +Inclusion is transitive, and a test reached twice runs once. +Only the tests of an included regression are taken. +Its sim modes, run modes, options and reseed do not apply, so the including regression sets its own. +A regression that includes one running all tests, such as `all`, runs all tests too. + ## Reseeds ## Run diff --git a/src/dvsim/regression.py b/src/dvsim/regression.py index f438dbad..2a413251 100644 --- a/src/dvsim/regression.py +++ b/src/dvsim/regression.py @@ -28,6 +28,10 @@ def __init__(self, regdict) -> None: self.tests = None self.test_names = [] + # Names of other regressions whose tests this one runs as well. Only their tests are taken, + # so the modes and options of this regression apply to every test it ends up with. + self.regressions = [] + self.reseed = None self.excl_tests = [] # TODO: add support for this self.en_sim_modes = [] @@ -74,7 +78,10 @@ def create_regressions(regdicts, sim_cfg, tests): regression_objs.append(new_regression) Regression.item_names.append(new_regression.name) - # Pass 2: Process dependencies + # Pass 2: Add the tests of the included regressions + Regression._include_regressions(regression_objs) + + # Pass 3: Process dependencies build_modes = getattr(sim_cfg, "build_modes", []) run_modes = getattr(sim_cfg, "run_modes", []) @@ -161,6 +168,55 @@ def create_regressions(regdicts, sim_cfg, tests): # Return the list of tests return regression_objs + @staticmethod + def _include_regressions(regression_objs: list["Regression"]) -> None: + """Add to each regression the tests of the regressions it includes. + + The `regressions` key names the regressions to include, and inclusion is transitive. A + regression that sets `regressions` but no `tests` runs only the included tests. Including + a regression that runs all tests, because it leaves `tests` unset, makes the including + regression run all tests too. + """ + regr_map = {regr.name: regr for regr in regression_objs} + resolved: dict[str, list[str] | None] = {} + + def resolve(regr: Regression, chain: list[str]) -> list[str] | None: + if regr.name in resolved: + return resolved[regr.name] + if not regr.regressions: + return regr.tests + if regr.name in chain: + log.error( + 'Regression "%s" includes itself through %s', + regr.name, + ", ".join(f'"{name}"' for name in [*chain, regr.name]), + ) + sys.exit(1) + + tests: list[str] | None = list(regr.tests or []) + for name in regr.regressions: + included = regr_map.get(name) + if included is None: + log.error( + 'Regression "%s" includes "%s", which is not a regression', + regr.name, + name, + ) + sys.exit(1) + included_tests = resolve(included, [*chain, regr.name]) + if included_tests is None: + tests = None + break + tests.extend(included_tests) + + # Keep the first occurrence of each test, in order + resolved[regr.name] = None if tests is None else list(dict.fromkeys(tests)) + return resolved[regr.name] + + for regr in regression_objs: + if regr.regressions: + regr.tests = resolve(regr, []) + def merge_regression_opts(self) -> None: processed_build_modes = [] for test in self.tests: diff --git a/tests/test_regression.py b/tests/test_regression.py new file mode 100644 index 00000000..d450213a --- /dev/null +++ b/tests/test_regression.py @@ -0,0 +1,149 @@ +# Copyright lowRISC contributors (OpenTitan project). +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for regressions that include other regressions.""" + +from dataclasses import dataclass +from types import SimpleNamespace + +import pytest + +from dvsim.regression import Regression +from dvsim.test import Test as DvsimTest + +TEST_NAMES = ("a", "b", "c", "d") + + +@dataclass(frozen=True) +class FakeTest: + """The part of a test that regression resolution reads. Hashable, as tests go in a set.""" + + name: str + + +@pytest.fixture(autouse=True) +def fresh_name_registries(monkeypatch: pytest.MonkeyPatch) -> None: + """Regression and Test record every name they create on the class itself.""" + monkeypatch.setattr(Regression, "item_names", []) + monkeypatch.setattr(DvsimTest, "item_names", []) + + +def create(regdicts: list[dict]) -> dict[str, Regression]: + """Create the regressions of a cfg that defines the tests in TEST_NAMES.""" + tests = [FakeTest(name) for name in TEST_NAMES] + sim_cfg = SimpleNamespace(tests=tests, en_build_modes=[], en_run_modes=[]) + regrs = Regression.create_regressions(regdicts, sim_cfg, tests) + return {regr.name: regr for regr in regrs} + + +def names_of(regr: Regression) -> set[str]: + """Names of the tests a regression runs.""" + return {test.name for test in regr.tests} + + +class TestIncludedRegressions: + def test_runs_the_union_of_the_included_tests(self) -> None: + """Setting only `regressions` must not fall back to running every test.""" + regrs = create( + [ + {"name": "regression_a", "tests": ["a", "b"]}, + {"name": "regression_b", "tests": ["c"]}, + {"name": "regression_a_b", "regressions": ["regression_a", "regression_b"]}, + ] + ) + + assert names_of(regrs["regression_a_b"]) == {"a", "b", "c"} + assert names_of(regrs["regression_a"]) == {"a", "b"} + assert names_of(regrs["regression_b"]) == {"c"} + + def test_keeps_its_own_tests(self) -> None: + regrs = create( + [ + {"name": "regression_a", "tests": ["a", "b"]}, + {"name": "combined", "tests": ["d"], "regressions": ["regression_a"]}, + ] + ) + + assert names_of(regrs["combined"]) == {"a", "b", "d"} + + def test_merges_with_a_declaration_in_another_file(self) -> None: + """A cfg adds `regressions` to a regression that a common cfg already declares.""" + regrs = create( + [ + {"name": "combined", "tests": ["a"], "reseed": 1}, + {"name": "regression_b", "tests": ["c"]}, + {"name": "combined", "regressions": ["regression_b"]}, + ] + ) + + assert names_of(regrs["combined"]) == {"a", "c"} + assert regrs["combined"].reseed == 1 + + def test_inclusion_is_transitive(self) -> None: + regrs = create( + [ + {"name": "outer", "regressions": ["middle"]}, + {"name": "middle", "tests": ["a"], "regressions": ["inner"]}, + {"name": "inner", "tests": ["d"]}, + ] + ) + + assert names_of(regrs["outer"]) == {"a", "d"} + assert names_of(regrs["middle"]) == {"a", "d"} + + def test_test_names_keep_order_without_duplicates(self) -> None: + regrs = create( + [ + {"name": "first", "tests": ["b", "a"]}, + {"name": "second", "tests": ["a", "c"]}, + {"name": "both", "regressions": ["first", "second"]}, + ] + ) + + assert regrs["both"].test_names == ["b", "a", "c"] + + def test_including_an_all_tests_regression_runs_all_tests(self) -> None: + regrs = create( + [ + {"name": "all"}, + {"name": "everything", "tests": ["a"], "regressions": ["all"]}, + ] + ) + + assert names_of(regrs["everything"]) == set(TEST_NAMES) + + def test_options_of_an_included_regression_do_not_apply(self) -> None: + regrs = create( + [ + {"name": "regression_b", "tests": ["c"], "run_opts": ["+opt_b=1"]}, + { + "name": "combined", + "regressions": ["regression_b"], + "run_opts": ["+opt_combined=1"], + }, + ] + ) + + assert regrs["combined"].run_opts == ["+opt_combined=1"] + assert regrs["regression_b"].run_opts == ["+opt_b=1"] + + +class TestIncludedRegressionErrors: + @pytest.mark.parametrize("name", ["missing", "a"]) + def test_including_something_that_is_not_a_regression_exits(self, name: str) -> None: + with pytest.raises(SystemExit): + create([{"name": "combined", "regressions": [name]}]) + + def test_including_itself_exits(self) -> None: + with pytest.raises(SystemExit): + create([{"name": "combined", "regressions": ["combined"]}]) + + def test_a_cycle_exits(self) -> None: + with pytest.raises(SystemExit): + create( + [ + {"name": "x", "tests": ["a"], "regressions": ["y"]}, + {"name": "y", "regressions": ["x"]}, + ] + )