diff --git a/doc/glossary.md b/doc/glossary.md index c5baca6a..d74b7425 100644 --- a/doc/glossary.md +++ b/doc/glossary.md @@ -107,6 +107,22 @@ DVSim provides these standard regression targets for all DUTs: - `all` (runs all tests with the preset reseeds, without coverage) - `all_once` (run all tests with only a single randomly chosen seed) +A regression can include other regressions by name with the `regressions` key, and then runs their tests along with its own: + +```hjson +regressions: [ + { + name: regression_a_b + regressions: ["regression_a", "regression_b"] + } +] +``` + +Inclusion is transitive, and a test reached twice runs once. +Only the tests of an included regression are taken. +Its sim modes, run modes, options and reseed do not apply, so the including regression sets its own. +A regression that includes one running all tests, such as `all`, runs all tests too. + ## Reseeds ## Run diff --git a/src/dvsim/regression.py b/src/dvsim/regression.py index f438dbad..2a413251 100644 --- a/src/dvsim/regression.py +++ b/src/dvsim/regression.py @@ -28,6 +28,10 @@ def __init__(self, regdict) -> None: self.tests = None self.test_names = [] + # Names of other regressions whose tests this one runs as well. Only their tests are taken, + # so the modes and options of this regression apply to every test it ends up with. + self.regressions = [] + self.reseed = None self.excl_tests = [] # TODO: add support for this self.en_sim_modes = [] @@ -74,7 +78,10 @@ def create_regressions(regdicts, sim_cfg, tests): regression_objs.append(new_regression) Regression.item_names.append(new_regression.name) - # Pass 2: Process dependencies + # Pass 2: Add the tests of the included regressions + Regression._include_regressions(regression_objs) + + # Pass 3: Process dependencies build_modes = getattr(sim_cfg, "build_modes", []) run_modes = getattr(sim_cfg, "run_modes", []) @@ -161,6 +168,55 @@ def create_regressions(regdicts, sim_cfg, tests): # Return the list of tests return regression_objs + @staticmethod + def _include_regressions(regression_objs: list["Regression"]) -> None: + """Add to each regression the tests of the regressions it includes. + + The `regressions` key names the regressions to include, and inclusion is transitive. A + regression that sets `regressions` but no `tests` runs only the included tests. Including + a regression that runs all tests, because it leaves `tests` unset, makes the including + regression run all tests too. + """ + regr_map = {regr.name: regr for regr in regression_objs} + resolved: dict[str, list[str] | None] = {} + + def resolve(regr: Regression, chain: list[str]) -> list[str] | None: + if regr.name in resolved: + return resolved[regr.name] + if not regr.regressions: + return regr.tests + if regr.name in chain: + log.error( + 'Regression "%s" includes itself through %s', + regr.name, + ", ".join(f'"{name}"' for name in [*chain, regr.name]), + ) + sys.exit(1) + + tests: list[str] | None = list(regr.tests or []) + for name in regr.regressions: + included = regr_map.get(name) + if included is None: + log.error( + 'Regression "%s" includes "%s", which is not a regression', + regr.name, + name, + ) + sys.exit(1) + included_tests = resolve(included, [*chain, regr.name]) + if included_tests is None: + tests = None + break + tests.extend(included_tests) + + # Keep the first occurrence of each test, in order + resolved[regr.name] = None if tests is None else list(dict.fromkeys(tests)) + return resolved[regr.name] + + for regr in regression_objs: + if regr.regressions: + regr.tests = resolve(regr, []) + def merge_regression_opts(self) -> None: processed_build_modes = [] for test in self.tests: diff --git a/tests/test_regression.py b/tests/test_regression.py new file mode 100644 index 00000000..d450213a --- /dev/null +++ b/tests/test_regression.py @@ -0,0 +1,149 @@ +# Copyright lowRISC contributors (OpenTitan project). +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for regressions that include other regressions.""" + +from dataclasses import dataclass +from types import SimpleNamespace + +import pytest + +from dvsim.regression import Regression +from dvsim.test import Test as DvsimTest + +TEST_NAMES = ("a", "b", "c", "d") + + +@dataclass(frozen=True) +class FakeTest: + """The part of a test that regression resolution reads. Hashable, as tests go in a set.""" + + name: str + + +@pytest.fixture(autouse=True) +def fresh_name_registries(monkeypatch: pytest.MonkeyPatch) -> None: + """Regression and Test record every name they create on the class itself.""" + monkeypatch.setattr(Regression, "item_names", []) + monkeypatch.setattr(DvsimTest, "item_names", []) + + +def create(regdicts: list[dict]) -> dict[str, Regression]: + """Create the regressions of a cfg that defines the tests in TEST_NAMES.""" + tests = [FakeTest(name) for name in TEST_NAMES] + sim_cfg = SimpleNamespace(tests=tests, en_build_modes=[], en_run_modes=[]) + regrs = Regression.create_regressions(regdicts, sim_cfg, tests) + return {regr.name: regr for regr in regrs} + + +def names_of(regr: Regression) -> set[str]: + """Names of the tests a regression runs.""" + return {test.name for test in regr.tests} + + +class TestIncludedRegressions: + def test_runs_the_union_of_the_included_tests(self) -> None: + """Setting only `regressions` must not fall back to running every test.""" + regrs = create( + [ + {"name": "regression_a", "tests": ["a", "b"]}, + {"name": "regression_b", "tests": ["c"]}, + {"name": "regression_a_b", "regressions": ["regression_a", "regression_b"]}, + ] + ) + + assert names_of(regrs["regression_a_b"]) == {"a", "b", "c"} + assert names_of(regrs["regression_a"]) == {"a", "b"} + assert names_of(regrs["regression_b"]) == {"c"} + + def test_keeps_its_own_tests(self) -> None: + regrs = create( + [ + {"name": "regression_a", "tests": ["a", "b"]}, + {"name": "combined", "tests": ["d"], "regressions": ["regression_a"]}, + ] + ) + + assert names_of(regrs["combined"]) == {"a", "b", "d"} + + def test_merges_with_a_declaration_in_another_file(self) -> None: + """A cfg adds `regressions` to a regression that a common cfg already declares.""" + regrs = create( + [ + {"name": "combined", "tests": ["a"], "reseed": 1}, + {"name": "regression_b", "tests": ["c"]}, + {"name": "combined", "regressions": ["regression_b"]}, + ] + ) + + assert names_of(regrs["combined"]) == {"a", "c"} + assert regrs["combined"].reseed == 1 + + def test_inclusion_is_transitive(self) -> None: + regrs = create( + [ + {"name": "outer", "regressions": ["middle"]}, + {"name": "middle", "tests": ["a"], "regressions": ["inner"]}, + {"name": "inner", "tests": ["d"]}, + ] + ) + + assert names_of(regrs["outer"]) == {"a", "d"} + assert names_of(regrs["middle"]) == {"a", "d"} + + def test_test_names_keep_order_without_duplicates(self) -> None: + regrs = create( + [ + {"name": "first", "tests": ["b", "a"]}, + {"name": "second", "tests": ["a", "c"]}, + {"name": "both", "regressions": ["first", "second"]}, + ] + ) + + assert regrs["both"].test_names == ["b", "a", "c"] + + def test_including_an_all_tests_regression_runs_all_tests(self) -> None: + regrs = create( + [ + {"name": "all"}, + {"name": "everything", "tests": ["a"], "regressions": ["all"]}, + ] + ) + + assert names_of(regrs["everything"]) == set(TEST_NAMES) + + def test_options_of_an_included_regression_do_not_apply(self) -> None: + regrs = create( + [ + {"name": "regression_b", "tests": ["c"], "run_opts": ["+opt_b=1"]}, + { + "name": "combined", + "regressions": ["regression_b"], + "run_opts": ["+opt_combined=1"], + }, + ] + ) + + assert regrs["combined"].run_opts == ["+opt_combined=1"] + assert regrs["regression_b"].run_opts == ["+opt_b=1"] + + +class TestIncludedRegressionErrors: + @pytest.mark.parametrize("name", ["missing", "a"]) + def test_including_something_that_is_not_a_regression_exits(self, name: str) -> None: + with pytest.raises(SystemExit): + create([{"name": "combined", "regressions": [name]}]) + + def test_including_itself_exits(self) -> None: + with pytest.raises(SystemExit): + create([{"name": "combined", "regressions": ["combined"]}]) + + def test_a_cycle_exits(self) -> None: + with pytest.raises(SystemExit): + create( + [ + {"name": "x", "tests": ["a"], "regressions": ["y"]}, + {"name": "y", "regressions": ["x"]}, + ] + )