gh-158895: Teach libregrtest to output github job summaries

Lib/test/libregrtest/main.py

@@ -15,6 +15,7 @@
 from .findtests import findtests, split_test_packages, list_cases, collect_cases
 from .logger import Logger
 from .pgo import setup_pgo_tests
+from .reporter import get_reporter
 from .result import TestResult
 from .results import TestResults, EXITCODE_INTERRUPTED
 from .runtests import RunTests, HuntRefleak
@@ -68,6 +69,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False):
 
         # Logger
         self.logger = Logger(self.results, self.quiet, self.pgo)
+        self.reporter = get_reporter()
 
         # Actions
         self.want_header: bool = ns.header
@@ -284,8 +286,10 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests:
             forever=False,
             fail_fast=False,
             match_tests_dict=match_tests_dict,
-            output_on_failure=False)
+            output_on_failure=False,
+            final_run=True)
         self.logger.set_tests(runtests)
+        self.reporter.run_started(runtests)
 
         msg = f"Re-running {len(tests)} failed tests in verbose mode"
         if not self.single_process:
@@ -395,6 +399,7 @@ def run_test(
             result = run_single_test(test_name, runtests)
 
         self.results.accumulate_result(result, runtests)
+        self.reporter.test_result(result)
 
         return result
 
@@ -459,7 +464,8 @@ def get_state(self) -> str:
 
     def _run_tests_mp(self, runtests: RunTests, num_workers: int) -> None:
         from .run_workers import RunWorkers
-        RunWorkers(num_workers, runtests, self.logger, self.results).run()
+        RunWorkers(num_workers, runtests, self.logger, self.results,
+                   self.reporter).run()
 
     def finalize_tests(self, coverage: trace.CoverageResults | None) -> None:
         if self.next_single_filename:
@@ -480,6 +486,8 @@ def finalize_tests(self, coverage: trace.CoverageResults | None) -> None:
         if self.junit_filename:
             self.results.write_junit(self.junit_filename)
 
+        self.reporter.summary(self.results, self.get_state())
+
     def display_summary(self) -> None:
         if self.first_runtests is None:
             raise ValueError(
@@ -517,6 +525,7 @@ def create_run_tests(self, tests: TestTuple) -> RunTests:
             hunt_refleak=self.hunt_refleak,
             test_dir=self.test_dir,
             use_junit=(self.junit_filename is not None),
+            final_run=not self.want_rerun or bool(self.python_cmd),
             coverage=self.coverage,
             memory_limit=self.memory_limit,
             gc_threshold=self.gc_threshold,
@@ -568,6 +577,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
             runtests = runtests.copy(tests=case_ids, case_groups=case_groups)
         self.first_runtests = runtests
         self.logger.set_tests(runtests)
+        self.reporter.run_started(runtests)
 
         if (runtests.hunt_refleak is not None) and (not self.num_workers):
             # gh-109739: WindowsLoadTracker thread interferes with refleak check

Lib/test/libregrtest/reporter.py

@@ -0,0 +1,137 @@
+import html
+import os
+import pathlib
+import re
+from _colorize import decolor  # type: ignore[import-not-found]
+from typing import TextIO
+
+from .findtests import findtestdir
+from .result import TestResult
+from .results import TestResults
+from .runtests import RunTests
+from .utils import count
+
+
+class Reporter:
+    """Report test failures, for example to a CI service. A no-op here."""
+
+    def run_started(self, runtests: RunTests) -> None:
+        """Called at the start of each run, and in the test process."""
+
+    def test_case_failed(self, test_case: str, traceback: str,
+                         stream: TextIO) -> None:
+        """Called in the test process after a test case failed."""
+
+    def test_result(self, result: TestResult) -> None:
+        """Called in the main process after a test file completed."""
+
+    def summary(self, results: TestResults, state: str) -> None:
+        """Called in the main process after all runs."""
+
+    @staticmethod
+    def parse_traceback(traceback: str) -> tuple[str | None, str | None, str]:
+        """Return (filename, line, exception) of a failure."""
+        # A frame is its "File" line and its source lines
+        frames = list(re.finditer(
+            r'^ *File "(.*)", line (\d+).*\n(?:    .*\n)*', traceback,
+            re.MULTILINE))
+        filename = line = None
+        exception = traceback
+        if frames:
+            filename, line = frames[0].groups()
+            exception = traceback[frames[-1].end():]
+        # Prefer the last frame in a test file, the assertion
+        for frame in frames:
+            if pathlib.Path(frame[1]).name.startswith("test_"):
+                filename, line = frame.groups()
+        return filename, line, exception.strip()
+
+
+def get_reporter() -> Reporter:
+    if GitHubActionsReporter.enabled():
+        return GitHubActionsReporter()
+    return Reporter()
+
+
+class GitHubActionsReporter(Reporter):
+    """Annotate failures in the job log and list them in the job summary."""
+
+    ESCAPE_DATA = str.maketrans({"%": "%25", "\r": "%0D", "\n": "%0A"})
+    ESCAPE_PROPERTY = ESCAPE_DATA | str.maketrans({":": "%3A", ",": "%2C"})
+
+    @staticmethod
+    def enabled() -> bool:
+        return "GITHUB_STEP_SUMMARY" in os.environ
+
+    def run_started(self, runtests: RunTests) -> None:
+        self.runtests = runtests
+
+    def test_case_failed(self, test_case: str, traceback: str,
+                         stream: TextIO) -> None:
+        if not self.runtests.final_run:
+            return
+        filename, line, exception = self.parse_traceback(decolor(traceback))
+        self.annotate(test_case, exception, filename, line, stream)
+
+    def test_result(self, result: TestResult) -> None:
+        if not self.runtests.final_run:
+            return
+        if not result.is_failed(self.runtests.fail_env_changed):
+            return
+        # Already annotated by test_case_failed()
+        if result.failed_test_cases:
+            return
+        # "test.test_x.test_y" is test_x/test_y.py or test_x/test_y/__init__.py
+        name = result.test_name.removeprefix("test.")
+        path = pathlib.Path(findtestdir(self.runtests.test_dir),
+                            *name.split("."))
+        filename = None
+        for candidate in (path.with_suffix(".py"), path / "__init__.py"):
+            if candidate.exists():
+                filename = str(candidate)
+        message = "\n".join([str(result), *(result.env_changed_reasons or ())])
+        self.annotate(result.test_name, decolor(message), filename)
+
+    def annotate(self, title: str, message: str, filename: str | None,
+                 line: str | None = None,
+                 stream: TextIO | None = None) -> None:
+        if filename:
+            # Make installed stdlib paths relative to the checkout
+            path = pathlib.Path(filename)
+            stdlib_dir = pathlib.Path(os.__file__).parent
+            if path.is_relative_to(stdlib_dir):
+                filename = ("Lib" / path.relative_to(stdlib_dir)).as_posix()
+            # GitHub needs a line to show the annotation on the file
+            line = line or "1"
+        props = {"file": filename, "line": line, "title": title}
+        props_text = ",".join(f"{key}={value.translate(self.ESCAPE_PROPERTY)}"
+                              for key, value in props.items() if value)
+        print(f"::error {props_text}::{message.translate(self.ESCAPE_DATA)}",
+              file=stream, flush=True)
+
+    def summary(self, results: TestResults, state: str) -> None:
+        failed = [result for result in results.rerun_results
+                  if result.is_failed(self.runtests.fail_env_changed)]
+        if not failed:
+            return
+        ncases = sum(len(result.failed_test_cases) for result in failed)
+        blocks = [f"## {decolor(state)}: "
+                  f"{count(len(failed), 'test file')} and "
+                  f"{count(ncases, 'test case')} failed"]
+        for result in failed:
+            blocks.append(f"### {decolor(str(result))}")
+            if result.env_changed_reasons:
+                blocks.append("\n".join(
+                    f"- {reason}" for reason in result.env_changed_reasons))
+            for name, traceback in result.failed_test_cases:
+                # Expand short tracebacks if there are only a few
+                if ncases <= 5 and traceback.count("\n") < 30:
+                    details = "<details open>"
+                else:
+                    details = "<details>"
+                blocks += [f"{details}<summary>{html.escape(name)}</summary>",
+                           f"```pytb\n{decolor(traceback).rstrip()}\n```",
+                           "</details>"]
+        with open(os.environ["GITHUB_STEP_SUMMARY"], "a",
+                  encoding="utf-8") as fp:
+            fp.write("".join(f"{block}\n\n" for block in blocks))

Lib/test/libregrtest/result.py

@@ -178,6 +178,10 @@ def __str__(self) -> str:
     def has_meaningful_duration(self):
         return State.has_meaningful_duration(self.state)
 
+    @property
+    def failed_test_cases(self) -> list[tuple[str, str]]:
+        return (self.errors or []) + (self.failures or [])
+
     def set_env_changed(self, *reasons):
         if self.state is None or self.state == State.PASSED:
             self.state = State.ENV_CHANGED

Lib/test/libregrtest/run_workers.py

@@ -16,6 +16,7 @@
 from test.support import os_helper, MS_WINDOWS
 
 from .logger import Logger
+from .reporter import Reporter
 from .result import TestResult, State
 from .results import TestResults
 from .runtests import RunTests, WorkerRunTests, JsonFile, JsonFileType
@@ -499,13 +500,15 @@ def get_running(workers: list[WorkerThread]) -> str | None:
 
 class RunWorkers:
     def __init__(self, num_workers: int, runtests: RunTests,
-                 logger: Logger, results: TestResults) -> None:
+                 logger: Logger, results: TestResults,
+                 reporter: Reporter) -> None:
         self.num_workers = num_workers
         self.runtests = runtests
         self.logger = logger
         self.log = logger.log
         self.display_progress = logger.display_progress
         self.results: TestResults = results
+        self.reporter = reporter
         self.live_worker_count = 0
 
         self.output: queue.Queue[QueueContent] = queue.Queue()
@@ -624,6 +627,8 @@ def _process_result(self, item: QueueOutput) -> TestResult:
             stdout = mp_result.worker_stdout
             if stdout:
                 print(stdout, flush=True)
+        # Report after the output: env changed warnings, crash traceback
+        self.reporter.test_result(result)
 
         return result
 

Lib/test/libregrtest/runtests.py

@@ -93,6 +93,8 @@ class RunTests:
     hunt_refleak: HuntRefleak | None
     test_dir: StrPath | None
     use_junit: bool
+    # Failed tests are not re-run after this run
+    final_run: bool
     coverage: bool
     memory_limit: str | None
     gc_threshold: int | None

Lib/test/libregrtest/setup.py

@@ -11,7 +11,9 @@
 from _colorize import can_colorize  # type: ignore[import-not-found]
 
 from .filter import set_match_tests
+from .reporter import get_reporter
 from .runtests import RunTests
+from .testresult import RegressionTestResult
 from .utils import (
     setup_unraisable_hook, setup_threading_excepthook,
     adjust_rlimit_nofile)
@@ -112,9 +114,11 @@ def setup_tests(runtests: RunTests) -> None:
 
     set_match_tests(runtests.match_tests)
 
+    reporter = get_reporter()
+    reporter.run_started(runtests)
+    RegressionTestResult.reporter = reporter
     if runtests.use_junit:
         support.junit_xml_list = []
-        from .testresult import RegressionTestResult
         RegressionTestResult.USE_XML = True
     else:
         support.junit_xml_list = None

Lib/test/libregrtest/testresult.py

@@ -9,10 +9,12 @@
 import traceback
 import unittest
 from test import support
+from test.libregrtest.reporter import Reporter
 from test.libregrtest.utils import sanitize_xml
 
 class RegressionTestResult(unittest.TextTestResult):
     USE_XML = False
+    reporter = Reporter()
 
     def __init__(self, stream, descriptions, verbosity):
         super().__init__(stream=stream, descriptions=descriptions,
@@ -130,6 +132,14 @@ def addUnexpectedSuccess(self, test):
         self._add_result(test, outcome='UNEXPECTED_SUCCESS')
         super().addUnexpectedSuccess(test)
 
+    def printErrorList(self, flavour, errors):
+        for test, err in errors:
+            super().printErrorList(flavour, [(test, err)])
+            self.report_failure(test, err)
+
+    def report_failure(self, test, err):
+        self.reporter.test_case_failed(str(test), err, self.stream)
+
     def get_xml_element(self):
         if not self.USE_XML:
             raise ValueError("USE_XML is false")
@@ -146,6 +156,9 @@ def __init__(self, stream, buffer=False):
 
     def run(self, test):
         test(self.result)
+        # Failures are not printed in quiet mode
+        for test, err in self.result.errors + self.result.failures:
+            self.result.report_failure(test, err)
         return self.result
 
 def get_test_runner_class(verbosity, buffer=False):