diff --git a/audits/services.py b/audits/services.py index d0490ba0..00a0aa26 100644 --- a/audits/services.py +++ b/audits/services.py @@ -417,3 +417,71 @@ def cancel_run(run: AuditRun, user) -> bool: run.save(update_fields=["status", "finished_at"]) append_event(run.pk, "_run", "run_cancelled", {"by": user.username}) return True + + +class RunExportError(Exception): + """A run cannot be exported in the engine's JSON format (``reason`` says why).""" + + def __init__(self, reason: str, *, unsupported: bool = False): + super().__init__(reason) + self.reason = reason + # True when the run can never be exported (as opposed to "not finished yet"). + self.unsupported = unsupported + + +def export_engine_results(run: AuditRun) -> dict: + """Reassemble a completed run into the engine's ``AuditResults`` JSON document. + + Built from the persisted ``ScenarioResult`` rows, so no engine run is needed. + The document has the engine's own keys (``timestamp``, ``summary``, ``issues``, + ``recommendations``, ``results``) plus a ``run`` block with the provenance + frozen on the run, and ``studio_provenance.target_model`` (the target's model + id) when the run has one. Runs with ``n_repetitions > 1`` are not supported yet. + """ + from audits.events import ScenarioResult + from infra import engine + from scenarios.models import ScenarioSetVersionItem + + if run.status != AuditRun.Status.COMPLETED: + raise RunExportError(f"Run is {run.status}; only completed runs can be exported.") + if int((run.generation_parameters_snapshot or {}).get("n_repetitions") or 1) > 1: + raise RunExportError("Runs with repetitions cannot be exported in this format yet.", unsupported=True) + + items = { + str(vi.pk): vi + for vi in ScenarioSetVersionItem.objects.filter(version=run.scenario_set_version).select_related("scenario") + } + payloads = [] + for sr in ScenarioResult.objects.filter(run_id=run.id).order_by("id"): + item = items.get(sr.version_item_id) + payload = dict(sr.result or {}) + payload.setdefault("scenario_name", item.scenario.title if item else sr.version_item_id) + payload.setdefault("scenario_description", "") + payload.setdefault("conversation", []) + payload.setdefault("severity", "pass" if sr.status == "completed" else "ERROR") + payload.setdefault("issues_found", []) + payload.setdefault("positive_behaviors", []) + payload.setdefault("summary", payload.get("error", "")) + payload.setdefault("recommendations", []) + payloads.append(payload) + + document = engine.build_audit_results_export( + payloads, timestamp=run.finished_at.isoformat() if run.finished_at else None + ) + document["run"] = { + "id": run.id, + "name": run.name, + "simpleaudit_version": run.simpleaudit_version, + "git_commit": run.git_commit, + "started_at": run.started_at.isoformat() if run.started_at else None, + "finished_at": run.finished_at.isoformat() if run.finished_at else None, + "target": frozen_name(run, "target", with_id=True), + "auditor": frozen_name(run, "auditor", with_id=True), + "judge": frozen_name(run, "judge", with_id=True), + } + # The Every Eval Ever converter (evaleval/every_eval_ever#312) names the model + # from ``studio_provenance.target_model``; engine files carry no model name. + target_model_id = frozen_model(run, "target")["model_id"] + if target_model_id: + document["studio_provenance"] = {"target_model": target_model_id} + return document diff --git a/docs/architecture.md b/docs/architecture.md index 3739bafc..86ab2b47 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -98,6 +98,7 @@ data directory (`~/.simpleaudit-studio/embedded-pg`, override with | `/experiments/`, `/experiments//` | Experiments and their pass-rate pivot | | `/monitors/`, `/monitors//` | Monitors and drift charts | | `/runs//`, `/runs//results//` | Run detail (live) and per-scenario result | +| `/runs//export/?format=simpleaudit` | A completed run as the engine's own `AuditResults` JSON (loads with `AuditResults.load`), rebuilt from the stored `ScenarioResult` rows; `format=json` and `format=csv` give the Studio list | | `/compare/?runs=a,b` | Side-by-side comparison of completed runs | | `/scenarios/`, `/connections/` | Scenario library; connections (servers + API keys) and the models registered on them | | `/judges/`, `/judges//` | Judges: start from a SimpleAudit judge or write own criteria, edit (new version), clone, version history, live prompt preview (`/judges/preview/`) | diff --git a/infra/engine.py b/infra/engine.py index 6adc6aa4..29683ccc 100644 --- a/infra/engine.py +++ b/infra/engine.py @@ -880,3 +880,24 @@ def _tracing_correlation() -> Any: "n_repetitions": len(reps), "_language": language, } + + +def build_audit_results_export(payloads: list[dict[str, Any]], *, timestamp: str | None = None) -> dict[str, Any]: + """Assemble the engine's ``AuditResults.to_dict()`` document from stored results. + + ``payloads`` are persisted per-scenario ``AuditResult.to_dict()`` records. + Keys the engine's ``AuditResult`` does not define (Studio bookkeeping such as + ``trace_ids`` or ``_language``) are dropped, and the summary block is computed + by the engine itself, so the document keeps the engine's own shape and loads + with ``AuditResults.load``. + """ + _ensure_engine_available() + import dataclasses + + from simpleaudit.results import AuditResult, AuditResults + + fields = {f.name for f in dataclasses.fields(AuditResult)} + collection = AuditResults([AuditResult(**{k: v for k, v in p.items() if k in fields}) for p in payloads]) + if timestamp: + collection.timestamp = timestamp + return collection.to_dict() diff --git a/infra/tests/test_run_export_simpleaudit.py b/infra/tests/test_run_export_simpleaudit.py new file mode 100644 index 00000000..cd7c8c26 --- /dev/null +++ b/infra/tests/test_run_export_simpleaudit.py @@ -0,0 +1,171 @@ +"""Tests for ``GET /runs//export/?format=simpleaudit``. + +The export reassembles a finished run from its stored ``ScenarioResult`` rows +into the engine's own ``AuditResults`` JSON, so the tests round-trip it through +the engine: results stored from ``AuditResult.to_dict()`` must come back out as +the document ``AuditResults.to_dict()`` builds, and load with ``AuditResults.load``. +""" +import json +import tempfile +from pathlib import Path + +from django.test import Client, TestCase +from django.utils import timezone +from simpleaudit.results import AuditResult, AuditResults + +from audits.events import ScenarioResult +from audits.models import AuditRun +from infra.tests.factories import ( + AuditRunFactory, + MembershipFactory, + ProjectFactory, + ScenarioFactory, + ScenarioSetVersionFactory, + ScenarioSetVersionItemFactory, + UserFactory, +) + + +def _engine_result(name: str, severity: str) -> AuditResult: + return AuditResult( + scenario_name=name, + scenario_description=f"{name} description", + conversation=[ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there"}, + ], + severity=severity, + issues_found=[] if severity == "pass" else [f"{name} issue"], + positive_behaviors=["polite"], + summary=f"{name} summary", + recommendations=[] if severity == "pass" else [f"{name} fix"], + judgment={"severity": severity}, + target_input_tokens=11, + target_output_tokens=7, + ) + + +class RunExportSimpleAuditTest(TestCase): + def setUp(self): + pw = "testpass" + "123" + self.user = UserFactory() + self.user.set_password(pw) + self.user.save() + self.project = ProjectFactory() + MembershipFactory(user=self.user, project=self.project, role="admin") + self.client = Client(SERVER_NAME="localhost") + self.client.login(username=self.user.username, password=pw) + self.client.session["project_id"] = self.project.pk + self.client.session.save() + + version = ScenarioSetVersionFactory() + self.run = AuditRunFactory( + project=self.project, + scenario_set_version=version, + status=AuditRun.Status.COMPLETED, + finished_at=timezone.now(), + ) + self.engine_results = [_engine_result("Alpha", "pass"), _engine_result("Beta", "high")] + for result in self.engine_results: + item = ScenarioSetVersionItemFactory(version=version, scenario=ScenarioFactory(title=result.scenario_name)) + payload = result.to_dict() + # Studio adds bookkeeping keys that are not part of the engine's AuditResult. + payload["_language"] = "en" + payload["trace_ids"] = ["abc"] + ScenarioResult.objects.create( + run_id=self.run.id, version_item_id=str(item.pk), status="completed", result=payload + ) + + def _export(self, run=None): + return self.client.get(f"/runs/{(run or self.run).id}/export/?format=simpleaudit") + + def test_round_trip_matches_engine_document(self): + resp = self._export() + + self.assertEqual(resp.status_code, 200) + self.assertIn(f"audit_{self.run.id}_simpleaudit.json", resp["Content-Disposition"]) + document = json.loads(resp.content) + expected = AuditResults(self.engine_results).to_dict() + for key in ("summary", "issues", "recommendations", "results"): + self.assertEqual(document[key], expected[key], key) + self.assertEqual(document["summary"]["total_scenarios"], 2) + + def test_export_loads_with_the_engine(self): + document = json.loads(self._export().content) + + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "results.json" + path.write_text(json.dumps(document), encoding="utf-8") + loaded = AuditResults.load(str(path)) + + self.assertEqual([r.to_dict() for r in loaded.results], [r.to_dict() for r in self.engine_results]) + + def test_run_provenance_is_included(self): + document = json.loads(self._export().content) + + self.assertEqual(document["run"]["id"], self.run.id) + self.assertEqual(document["run"]["simpleaudit_version"], self.run.simpleaudit_version) + self.assertEqual(document["run"]["git_commit"], self.run.git_commit) + self.assertEqual(document["timestamp"], self.run.finished_at.isoformat()) + + def test_studio_provenance_names_the_frozen_target_model(self): + # The Every Eval Ever converter reads the model id from here. + self.run.target_config_snapshot = {"model_id": "acme/frozen-model", "display_name": "Frozen"} + self.run.save(update_fields=["target_config_snapshot"]) + + document = json.loads(self._export().content) + + self.assertEqual(document["studio_provenance"], {"target_model": "acme/frozen-model"}) + self.assertIn("run", document) + + def test_studio_provenance_is_omitted_without_a_model_id(self): + self.run.target_config_snapshot = {"model_id": "", "display_name": "Unnamed"} + self.run.save(update_fields=["target_config_snapshot"]) + self.run.target_model.model_id = "" + self.run.target_model.save(update_fields=["model_id"]) + + document = json.loads(self._export().content) + + self.assertNotIn("studio_provenance", document) + + def test_failed_scenario_is_exported_as_error(self): + item = ScenarioSetVersionItemFactory(version=self.run.scenario_set_version, scenario=ScenarioFactory(title="Gamma")) + ScenarioResult.objects.create( + run_id=self.run.id, version_item_id=str(item.pk), status="failed", result={"error": "target timed out"} + ) + + document = json.loads(self._export().content) + + gamma = document["results"][-1] + self.assertEqual(gamma["scenario_name"], "Gamma") + self.assertEqual(gamma["severity"], "ERROR") + self.assertEqual(gamma["summary"], "target timed out") + + def test_unfinished_run_is_a_conflict(self): + run = AuditRunFactory(project=self.project, status=AuditRun.Status.AUDITING) + + self.assertEqual(self._export(run).status_code, 409) + + def test_run_with_repetitions_is_not_supported(self): + run = AuditRunFactory( + project=self.project, status=AuditRun.Status.COMPLETED, generation_parameters_snapshot={"n_repetitions": 3} + ) + + self.assertEqual(self._export(run).status_code, 422) + + def test_other_projects_run_is_not_found(self): + other = AuditRunFactory(project=ProjectFactory(), status=AuditRun.Status.COMPLETED) + + self.assertEqual(self._export(other).status_code, 404) + + def test_login_is_required(self): + resp = Client(SERVER_NAME="localhost").get(f"/runs/{self.run.id}/export/?format=simpleaudit") + + self.assertEqual(resp.status_code, 302) + self.assertIn("login", resp["Location"]) + + def test_existing_json_export_is_unchanged(self): + resp = self.client.get(f"/runs/{self.run.id}/export/?format=json") + + self.assertEqual(resp.status_code, 200) + self.assertEqual([row["scenario_name"] for row in json.loads(resp.content)], ["Alpha", "Beta"]) diff --git a/infra/ui.py b/infra/ui.py index ed70768a..4d3336b0 100644 --- a/infra/ui.py +++ b/infra/ui.py @@ -3564,6 +3564,9 @@ def get(self, request, run_id): run = get_object_or_404(AuditRun, pk=run_id, project=request.project) fmt = request.GET.get("format", "json").lower() + if fmt == "simpleaudit": + return self._engine_json(run) + items = {str(vi.pk): vi for vi in ScenarioSetVersionItem.objects.filter(version=run.scenario_set_version).select_related("scenario")} rows = [] for sr in ScenarioResult.objects.filter(run_id=run.id).order_by("id"): @@ -3593,6 +3596,20 @@ def get(self, request, run_id): response["Content-Disposition"] = f'attachment; filename="{filename}"' return response + @staticmethod + def _engine_json(run): + """``?format=simpleaudit``: the run as the engine's own results JSON.""" + from audits.services import RunExportError, export_engine_results + + try: + document = export_engine_results(run) + except RunExportError as exc: + return JsonResponse({"error": exc.reason}, status=422 if exc.unsupported else 409) + payload = json.dumps(document, indent=2, ensure_ascii=False) + response = HttpResponse(payload, content_type="application/json; charset=utf-8") + response["Content-Disposition"] = f'attachment; filename="audit_{run.id}_simpleaudit.json"' + return response + class RunScriptView(ProjectMixin, View): """A standalone Python script that re-runs this run with the plain diff --git a/templates/partials/run_results.html b/templates/partials/run_results.html index 5e5da566..19d4b4a3 100644 --- a/templates/partials/run_results.html +++ b/templates/partials/run_results.html @@ -6,7 +6,12 @@

Results ({{ results|length }})

{% if results %} - Export +
+ Export + {% if run.status == "completed" %} + Export (SimpleAudit JSON) + {% endif %} +
{% endif %}
{% if results %}