From 4273d9a52c6d2fc214f920c1b812d0be48730b07 Mon Sep 17 00:00:00 2001 From: shashank Date: Mon, 5 Oct 2026 22:00:16 +0530 Subject: [PATCH 1/3] FEAT Add explicit Scenario progress result roles Scenario progress now says what each persisted result represents, so consumers no longer interpret class names, technique names, or empty conversations. - AttackResultRole: every AttackStrategy declares RESULT_ROLE (target_facing by default, orchestration for SequentialAttack). It is copied onto the context and written to attribution_data["result_role"] for completed and error results. No migration; rows without it read as unknown, and nothing is inferred from an empty conversation ID. - ScenarioRunPlanGroupKind: AtomicAttack takes a group_kind (attack, baseline from build_baseline_atomic_attack, adaptive from AdaptiveScenario), recorded in the run plan. It is not part of any identity, so runs resume unchanged. Plans saved before this change round-trip without a kind and read as unknown. - SequentialAttack gives each child its 1-based attempt_index, matching the existing _adaptive_attempt label. - The progress query projects attack_metadata, and ScenarioProgressResult exposes result_role, the stored ordered child_attack_result_ids, and attempt_index. ScenarioAtomicGroupProgress exposes kind. Planned-unit completion, retry and error counters, success rates, and preparation-failure semantics are unchanged. Fixes #2992 --- doc/code/scenarios/3_adaptive_scenarios.ipynb | 49 +++ doc/code/scenarios/3_adaptive_scenarios.py | 44 ++ .../services/scenario_progress_read_model.py | 52 ++- .../attack/compound/sequential_attack.py | 12 +- .../attack/core/attack_result_attribution.py | 5 + pyrit/executor/attack/core/attack_strategy.py | 18 +- pyrit/memory/memory_interface.py | 2 + pyrit/models/__init__.py | 5 +- pyrit/models/results/attack_result.py | 21 + pyrit/models/scenario_progress.py | 30 +- pyrit/scenario/core/atomic_attack.py | 12 + .../core/matrix_atomic_attack_builder.py | 3 +- pyrit/scenario/core/scenario.py | 1 + .../scenarios/adaptive/adaptive_scenario.py | 2 + .../test_scenario_progress_read_model.py | 381 ++++++++++++++++++ .../attack/compound/test_sequential_attack.py | 53 ++- .../attack/core/test_attack_strategy.py | 41 ++ tests/unit/scenario/core/test_scenario.py | 8 + .../core/test_scenario_partial_results.py | 2 + .../unit/scenario/core/test_scenario_retry.py | 6 + .../scenarios/adaptive/test_text_adaptive.py | 28 +- 21 files changed, 762 insertions(+), 13 deletions(-) diff --git a/doc/code/scenarios/3_adaptive_scenarios.ipynb b/doc/code/scenarios/3_adaptive_scenarios.ipynb index c06cbd95b5..d3759eef83 100644 --- a/doc/code/scenarios/3_adaptive_scenarios.ipynb +++ b/doc/code/scenarios/3_adaptive_scenarios.ipynb @@ -770,6 +770,55 @@ "cell_type": "markdown", "id": "11", "metadata": {}, + "source": [ + "## Result roles in scenario progress\n", + "\n", + "The strategy that produces each result records what the result represents, and the scenario progress\n", + "API (`GET /api/scenarios/runs/{scenario_result_id}/progress`) returns it. Clients read these fields\n", + "instead of inferring a parent from a class name or an empty conversation ID:\n", + "\n", + "- `result_role` is `target_facing` for an attack that sends its own requests to the objective target,\n", + " `orchestration` for a parent that only runs other attacks (such as the per-objective\n", + " `SequentialAttack`), and `unknown` for rows saved before roles were recorded. A `target_facing` role\n", + " does not prove a request reached the target: an attack that ends in a preparation failure is still\n", + " `target_facing`.\n", + "- `child_attack_result_ids` lists an orchestration parent's children in the order they ran.\n", + "- `attempt_index` is a child's 1-based position under its parent. For Adaptive it matches the\n", + " `_adaptive_attempt` memory label.\n", + "- Each `summary.atomic_groups` entry has a `kind`: `attack`, `baseline`, `adaptive`, or `unknown` for\n", + " plans saved before kinds were recorded.\n", + "\n", + "Roles describe results without changing how progress is counted: a parent and its children still\n", + "belong to one planned unit.\n", + "\n", + "This excerpt of a progress response shows one Adaptive objective whose first attempt failed and whose\n", + "second succeeded (other fields omitted):\n", + "\n", + "```json\n", + "{\n", + " \"results\": [\n", + " {\"attack_result_id\": \"child-1\", \"conversation_id\": \"conversation-1\", \"outcome\": \"failure\",\n", + " \"result_role\": \"target_facing\", \"child_attack_result_ids\": [], \"attempt_index\": 1},\n", + " {\"attack_result_id\": \"child-2\", \"conversation_id\": \"conversation-2\", \"outcome\": \"success\",\n", + " \"result_role\": \"target_facing\", \"child_attack_result_ids\": [], \"attempt_index\": 2},\n", + " {\"attack_result_id\": \"parent\", \"conversation_id\": \"\", \"outcome\": \"success\",\n", + " \"result_role\": \"orchestration\", \"child_attack_result_ids\": [\"child-1\", \"child-2\"],\n", + " \"attempt_index\": null}\n", + " ],\n", + " \"summary\": {\n", + " \"atomic_groups\": [\n", + " {\"atomic_attack_name\": \"baseline\", \"kind\": \"baseline\", \"completed\": 1, \"planned\": 1},\n", + " {\"atomic_attack_name\": \"adaptive_airt_hate::4f0c...\", \"kind\": \"adaptive\", \"completed\": 1, \"planned\": 1}\n", + " ]\n", + " }\n", + "}\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "12", + "metadata": {}, "source": [ "## Running from the scanner CLI\n", "\n", diff --git a/doc/code/scenarios/3_adaptive_scenarios.py b/doc/code/scenarios/3_adaptive_scenarios.py index ba29d7b68d..4b26ac8aad 100644 --- a/doc/code/scenarios/3_adaptive_scenarios.py +++ b/doc/code/scenarios/3_adaptive_scenarios.py @@ -232,6 +232,50 @@ def _technique_label(result) -> str: for technique, n in total_picks.most_common(): print(f"{technique:40s} {total_wins[technique]:>4} / {n:<4} {total_wins[technique] / n:.0%}") +# %% [markdown] +# ## Result roles in scenario progress +# +# The strategy that produces each result records what the result represents, and the scenario progress +# API (`GET /api/scenarios/runs/{scenario_result_id}/progress`) returns it. Clients read these fields +# instead of inferring a parent from a class name or an empty conversation ID: +# +# - `result_role` is `target_facing` for an attack that sends its own requests to the objective target, +# `orchestration` for a parent that only runs other attacks (such as the per-objective +# `SequentialAttack`), and `unknown` for rows saved before roles were recorded. A `target_facing` role +# does not prove a request reached the target: an attack that ends in a preparation failure is still +# `target_facing`. +# - `child_attack_result_ids` lists an orchestration parent's children in the order they ran. +# - `attempt_index` is a child's 1-based position under its parent. For Adaptive it matches the +# `_adaptive_attempt` memory label. +# - Each `summary.atomic_groups` entry has a `kind`: `attack`, `baseline`, `adaptive`, or `unknown` for +# plans saved before kinds were recorded. +# +# Roles describe results without changing how progress is counted: a parent and its children still +# belong to one planned unit. +# +# This excerpt of a progress response shows one Adaptive objective whose first attempt failed and whose +# second succeeded (other fields omitted): +# +# ```json +# { +# "results": [ +# {"attack_result_id": "child-1", "conversation_id": "conversation-1", "outcome": "failure", +# "result_role": "target_facing", "child_attack_result_ids": [], "attempt_index": 1}, +# {"attack_result_id": "child-2", "conversation_id": "conversation-2", "outcome": "success", +# "result_role": "target_facing", "child_attack_result_ids": [], "attempt_index": 2}, +# {"attack_result_id": "parent", "conversation_id": "", "outcome": "success", +# "result_role": "orchestration", "child_attack_result_ids": ["child-1", "child-2"], +# "attempt_index": null} +# ], +# "summary": { +# "atomic_groups": [ +# {"atomic_attack_name": "baseline", "kind": "baseline", "completed": 1, "planned": 1}, +# {"atomic_attack_name": "adaptive_airt_hate::4f0c...", "kind": "adaptive", "completed": 1, "planned": 1} +# ] +# } +# } +# ``` + # %% [markdown] # ## Running from the scanner CLI # diff --git a/pyrit/backend/services/scenario_progress_read_model.py b/pyrit/backend/services/scenario_progress_read_model.py index 13278af3a6..7e8704ad23 100644 --- a/pyrit/backend/services/scenario_progress_read_model.py +++ b/pyrit/backend/services/scenario_progress_read_model.py @@ -10,7 +10,7 @@ from dataclasses import dataclass, field from datetime import UTC, datetime from threading import Lock -from typing import Literal +from typing import Any, Literal from pyrit.common.async_compatibility import legacy_sync_override from pyrit.common.deprecation import print_deprecation_message @@ -21,6 +21,7 @@ AtomicAttackIdentifier, AttackOutcome, AttackResult, + AttackResultRole, AttackTechniqueIdentifier, ComponentIdentifier, ScenarioAtomicGroupProgress, @@ -36,6 +37,7 @@ ScenarioResult, ScenarioRunPlan, ScenarioRunPlanAtomicGroup, + ScenarioRunPlanGroupKind, ScenarioRunPlanSeedGroup, ScenarioScorerIdentity, ScenarioSeedGroupProgress, @@ -622,6 +624,7 @@ def aggregate(*, units: Sequence[ResultUnitIdentity], planned: int | None) -> Sc display_group=group.display_group, status=group_status, technique_details=technique_details_by_group.get(group.id), + kind=group.kind or ScenarioRunPlanGroupKind.UNKNOWN, **counts.model_dump(), ) ) @@ -912,8 +915,55 @@ def _map_progress_delta( error_type=delta.error_type, error_message=delta.error_message, score=delta.score, + result_role=ScenarioProgressReadModel._read_result_role(attribution_data=delta.attribution_data), + child_attack_result_ids=ScenarioProgressReadModel._read_child_attack_result_ids( + attack_metadata=delta.attack_metadata + ), + attempt_index=ScenarioProgressReadModel._read_attempt_index(attribution_data=delta.attribution_data), ) + @staticmethod + def _read_result_role(*, attribution_data: dict[str, Any]) -> AttackResultRole: + """ + Read the role recorded by the producing strategy. + + A row without a recognized role is ``UNKNOWN``. Nothing is inferred from other + fields, such as an empty conversation ID. + + Returns: + AttackResultRole: The recorded role, or ``UNKNOWN``. + """ + try: + return AttackResultRole(attribution_data.get("result_role")) + except ValueError: + return AttackResultRole.UNKNOWN + + @staticmethod + def _read_child_attack_result_ids(*, attack_metadata: dict[str, Any]) -> list[str]: + """ + Read the ordered child result IDs that ``SequentialAttack`` stores in its metadata. + + Returns: + list[str]: The child IDs in stored order, or an empty list when none are recorded. + """ + child_ids = attack_metadata.get("child_attack_result_ids") + if isinstance(child_ids, list) and all(isinstance(child_id, str) for child_id in child_ids): + return list(child_ids) + return [] + + @staticmethod + def _read_attempt_index(*, attribution_data: dict[str, Any]) -> int | None: + """ + Read a child result's 1-based position under its orchestration parent. + + Returns: + int | None: The recorded position, or None when absent or invalid. + """ + attempt_index = attribution_data.get("attempt_index") + if isinstance(attempt_index, int) and not isinstance(attempt_index, bool) and attempt_index >= 1: + return attempt_index + return None + @staticmethod def _synthesize_legacy_plan(*, deltas: list[ScenarioAttackResultDelta]) -> ScenarioRunPlan: """ diff --git a/pyrit/executor/attack/compound/sequential_attack.py b/pyrit/executor/attack/compound/sequential_attack.py index 35416f7b50..e364b2db35 100644 --- a/pyrit/executor/attack/compound/sequential_attack.py +++ b/pyrit/executor/attack/compound/sequential_attack.py @@ -23,7 +23,7 @@ import logging import uuid -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from datetime import UTC, datetime from enum import Enum from typing import TYPE_CHECKING, Any, ClassVar @@ -33,7 +33,7 @@ from pyrit.executor.attack.core.attack_executor import AttackExecutor from pyrit.executor.attack.core.attack_parameters import AttackParameters from pyrit.executor.attack.core.attack_strategy import AttackContext, AttackStrategy -from pyrit.models import AttackOutcome, AttackResult, AttackSeedGroup, ScoringExpectation +from pyrit.models import AttackOutcome, AttackResult, AttackResultRole, AttackSeedGroup, ScoringExpectation if TYPE_CHECKING: from collections.abc import Mapping, Sequence @@ -191,6 +191,8 @@ class SequentialAttack(AttackStrategy[AttackContext[AttackParameters], Sequentia DELEGATES_SCORING: ClassVar[bool] = True + RESULT_ROLE: ClassVar[AttackResultRole] = AttackResultRole.ORCHESTRATION + CHILD_ATTACK_RESULT_IDS_KEY: str = "child_attack_result_ids" """Metadata key under which the per-child-attack result IDs are stored.""" @@ -249,12 +251,14 @@ async def _teardown_async(self, *, context: AttackContext[AttackParameters]) -> async def _perform_async(self, *, context: AttackContext[AttackParameters]) -> SequentialAttackResult: results: list[AttackResult] = [] - for child_attack in self._child_attacks: + for attempt_index, child_attack in enumerate(self._child_attacks, start=1): labels = {**context.memory_labels, **dict(child_attack.memory_labels)} + # Each child shares the parent's attribution plus its own position. + attribution = replace(context._attribution, attempt_index=attempt_index) if context._attribution else None result = await self._run_child_attack_async( child_attack=child_attack, memory_labels=labels, - attribution=context._attribution, + attribution=attribution, expectation=context.params.expectation, ) results.append(result) diff --git a/pyrit/executor/attack/core/attack_result_attribution.py b/pyrit/executor/attack/core/attack_result_attribution.py index 93bb0efb3a..547d583e46 100644 --- a/pyrit/executor/attack/core/attack_result_attribution.py +++ b/pyrit/executor/attack/core/attack_result_attribution.py @@ -46,9 +46,14 @@ class AttackResultAttribution: ``AtomicAttackEvaluationIdentifier``). seed_group_id (str | None): Optional logical seed-group fingerprint for per-task progress attribution. + attempt_index (int | None): Optional 1-based position of this result among the + children of an orchestration parent. Persisted into + ``AttackResultEntry.attribution_data``. ``SequentialAttack`` sets it for each + child attack it dispatches, e.g. ``2`` for the second child. """ parent_id: str parent_collection: str parent_eval_hash: str | None = None seed_group_id: str | None = None + attempt_index: int | None = None diff --git a/pyrit/executor/attack/core/attack_strategy.py b/pyrit/executor/attack/core/attack_strategy.py index a47ed5f672..92708a0740 100644 --- a/pyrit/executor/attack/core/attack_strategy.py +++ b/pyrit/executor/attack/core/attack_strategy.py @@ -33,6 +33,7 @@ AttackIdentifier, AttackOutcome, AttackResult, + AttackResultRole, ComponentIdentifier, ConversationReference, ConverterIdentifier, @@ -206,6 +207,12 @@ class AttackContext(StrategyContext, ABC, Generic[AttackParamsT]): # ID of the AttackResult this execution produces. Allocated when execution starts. _attack_result_id: str | None = field(default=None, init=False, repr=False, compare=False) + # Role of the AttackResult this execution produces. Copied from the strategy's + # ``RESULT_ROLE`` when execution starts, so success and error results both carry it. + _result_role: AttackResultRole = field( + default=AttackResultRole.TARGET_FACING, init=False, repr=False, compare=False + ) + _expectation: ScoringExpectation = field(init=False, repr=False) def __post_init__(self) -> None: @@ -442,7 +449,8 @@ def _apply_attribution( the AttackExecutor when an upstream orchestrator supplied a factory). When present, writes ``attribution_parent_id`` and a fixed-schema ``attribution_data`` dict onto the result so they round-trip into - ``AttackResultEntry``. + ``AttackResultEntry``. The dict also records ``result_role``, the + producing strategy's ``RESULT_ROLE``. Args: context: The per-task AttackContext. @@ -454,11 +462,14 @@ def _apply_attribution( result.attribution_parent_id = attribution.parent_id attribution_data: dict[str, Any] = { "parent_collection": attribution.parent_collection, + "result_role": context._result_role.value, } if attribution.parent_eval_hash is not None: attribution_data["parent_eval_hash"] = attribution.parent_eval_hash if attribution.seed_group_id is not None: attribution_data["seed_group_id"] = attribution.seed_group_id + if attribution.attempt_index is not None: + attribution_data["attempt_index"] = attribution.attempt_index result.attribution_data = attribution_data @staticmethod @@ -588,6 +599,10 @@ class AttackStrategy(Strategy[AttackStrategyContextT, AttackStrategyResultT], Id #: No scoring configuration alone does not imply delegation. DELEGATES_SCORING: ClassVar[bool] = False + #: What this strategy's persisted results represent. Compound attacks that only coordinate + #: child attacks, and never call the objective target themselves, set ``ORCHESTRATION``. + RESULT_ROLE: ClassVar[AttackResultRole] = AttackResultRole.TARGET_FACING + def __init_subclass__(cls, **kwargs: Any) -> None: """ Enforce the keyword-only constructor contract on subclasses. @@ -858,6 +873,7 @@ async def execute_with_context_async(self, *, context: AttackStrategyContextT) - self._validate_scoring_expectation(context=context) context._error_result_persistence_error = None context._attack_result_id = str(uuid.uuid4()) + context._result_role = self.RESULT_ROLE lifecycle = _ObjectiveTargetConversationLifecycle( objective_target=self._objective_target, logger=self._logger, diff --git a/pyrit/memory/memory_interface.py b/pyrit/memory/memory_interface.py index 987e18f74c..3a40a95d58 100644 --- a/pyrit/memory/memory_interface.py +++ b/pyrit/memory/memory_interface.py @@ -5955,6 +5955,7 @@ def _execute_get_scenario_attack_result_deltas( AttackResultEntry.error_type, AttackResultEntry.error_message, AttackResultEntry.attribution_data, + AttackResultEntry.attack_metadata, ScoreEntry.id.label("score_id"), ScoreEntry.score_value, ScoreEntry.score_type, @@ -6014,6 +6015,7 @@ def _execute_get_scenario_attack_result_deltas( error_type=row.error_type, error_message=row.error_message, attribution_data=row.attribution_data or {}, + attack_metadata=row.attack_metadata or {}, score=score, ) ) diff --git a/pyrit/models/__init__.py b/pyrit/models/__init__.py index a1754fc22e..724ee0b3ec 100644 --- a/pyrit/models/__init__.py +++ b/pyrit/models/__init__.py @@ -129,7 +129,7 @@ display_choices, ) from pyrit.models.question_answering import QuestionAnsweringDataset, QuestionAnsweringEntry, QuestionChoice - from pyrit.models.results.attack_result import AttackOutcome, AttackResult, AttackResultT + from pyrit.models.results.attack_result import AttackOutcome, AttackResult, AttackResultRole, AttackResultT from pyrit.models.results.scenario_result import ScenarioResult, ScenarioRunState from pyrit.models.results.strategy_result import StrategyResult, StrategyResultT from pyrit.models.retry_event import RetryEvent @@ -163,6 +163,7 @@ ScenarioQueueSnapshot, ScenarioRunPlan, ScenarioRunPlanAtomicGroup, + ScenarioRunPlanGroupKind, ScenarioRunPlanSeedGroup, ScenarioRunPlanSeedPrompt, ScenarioRunProgress, @@ -282,6 +283,7 @@ "AttackIdentifier": "pyrit.models.identifiers", "AttackTechniqueIdentifier": "pyrit.models.identifiers", "AttackResult": "pyrit.models.results.attack_result", + "AttackResultRole": "pyrit.models.results.attack_result", "AttackResultT": "pyrit.models.results.attack_result", "AttackOutcome": "pyrit.models.results.attack_result", "ChatMessage": "pyrit.models.messages.chat_message", @@ -406,6 +408,7 @@ "ScenarioQueueSnapshot": "pyrit.models.scenario_progress", "ScenarioRunPlan": "pyrit.models.scenario_progress", "ScenarioRunPlanAtomicGroup": "pyrit.models.scenario_progress", + "ScenarioRunPlanGroupKind": "pyrit.models.scenario_progress", "ScenarioRunPlanSeedPrompt": "pyrit.models.scenario_progress", "ScenarioRunPlanSeedGroup": "pyrit.models.scenario_progress", "ScenarioRunProgress": "pyrit.models.scenario_progress", diff --git a/pyrit/models/results/attack_result.py b/pyrit/models/results/attack_result.py index c6d6d627d1..c50b02aec0 100644 --- a/pyrit/models/results/attack_result.py +++ b/pyrit/models/results/attack_result.py @@ -95,6 +95,27 @@ class AttackOutcome(str, Enum): UNDETERMINED = "undetermined" +class AttackResultRole(str, Enum): + """ + What a persisted attack result represents, recorded by the strategy that produced it. + + The role says which kind of record this is, not what happened. A target-facing result + can still end before any request reaches the objective target (for example, a preparation + failure), so the role is not proof that the target was called. + """ + + #: Produced by a strategy that sends its own requests to the objective target. + TARGET_FACING = "target_facing" + + #: Produced by a strategy that only coordinates other attacks and has no target + #: conversation of its own, such as ``SequentialAttack``. + ORCHESTRATION = "orchestration" + + #: Read-side value for records that carry no recognized role, such as rows persisted + #: before roles were recorded. Producers never write it. + UNKNOWN = "unknown" + + class AttackResult(StrategyResult): """Base class for all attack results.""" diff --git a/pyrit/models/scenario_progress.py b/pyrit/models/scenario_progress.py index 8e9ef9bd95..38992cf450 100644 --- a/pyrit/models/scenario_progress.py +++ b/pyrit/models/scenario_progress.py @@ -4,13 +4,14 @@ """Canonical models for durable scenario run plans and incremental progress.""" from datetime import datetime +from enum import Enum from typing import Any, Literal from pydantic import AwareDatetime, BaseModel, Field, model_validator from pyrit.models.catalog.scenario import ScenarioOverloadSummary, ScenarioTargetSummary # noqa: TC001 from pyrit.models.identifiers.atomic_attack_identifier import AtomicAttackIdentifier -from pyrit.models.results.attack_result import AttackOutcome +from pyrit.models.results.attack_result import AttackOutcome, AttackResultRole from pyrit.models.results.scenario_result import ScenarioRunState from pyrit.models.retry_event import RetryEvent from pyrit.models.score.score import ScoreStatus @@ -20,6 +21,23 @@ SCENARIO_RUN_PLAN_VERSION = 1 +class ScenarioRunPlanGroupKind(str, Enum): + """What a planned atomic group runs, recorded by the code that builds the group.""" + + #: An ordinary technique attack. + ATTACK = "attack" + + #: The unmodified comparison built by ``build_baseline_atomic_attack``. + BASELINE = "baseline" + + #: One Adaptive objective, run as an orchestration parent and its attempts. + ADAPTIVE = "adaptive" + + #: Read-side value for groups whose plan does not record a kind, such as plans persisted + #: before kinds were recorded. Plans never store it. + UNKNOWN = "unknown" + + class ScenarioRunPlanSeedGroup(BaseModel): """A de-duplicated logical seed group in a scenario run plan.""" @@ -50,6 +68,8 @@ class ScenarioRunPlanAtomicGroup(BaseModel): seed_group_ids: list[str] description: str | None = None tags: list[str] = Field(default_factory=list) + #: None for plans persisted before kinds were recorded, so those plans round-trip unchanged. + kind: ScenarioRunPlanGroupKind | None = None class ScenarioRunPlan(BaseModel): @@ -154,6 +174,12 @@ class ScenarioProgressResult(BaseModel): error_type: str | None = None error_message: str | None = None score: ScenarioProgressScore | None = None + #: Recorded by the producing strategy. ``unknown`` when the row predates roles. + result_role: AttackResultRole = AttackResultRole.UNKNOWN + #: Ordered results this orchestration parent ran, as persisted by ``SequentialAttack``. + child_attack_result_ids: list[str] = Field(default_factory=list) + #: 1-based position of this result among its orchestration parent's children. + attempt_index: int | None = Field(default=None, ge=1) class ScenarioProgressCounts(BaseModel): @@ -204,6 +230,7 @@ class ScenarioAtomicGroupProgress(ScenarioProgressCounts): display_group: str status: Literal["RUNNING", "PENDING", "INCOMPLETE", "COMPLETED"] technique_details: ScenarioAttackTechniqueDetails | None = None + kind: ScenarioRunPlanGroupKind = ScenarioRunPlanGroupKind.UNKNOWN class ScenarioObjectiveScorerMetrics(BaseModel): @@ -291,6 +318,7 @@ class ScenarioAttackResultDelta(BaseModel): error_type: str | None = None error_message: str | None = None attribution_data: dict[str, Any] = Field(default_factory=dict) + attack_metadata: dict[str, Any] = Field(default_factory=dict) score: ScenarioProgressScore | None = None diff --git a/pyrit/scenario/core/atomic_attack.py b/pyrit/scenario/core/atomic_attack.py index 55fa692e6a..f08f77c497 100644 --- a/pyrit/scenario/core/atomic_attack.py +++ b/pyrit/scenario/core/atomic_attack.py @@ -27,6 +27,7 @@ AtomicAttackIdentifier, AttackResult, AttackSeedGroup, + ScenarioRunPlanGroupKind, config_hash, ) @@ -65,6 +66,7 @@ def __init__( adversarial_chat: PromptTarget | None = None, objective_scorer: TrueFalseScorer | None = None, memory_labels: dict[str, str] | None = None, + group_kind: ScenarioRunPlanGroupKind = ScenarioRunPlanGroupKind.ATTACK, **attack_execute_params: Any, ) -> None: """ @@ -88,6 +90,10 @@ def __init__( objective_scorer: Optional scorer for evaluating simulated conversations. memory_labels: Additional labels to apply to prompts. + group_kind: What this group runs, recorded in the scenario's run plan. + ``build_baseline_atomic_attack`` passes ``BASELINE`` and Adaptive + scenarios pass ``ADAPTIVE``. It describes the group and does not + change its identity or what it executes. **attack_execute_params: Additional parameters to pass to the attack execution method. @@ -98,6 +104,7 @@ def __init__( self.atomic_attack_name = atomic_attack_name self.display_group = display_group or atomic_attack_name self._technique_name = technique_name + self._group_kind = group_kind self._attack_technique = attack_technique @@ -187,6 +194,11 @@ def technique_name(self) -> str | None: """Catalog name of the technique that built this attack.""" return self._technique_name + @property + def group_kind(self) -> ScenarioRunPlanGroupKind: + """What this group runs, as recorded in the scenario's run plan.""" + return self._group_kind + @property def technique_eval_hash(self) -> str: """ diff --git a/pyrit/scenario/core/matrix_atomic_attack_builder.py b/pyrit/scenario/core/matrix_atomic_attack_builder.py index 163fe7c299..bcb3a4cd4d 100644 --- a/pyrit/scenario/core/matrix_atomic_attack_builder.py +++ b/pyrit/scenario/core/matrix_atomic_attack_builder.py @@ -24,7 +24,7 @@ from pyrit.executor.attack import AttackScoringConfig from pyrit.executor.attack.single_turn.prompt_sending import PromptSendingAttack -from pyrit.models import AttackSeedGroup +from pyrit.models import AttackSeedGroup, ScenarioRunPlanGroupKind from pyrit.prompt_normalizer import ConverterConfiguration from pyrit.scenario.core._technique_resolution import ( TechniqueResolutionError, @@ -144,6 +144,7 @@ def build_baseline_atomic_attack( objective_scorer=cast("TrueFalseScorer", objective_scorer), memory_labels=memory_labels or {}, display_group=display_group, + group_kind=ScenarioRunPlanGroupKind.BASELINE, ) diff --git a/pyrit/scenario/core/scenario.py b/pyrit/scenario/core/scenario.py index 6e8179369b..acfa66e00d 100644 --- a/pyrit/scenario/core/scenario.py +++ b/pyrit/scenario/core/scenario.py @@ -1043,6 +1043,7 @@ def _build_run_plan(self) -> ScenarioRunPlan: seed_group_ids=seed_group_ids, description=technique.description if technique else None, tags=sorted(technique.tags) if technique else [], + kind=atomic_attack.group_kind, ) ) return ScenarioRunPlan( diff --git a/pyrit/scenario/scenarios/adaptive/adaptive_scenario.py b/pyrit/scenario/scenarios/adaptive/adaptive_scenario.py index 0ada73f669..08a57158ca 100644 --- a/pyrit/scenario/scenarios/adaptive/adaptive_scenario.py +++ b/pyrit/scenario/scenarios/adaptive/adaptive_scenario.py @@ -25,6 +25,7 @@ from pyrit.executor.attack import AttackScoringConfig from pyrit.models import ( BoundedDatasetSize, + ScenarioRunPlanGroupKind, ScenarioRunSizeComponent, ScenarioRunSizeEstimate, ) @@ -454,6 +455,7 @@ async def _build_atomics_for_dataset_async( objective_scorer=self._objective_scorer, memory_labels=dict(self._memory_labels), display_group=dataset_name, + group_kind=ScenarioRunPlanGroupKind.ADAPTIVE, ) ) diff --git a/tests/unit/backend/test_scenario_progress_read_model.py b/tests/unit/backend/test_scenario_progress_read_model.py index 425751d61d..8f5b44a394 100644 --- a/tests/unit/backend/test_scenario_progress_read_model.py +++ b/tests/unit/backend/test_scenario_progress_read_model.py @@ -5,24 +5,52 @@ import asyncio import uuid +from collections.abc import Callable from datetime import UTC, datetime, timedelta +from typing import ClassVar from unittest.mock import AsyncMock, MagicMock, patch import pytest from pyrit.backend.services.scenario_progress_read_model import ( + ScenarioPlanLookup, ScenarioProgressReadModel, ScenarioProgressSnapshot, ) +from pyrit.exceptions import ScenarioPartialFailureException +from pyrit.executor.attack import ( + AttackScoringConfig, + PromptSendingAttack, + SequentialAttack, + SequentialChildAttack, +) +from pyrit.executor.attack.core.attack_preparation import AttackPreparationFailure, AttackPreparationFailureKind from pyrit.memory import AttackResultKeysetCursor from pyrit.memory.memory_interface import MemoryInterface from pyrit.models import ( + SCENARIO_RUN_PLAN_METADATA_KEY, AttackOutcome, + AttackResult, + AttackResultRole, + AttackSeedGroup, + ComponentIdentifier, + Message, ScenarioAttackResultDelta, + ScenarioProgressResult, ScenarioRunPlan, ScenarioRunPlanAtomicGroup, + ScenarioRunPlanGroupKind, ScenarioRunPlanSeedGroup, + SeedObjective, ) +from pyrit.prompt_target import PromptTarget +from pyrit.scenario import DatasetConfiguration +from pyrit.scenario.core import AtomicAttack, BaselineAttackPolicy, Scenario, ScenarioTechnique +from pyrit.scenario.core.attack_technique import AttackTechnique +from pyrit.scenario.core.matrix_atomic_attack_builder import build_baseline_atomic_attack +from pyrit.scenario.scenarios.adaptive.dispatcher import ADAPTIVE_ATTEMPT_LABEL +from pyrit.score import Scorer, SubStringScorer +from unit.mocks import MockPromptTarget, get_mock_target_identifier, make_scenario_result def _make_delta(*, run_id: str, index: int = 0) -> ScenarioAttackResultDelta: @@ -322,3 +350,356 @@ async def read_run_async(run_id: str) -> ScenarioProgressSnapshot: assert [snapshot.results[0].conversation_id for snapshot in snapshots] == [ f"conversation-{run_id}-0" for run_id in run_ids ] + + +class _UnavailableTarget(MockPromptTarget): + """A target whose every request fails, so the attacks that call it end in errors.""" + + async def _send_prompt_to_target_async(self, *, normalized_conversation: list[Message]) -> list[Message]: + raise RuntimeError("objective target unavailable") + + +class _RoleScenario(Scenario): + """Minimal scenario that runs the atomic attacks a test builds, through the real plan and run path.""" + + BASELINE_ATTACK_POLICY: ClassVar[BaselineAttackPolicy] = BaselineAttackPolicy.Forbidden + + def __init__(self, *, build_atomic_attacks: Callable[[PromptTarget], list[AtomicAttack]]) -> None: + class _Technique(ScenarioTechnique): + TEST = ("test", {"concrete"}) + ALL = ("all", {"all"}) + + @classmethod + def get_aggregate_tags(cls) -> set[str]: + return {"all"} + + scorer = MagicMock(spec=Scorer) + scorer.get_identifier.return_value = ComponentIdentifier(class_name="RoleTestScorer", class_module="tests") + scorer.get_scorer_metrics.return_value = None + super().__init__( + name="RoleScenario", + version=1, + technique_class=_Technique, + default_dataset_config=DatasetConfiguration(), + objective_scorer=scorer, + ) + self._build = build_atomic_attacks + + async def _resolve_seed_groups_by_dataset_async(self, *, apply_sampling: bool = True): + return {} + + async def _build_atomic_attacks_async(self, *, context): + return self._build(context.objective_target) + + +_SEED_GROUP = AttackSeedGroup(seeds=[SeedObjective(value="describe the objective")]) + + +def _prompt_sending(*, target: PromptTarget) -> PromptSendingAttack: + # The mock target always answers "default", so this scorer always reports FAILURE and + # FIRST_SUCCESS keeps dispatching children. + scorer = SubStringScorer(substring="never-in-the-response") + return PromptSendingAttack( + objective_target=target, attack_scoring_config=AttackScoringConfig(objective_scorer=scorer) + ) + + +def _adaptive_style_group(*, target: PromptTarget, name: str = "adaptive_objective") -> AtomicAttack: + """One orchestration parent with two target-facing children, wired the way Adaptive wires them.""" + parent = SequentialAttack( + objective_target=target, + child_attacks=[ + SequentialChildAttack( + strategy=_prompt_sending(target=target), + seed_group=_SEED_GROUP, + memory_labels={ADAPTIVE_ATTEMPT_LABEL: str(attempt)}, + ) + for attempt in (1, 2) + ], + ) + return AtomicAttack( + atomic_attack_name=name, + attack_technique=AttackTechnique(attack=parent), + seed_groups=[_SEED_GROUP], + group_kind=ScenarioRunPlanGroupKind.ADAPTIVE, + ) + + +async def _run_and_read_progress_async( + *, + memory: MemoryInterface, + build_atomic_attacks: Callable[[PromptTarget], list[AtomicAttack]], + target: PromptTarget, + expect_partial_failure: bool = False, +) -> tuple[ScenarioProgressSnapshot, ScenarioRunPlan, str]: + """Initialize and run a real scenario, then read its persisted plan and rows through the read model.""" + scenario = _RoleScenario(build_atomic_attacks=build_atomic_attacks) + scenario.set_params_from_args(args={"objective_target": target}) + await scenario.initialize_async() + if expect_partial_failure: + with pytest.raises(ScenarioPartialFailureException): + await scenario.run_async() + else: + await scenario.run_async() + run_id = str(scenario._scenario_result_id) + [stored] = await memory.get_scenario_results_async(scenario_result_ids=[run_id]) + plan = ScenarioRunPlan.model_validate(stored.metadata[SCENARIO_RUN_PLAN_METADATA_KEY]) + snapshot = await ScenarioProgressReadModel(memory=memory).get_snapshot_async( + scenario_result_id=run_id, + plan=plan, + plan_complete=True, + active_group_ids=(), + terminal=True, + objective_scorer_identifier=None, + ) + return snapshot, plan, run_id + + +@pytest.mark.usefixtures("patch_central_database") +class TestScenarioResultRoles: + async def test_real_run_projects_roles_children_attempts_and_group_kinds_async( + self, sqlite_instance: MemoryInterface + ) -> None: + def build(target: PromptTarget) -> list[AtomicAttack]: + return [ + build_baseline_atomic_attack( + objective_target=target, + objective_scorer=SubStringScorer(substring="never-in-the-response"), + seed_groups=[_SEED_GROUP], + ), + AtomicAttack( + atomic_attack_name="plain_attack", + attack_technique=AttackTechnique(attack=_prompt_sending(target=target)), + seed_groups=[_SEED_GROUP], + ), + _adaptive_style_group(target=target), + ] + + snapshot, plan, _ = await _run_and_read_progress_async( + memory=sqlite_instance, build_atomic_attacks=build, target=MockPromptTarget() + ) + + assert {group.atomic_attack_name: group.kind for group in plan.atomic_groups} == { + "baseline": ScenarioRunPlanGroupKind.BASELINE, + "plain_attack": ScenarioRunPlanGroupKind.ATTACK, + "adaptive_objective": ScenarioRunPlanGroupKind.ADAPTIVE, + } + assert {group.atomic_attack_name: group.kind for group in snapshot.summary.atomic_groups} == { + "baseline": ScenarioRunPlanGroupKind.BASELINE, + "plain_attack": ScenarioRunPlanGroupKind.ATTACK, + "adaptive_objective": ScenarioRunPlanGroupKind.ADAPTIVE, + } + + by_name: dict[str, list[ScenarioProgressResult]] = {} + for result in snapshot.results: + by_name.setdefault(result.atomic_attack_name, []).append(result) + for name in ("baseline", "plain_attack"): + [result] = by_name[name] + assert result.result_role is AttackResultRole.TARGET_FACING + assert result.child_attack_result_ids == [] + assert result.attempt_index is None + + [parent] = [r for r in by_name["adaptive_objective"] if r.result_role is AttackResultRole.ORCHESTRATION] + children = sorted( + (r for r in by_name["adaptive_objective"] if r.result_role is AttackResultRole.TARGET_FACING), + key=lambda r: r.attempt_index or 0, + ) + assert parent.conversation_id == "" + assert parent.attempt_index is None + assert [child.attempt_index for child in children] == [1, 2] + # The stored order survives the database round trip and matches the dispatch order. + assert parent.child_attack_result_ids == [child.attack_result_id for child in children] + assert all(child.child_attack_result_ids == [] for child in children) + # Parent and children share one planned unit, as before this change. + assert {r.atomic_group_id for r in (parent, *children)} == {parent.atomic_group_id} + assert {r.seed_group_id for r in (parent, *children)} == {parent.seed_group_id} + + # Adaptive's attempt label is kept on each stored child result and agrees with its index. + stored_children = await sqlite_instance.get_attack_results_async( + attack_result_ids=[child.attack_result_id for child in children] + ) + assert {stored.attack_result_id: stored.labels[ADAPTIVE_ATTEMPT_LABEL] for stored in stored_children} == { + child.attack_result_id: str(child.attempt_index) for child in children + } + + # The REST wire format carries the contract as plain strings. + wire_parent = parent.model_dump(mode="json") + assert wire_parent["result_role"] == "orchestration" + assert wire_parent["child_attack_result_ids"] == [child.attack_result_id for child in children] + assert children[1].model_dump(mode="json")["attempt_index"] == 2 + assert {g["kind"] for g in snapshot.summary.model_dump(mode="json")["atomic_groups"]} == { + "baseline", + "attack", + "adaptive", + } + + async def test_failed_orchestration_parent_keeps_its_role_despite_having_a_conversation_id_async( + self, sqlite_instance: MemoryInterface + ) -> None: + snapshot, _, _ = await _run_and_read_progress_async( + memory=sqlite_instance, + build_atomic_attacks=lambda target: [_adaptive_style_group(target=target)], + target=_UnavailableTarget(), + expect_partial_failure=True, + ) + + by_role = {result.result_role: result for result in snapshot.results} + assert set(by_role) == {AttackResultRole.ORCHESTRATION, AttackResultRole.TARGET_FACING} + parent = by_role[AttackResultRole.ORCHESTRATION] + child = by_role[AttackResultRole.TARGET_FACING] + assert parent.outcome == child.outcome == AttackOutcome.ERROR + # An error result gets a generated conversation ID, so an empty ID would misclassify it. + assert parent.conversation_id != "" + assert child.attempt_index == 1 + + async def test_roles_do_not_change_progress_counts_async(self, sqlite_instance: MemoryInterface) -> None: + snapshot, plan, run_id = await _run_and_read_progress_async( + memory=sqlite_instance, + build_atomic_attacks=lambda target: [ + _adaptive_style_group(target=target, name="adaptive_a"), + _adaptive_style_group(target=target, name="adaptive_b"), + ], + target=MockPromptTarget(), + ) + legacy_deltas = [ + delta.model_copy( + update={ + "attribution_data": { + key: value + for key, value in delta.attribution_data.items() + if key not in ("result_role", "attempt_index") + }, + "attack_metadata": {}, + } + ) + for delta in snapshot.deltas + ] + legacy_memory = MagicMock(spec=MemoryInterface) + legacy_memory.get_scenario_attack_result_deltas_async = AsyncMock(return_value=(legacy_deltas, False)) + + legacy = await ScenarioProgressReadModel(memory=legacy_memory).get_snapshot_async( + scenario_result_id=run_id, + plan=plan, + plan_complete=True, + active_group_ids=(), + terminal=True, + objective_scorer_identifier=None, + ) + + assert {result.result_role for result in legacy.results} == {AttackResultRole.UNKNOWN} + assert legacy.summary == snapshot.summary + assert snapshot.summary.overall.planned == 2 + + +@pytest.mark.parametrize( + ("attribution_data", "attack_metadata", "expected_role", "expected_children", "expected_attempt"), + [ + # A legacy envelope: no recorded role and no conversation. Nothing is inferred. + ({"parent_collection": "attack"}, {"child_attack_result_ids": ["c1", "c2"]}, "unknown", ["c1", "c2"], None), + ({"parent_collection": "attack", "result_role": "a_future_role"}, {}, "unknown", [], None), + ({"parent_collection": "attack", "result_role": ["not", "a", "string"]}, {}, "unknown", [], None), + ( + {"parent_collection": "attack", "result_role": "target_facing", "attempt_index": 0}, + {}, + "target_facing", + [], + None, + ), + ({"parent_collection": "attack", "attempt_index": True}, {}, "unknown", [], None), + ({"parent_collection": "attack", "attempt_index": "2"}, {"child_attack_result_ids": "c1"}, "unknown", [], None), + ({"parent_collection": "attack"}, {"child_attack_result_ids": ["c1", 7]}, "unknown", [], None), + ], +) +def test_map_progress_delta_reads_legacy_and_malformed_rows_conservatively( + attribution_data: dict[str, object], + attack_metadata: dict[str, object], + expected_role: str, + expected_children: list[str], + expected_attempt: int | None, +) -> None: + delta = ScenarioAttackResultDelta( + attack_result_id="row", + conversation_id="", + objective="objective", + outcome=AttackOutcome.FAILURE, + execution_time_ms=1, + timestamp=datetime(2025, 1, 1, tzinfo=UTC), + attribution_data=attribution_data, + attack_metadata=attack_metadata, + ) + + mapped = ScenarioProgressReadModel._map_progress_delta( + delta=delta, plan_lookup=ScenarioPlanLookup.from_plan(plan=None) + ) + + assert mapped.result_role.value == expected_role + assert mapped.child_attack_result_ids == expected_children + assert mapped.attempt_index == expected_attempt + + +async def test_legacy_plan_groups_read_as_unknown_kind_async() -> None: + memory = MagicMock(spec=MemoryInterface) + memory.get_scenario_attack_result_deltas_async = AsyncMock(return_value=([_make_delta(run_id="legacy")], False)) + stored_plan = { + "version": 1, + "atomic_groups": [ + { + "id": "group", + "atomic_attack_name": "attack", + "display_group": "attack", + "technique_eval_hash": "", + "seed_group_ids": ["seed-0"], + } + ], + "seed_groups": [{"id": "seed-0", "objective_sha256": "sha-legacy-0", "objective": "objective-legacy-0"}], + } + + with_plan = await ScenarioProgressReadModel(memory=memory).get_snapshot_async( + scenario_result_id="legacy", + plan=ScenarioRunPlan.model_validate(stored_plan), + plan_complete=True, + active_group_ids=(), + terminal=True, + objective_scorer_identifier=None, + ) + without_plan = await _get_snapshot_async(read_model=ScenarioProgressReadModel(memory=memory), run_id="legacy") + + # A stored legacy plan is re-saved without gaining a kind, and progress reports it as unknown. + resaved = ScenarioRunPlan.model_validate(stored_plan).model_dump(mode="json", exclude_none=True) + assert all("kind" not in group for group in resaved["atomic_groups"]) + assert [group.kind for group in with_plan.summary.atomic_groups] == [ScenarioRunPlanGroupKind.UNKNOWN] + assert [group.kind for group in without_plan.summary.atomic_groups] == [ScenarioRunPlanGroupKind.UNKNOWN] + + +@pytest.mark.usefixtures("patch_central_database") +async def test_preparation_failure_stays_separate_from_result_role_async(sqlite_instance: MemoryInterface) -> None: + scenario = make_scenario_result(attack_results={}, objective_target_identifier=get_mock_target_identifier()) + await sqlite_instance.add_scenario_results_to_memory_async(scenario_results=[scenario]) + failure = AttackPreparationFailure(kind=AttackPreparationFailureKind.ADVERSARIAL_CHAT_REFUSED, reason="refused") + await sqlite_instance.add_attack_results_to_memory_async( + attack_results=[ + AttackResult( + conversation_id="conversation", + objective="objective", + outcome=AttackOutcome.UNDETERMINED, + outcome_reason="refused", + metadata=failure.to_metadata(), + attribution_parent_id=str(scenario.id), + attribution_data={"parent_collection": "attack", "result_role": "target_facing"}, + ) + ] + ) + + [delta], _ = await sqlite_instance.get_scenario_attack_result_deltas_async( + scenario_result_id=str(scenario.id), cursor=None, limit=10 + ) + mapped = ScenarioProgressReadModel._map_progress_delta( + delta=delta, plan_lookup=ScenarioPlanLookup.from_plan(plan=None) + ) + [stored] = await sqlite_instance.get_attack_results_async(attack_result_ids=[delta.attack_result_id]) + + # The role describes the record, not whether the target was reached; the failure signal is unchanged. + assert mapped.result_role is AttackResultRole.TARGET_FACING + assert mapped.outcome is AttackOutcome.UNDETERMINED + assert mapped.child_attack_result_ids == [] + assert AttackPreparationFailure.from_result(result=stored) == failure diff --git a/tests/unit/executor/attack/compound/test_sequential_attack.py b/tests/unit/executor/attack/compound/test_sequential_attack.py index 9b98e1cc1a..07d16fc573 100644 --- a/tests/unit/executor/attack/compound/test_sequential_attack.py +++ b/tests/unit/executor/attack/compound/test_sequential_attack.py @@ -18,7 +18,14 @@ from pyrit.executor.attack.core.attack_executor import AttackExecutor, AttackExecutorResult from pyrit.executor.attack.core.attack_parameters import AttackParameters from pyrit.executor.attack.core.attack_strategy import AttackContext -from pyrit.models import AttackOutcome, AttackResult, AttackSeedGroup, ScoringExpectation, SeedObjective +from pyrit.models import ( + AttackOutcome, + AttackResult, + AttackResultRole, + AttackSeedGroup, + ScoringExpectation, + SeedObjective, +) def _make_strategy(*, outcomes: list[AttackOutcome], name: str = "attack") -> MagicMock: @@ -534,7 +541,8 @@ async def _fake_execute(**kwargs): async def test_executor_receives_context_attribution(self, target, seed_group): """When the compound's context carries attribution (e.g. nested under a Scenario), it must be forwarded to the executor so the inner - ``AttackResult`` rows can be attributed to the parent.""" + ``AttackResult`` rows can be attributed to the parent, with the + child's 1-based position added.""" from pyrit.executor.attack.core.attack_result_attribution import AttackResultAttribution a = _make_strategy(outcomes=[AttackOutcome.SUCCESS], name="a") @@ -559,7 +567,46 @@ async def _fake_execute(**kwargs): ): await compound._perform_async(context=context) - assert executor_call_kwargs["attribution"] is attribution + assert executor_call_kwargs["attribution"] == AttackResultAttribution( + parent_id="scenario-1", parent_collection="scenario_results", attempt_index=1 + ) + + +@pytest.mark.usefixtures("patch_central_database") +class TestResultRoles: + def test_sequential_attack_is_orchestration_and_children_are_target_facing(self) -> None: + assert SequentialAttack.RESULT_ROLE is AttackResultRole.ORCHESTRATION + assert PromptSendingAttack.RESULT_ROLE is AttackResultRole.TARGET_FACING + + async def test_each_child_receives_its_position_and_the_parent_attribution(self, target, seed_group): + from pyrit.executor.attack.core.attack_result_attribution import AttackResultAttribution + + a = _make_strategy(outcomes=[AttackOutcome.FAILURE], name="a") + b = _make_strategy(outcomes=[AttackOutcome.FAILURE], name="b") + c = _make_strategy(outcomes=[AttackOutcome.SUCCESS], name="c") + compound = SequentialAttack( + objective_target=target, + child_attacks=[SequentialChildAttack(strategy=s, seed_group=seed_group) for s in (a, b, c)], + ) + parent = AttackResultAttribution( + parent_id="scenario-1", parent_collection="adaptive_x", parent_eval_hash="eval", seed_group_id="seed" + ) + context = _make_context() + context._attribution = parent + + patcher, calls = _patch_run_child_attack(strategies_by_id={id(a): a, id(b): b, id(c): c}) + with patcher: + await compound._perform_async(context=context) + + assert [call["attribution"].attempt_index for call in calls] == [1, 2, 3] + for call in calls: + assert call["attribution"].parent_id == parent.parent_id + assert call["attribution"].parent_collection == parent.parent_collection + assert call["attribution"].parent_eval_hash == parent.parent_eval_hash + assert call["attribution"].seed_group_id == parent.seed_group_id + # The parent's own attribution is unchanged, so its row carries no position. + assert context._attribution is parent + assert parent.attempt_index is None @pytest.mark.usefixtures("patch_central_database") diff --git a/tests/unit/executor/attack/core/test_attack_strategy.py b/tests/unit/executor/attack/core/test_attack_strategy.py index 3717c1fc1e..43abc34d69 100644 --- a/tests/unit/executor/attack/core/test_attack_strategy.py +++ b/tests/unit/executor/attack/core/test_attack_strategy.py @@ -25,6 +25,7 @@ from pyrit.models import ( AttackOutcome, AttackResult, + AttackResultRole, ComponentIdentifier, ConversationReference, ConversationType, @@ -889,6 +890,7 @@ async def test_on_post_execute_stamps_scenario_attribution_when_present( assert sample_attack_result.attribution_parent_id == "scenario-1" assert sample_attack_result.attribution_data == { "parent_collection": "atomic_a", + "result_role": "target_facing", "seed_group_id": "seed-a", } @@ -944,9 +946,48 @@ async def test_on_error_stamps_scenario_attribution_when_present(self, sample_at assert persisted.attribution_parent_id == "scenario-err" assert persisted.attribution_data == { "parent_collection": "atomic_err", + "result_role": "target_facing", "seed_group_id": "seed-error", } + @pytest.mark.parametrize("event", [StrategyEvent.ON_POST_EXECUTE, StrategyEvent.ON_ERROR]) + async def test_attribution_records_result_role_and_attempt_index( + self, event, sample_attack_context, sample_attack_result, mock_memory + ): + """Completed and error results both carry the context's role and the child's position.""" + from pyrit.executor.attack.core.attack_result_attribution import AttackResultAttribution + + with patch("pyrit.memory.central_memory.CentralMemory.get_memory_instance", return_value=mock_memory): + handler = _DefaultAttackStrategyEventHandler() + sample_attack_context.start_time = 100.0 + sample_attack_context._result_role = AttackResultRole.ORCHESTRATION + sample_attack_context._attribution = AttackResultAttribution( + parent_id="scenario-1", + parent_collection="atomic_a", + attempt_index=2, + ) + is_error = event is StrategyEvent.ON_ERROR + event_data = StrategyEventData( + event=event, + strategy_name="TestStrategy", + strategy_id="test-id", + context=sample_attack_context, + result=None if is_error else sample_attack_result, + error=RuntimeError("boom") if is_error else None, + ) + await handler.on_event_async(event_data) + + persisted = ( + mock_memory.add_attack_results_to_memory_async.call_args.kwargs["attack_results"][0] + if is_error + else sample_attack_result + ) + assert persisted.attribution_data == { + "parent_collection": "atomic_a", + "result_role": "orchestration", + "attempt_index": 2, + } + async def test_on_post_execute_stamps_targeted_harm_categories(self, sample_attack_result, mock_memory): """Harm categories from context.params are stamped onto the persisted result.""" diff --git a/tests/unit/scenario/core/test_scenario.py b/tests/unit/scenario/core/test_scenario.py index bfef6b0280..343ea7ff14 100644 --- a/tests/unit/scenario/core/test_scenario.py +++ b/tests/unit/scenario/core/test_scenario.py @@ -19,6 +19,7 @@ AttackResult, AttackSeedGroup, ComponentIdentifier, + ScenarioRunPlanGroupKind, ScenarioRunState, SeedObjective, SeedPrompt, @@ -107,6 +108,7 @@ def mock_atomic_attacks(): mock_attack.get_attack_scoring_config.return_value = MagicMock() run1 = MagicMock(spec=AtomicAttack) + run1.group_kind = ScenarioRunPlanGroupKind.ATTACK run1.atomic_attack_name = "attack_run_1" run1.display_group = "attack_run_1" run1._attack = mock_attack @@ -115,6 +117,7 @@ def mock_atomic_attacks(): type(run1).objectives = PropertyMock(return_value=["objective1"]) run2 = MagicMock(spec=AtomicAttack) + run2.group_kind = ScenarioRunPlanGroupKind.ATTACK run2.atomic_attack_name = "attack_run_2" run2.display_group = "attack_run_2" run2._attack = mock_attack @@ -123,6 +126,7 @@ def mock_atomic_attacks(): type(run2).objectives = PropertyMock(return_value=["objective2"]) run3 = MagicMock(spec=AtomicAttack) + run3.group_kind = ScenarioRunPlanGroupKind.ATTACK run3.atomic_attack_name = "attack_run_3" run3.display_group = "attack_run_3" run3._attack = mock_attack @@ -319,6 +323,7 @@ async def test_initialize_async_deduplicates_logical_seed_groups_in_run_plan(sel AttackSeedGroup(seeds=[SeedObjective(value="duplicate objective")]), ] atomic_attack = MagicMock(spec=AtomicAttack) + atomic_attack.group_kind = ScenarioRunPlanGroupKind.ATTACK atomic_attack.atomic_attack_name = "duplicate_attack" atomic_attack.display_group = "duplicate_attack" atomic_attack.technique_eval_hash = "duplicate-technique" @@ -352,6 +357,7 @@ async def test_build_run_plan_preserves_unique_seed_group_order(self, mock_objec AttackSeedGroup(seeds=[SeedObjective(value="second objective")]), ] atomic_attack = MagicMock(spec=AtomicAttack) + atomic_attack.group_kind = ScenarioRunPlanGroupKind.ATTACK atomic_attack.atomic_attack_name = "unique_attack" atomic_attack.display_group = "custom display group" atomic_attack.technique_name = "test" @@ -756,6 +762,7 @@ async def test_atomic_attack_count_with_different_sizes(self, mock_objective_tar mock_attack.get_attack_scoring_config.return_value = MagicMock() single_run_mock = MagicMock(spec=AtomicAttack) + single_run_mock.group_kind = ScenarioRunPlanGroupKind.ATTACK single_run_mock.atomic_attack_name = "attack_1" single_run_mock.display_group = "attack_1" single_run_mock._attack = mock_attack @@ -778,6 +785,7 @@ async def test_atomic_attack_count_with_different_sizes(self, mock_objective_tar many_runs = [] for i in range(10): run = MagicMock(spec=AtomicAttack) + run.group_kind = ScenarioRunPlanGroupKind.ATTACK run.atomic_attack_name = f"attack_{i}" run.display_group = f"attack_{i}" run._attack = mock_attack diff --git a/tests/unit/scenario/core/test_scenario_partial_results.py b/tests/unit/scenario/core/test_scenario_partial_results.py index 5bfc8eabea..1c63f7b662 100644 --- a/tests/unit/scenario/core/test_scenario_partial_results.py +++ b/tests/unit/scenario/core/test_scenario_partial_results.py @@ -17,6 +17,7 @@ AttackResult, AttackSeedGroup, ComponentIdentifier, + ScenarioRunPlanGroupKind, ScenarioRunState, SeedObjective, config_hash, @@ -77,6 +78,7 @@ def create_mock_atomic_attack(name: str, objectives: list[str]) -> MagicMock: mock_attack_strategy.get_attack_scoring_config.return_value = MagicMock() attack = MagicMock(spec=AtomicAttack) + attack.group_kind = ScenarioRunPlanGroupKind.ATTACK attack.atomic_attack_name = name attack.display_group = name attack.technique_eval_hash = config_hash({"name": name, "objectives": objectives}) diff --git a/tests/unit/scenario/core/test_scenario_retry.py b/tests/unit/scenario/core/test_scenario_retry.py index a404b37f78..916698bec3 100644 --- a/tests/unit/scenario/core/test_scenario_retry.py +++ b/tests/unit/scenario/core/test_scenario_retry.py @@ -22,6 +22,7 @@ AttackSeedGroup, ComponentIdentifier, Message, + ScenarioRunPlanGroupKind, ScenarioRunState, SeedObjective, config_hash, @@ -150,6 +151,7 @@ def create_mock_atomic_attack(name: str, objectives: list[str], run_async_mock: mock_attack_strategy.get_attack_scoring_config.return_value = MagicMock() attack = MagicMock(spec=AtomicAttack) + attack.group_kind = ScenarioRunPlanGroupKind.ATTACK attack.atomic_attack_name = name attack.display_group = name attack.technique_eval_hash = config_hash({"name": name, "objectives": objectives}) @@ -1099,6 +1101,7 @@ def _make_scenario_with_atomics(self, atomics): def test_noop_when_metadata_has_no_persisted_hashes(self): atomic = MagicMock(spec=AtomicAttack) + atomic.group_kind = ScenarioRunPlanGroupKind.ATTACK scenario = self._make_scenario_with_atomics([atomic]) stored = MagicMock() stored.metadata = {} @@ -1107,8 +1110,10 @@ def test_noop_when_metadata_has_no_persisted_hashes(self): def test_replays_persisted_subset_across_atomics(self): atomic_a = MagicMock(spec=AtomicAttack) + atomic_a.group_kind = ScenarioRunPlanGroupKind.ATTACK atomic_a.keep_seed_groups_with_hashes.return_value = {"h1", "h2"} atomic_b = MagicMock(spec=AtomicAttack) + atomic_b.group_kind = ScenarioRunPlanGroupKind.ATTACK atomic_b.keep_seed_groups_with_hashes.return_value = {"h3"} scenario = self._make_scenario_with_atomics([atomic_a, atomic_b]) @@ -1121,6 +1126,7 @@ def test_replays_persisted_subset_across_atomics(self): def test_raises_when_persisted_hash_is_missing(self): atomic = MagicMock(spec=AtomicAttack) + atomic.group_kind = ScenarioRunPlanGroupKind.ATTACK atomic.keep_seed_groups_with_hashes.return_value = {"h1"} # h2 missing scenario = self._make_scenario_with_atomics([atomic]) diff --git a/tests/unit/scenario/scenarios/adaptive/test_text_adaptive.py b/tests/unit/scenario/scenarios/adaptive/test_text_adaptive.py index 81fa4adcf5..b54adbb009 100644 --- a/tests/unit/scenario/scenarios/adaptive/test_text_adaptive.py +++ b/tests/unit/scenario/scenarios/adaptive/test_text_adaptive.py @@ -11,7 +11,7 @@ import pytest -from pyrit.models import AttackSeedGroup, SeedObjective +from pyrit.models import AttackSeedGroup, ScenarioRunPlanGroupKind, SeedObjective from pyrit.models.identifiers import ComponentIdentifier from pyrit.prompt_target import PromptTarget from pyrit.registry.components.attack_technique_registry import AttackTechniqueRegistry @@ -698,3 +698,29 @@ async def test_baseline_emitted_at_index_zero_by_default(self, mock_objective_ta assert scenario._atomic_attacks[0].atomic_attack_name == "baseline", ( f"baseline must be prepended at index 0; got {[a.atomic_attack_name for a in scenario._atomic_attacks]}" ) + + async def test_run_plan_records_baseline_and_adaptive_group_kinds( + self, mock_objective_target, mock_objective_scorer + ): + groups = { + "violence": [ + _make_seed_group(value="obj-1", harm_categories=["violence"]), + _make_seed_group(value="obj-2", harm_categories=["violence"]), + ] + } + with patch.object( + CompoundDatasetAttackConfiguration, + "get_attack_groups_by_dataset_async", + new_callable=AsyncMock, + return_value=groups, + ): + scenario = TextAdaptive(objective_scorer=mock_objective_scorer) + scenario.set_params_from_args(args={"objective_target": mock_objective_target}) + await scenario.initialize_async() + + kinds = [group.kind for group in scenario._build_run_plan().atomic_groups] + assert kinds == [ + ScenarioRunPlanGroupKind.BASELINE, + ScenarioRunPlanGroupKind.ADAPTIVE, + ScenarioRunPlanGroupKind.ADAPTIVE, + ] From 908695de6bc8b749c6fdd8b010c32c325ef1e8ab Mon Sep 17 00:00:00 2001 From: shashank Date: Tue, 6 Oct 2026 23:31:35 +0530 Subject: [PATCH 2/3] FIX Record result role for attacks run outside a Scenario _apply_attribution returned early when the context had no Scenario attribution, so standalone execute_async calls stored results with no result_role and read back as unknown. Always stamp result_role, and only set attribution_parent_id and the parent linkage fields when attribution is present. Add real-SQLite tests for standalone PromptSendingAttack and SequentialAttack on both the completed and error paths. --- pyrit/executor/attack/core/attack_strategy.py | 23 +++++----- .../core/test_attack_result_correlation.py | 46 +++++++++++++++++++ .../attack/core/test_attack_strategy.py | 8 ++-- 3 files changed, 62 insertions(+), 15 deletions(-) diff --git a/pyrit/executor/attack/core/attack_strategy.py b/pyrit/executor/attack/core/attack_strategy.py index 92708a0740..63f7828b42 100644 --- a/pyrit/executor/attack/core/attack_strategy.py +++ b/pyrit/executor/attack/core/attack_strategy.py @@ -418,7 +418,7 @@ async def _on_post_execute_async( # Stamp attribution onto the result before persistence so the # AttackResultEntry row records its lineage. Outside an orchestrator - # _attribution is None and both attribution fields stay None. + # _attribution is None, so only the result role is recorded. event_data.result.related_conversations.update(event_data.context.related_conversations) self._apply_attribution(context=event_data.context, result=event_data.result) self._apply_targeted_harm_categories(context=event_data.context, result=event_data.result) @@ -445,25 +445,26 @@ def _apply_attribution( """ Copy attribution from the AttackContext onto the AttackResult. - Reads ``context._attribution`` (an ``AttackResultAttribution`` set by - the AttackExecutor when an upstream orchestrator supplied a factory). - When present, writes ``attribution_parent_id`` and a fixed-schema - ``attribution_data`` dict onto the result so they round-trip into - ``AttackResultEntry``. The dict also records ``result_role``, the - producing strategy's ``RESULT_ROLE``. + Always writes a fixed-schema ``attribution_data`` dict recording + ``result_role``, the producing strategy's ``RESULT_ROLE``, so standalone + results are classified too. When ``context._attribution`` (an + ``AttackResultAttribution`` set by the AttackExecutor when an upstream + orchestrator supplied a factory) is present, also writes + ``attribution_parent_id`` and the parent linkage fields so they + round-trip into ``AttackResultEntry``. Without it, + ``attribution_parent_id`` stays None. Args: context: The per-task AttackContext. result: The AttackResult that is about to be persisted. """ + attribution_data: dict[str, Any] = {"result_role": context._result_role.value} attribution = context._attribution if attribution is None: + result.attribution_data = attribution_data return result.attribution_parent_id = attribution.parent_id - attribution_data: dict[str, Any] = { - "parent_collection": attribution.parent_collection, - "result_role": context._result_role.value, - } + attribution_data["parent_collection"] = attribution.parent_collection if attribution.parent_eval_hash is not None: attribution_data["parent_eval_hash"] = attribution.parent_eval_hash if attribution.seed_group_id is not None: diff --git a/tests/unit/executor/attack/core/test_attack_result_correlation.py b/tests/unit/executor/attack/core/test_attack_result_correlation.py index 2da3c0c004..c0b9573ab4 100644 --- a/tests/unit/executor/attack/core/test_attack_result_correlation.py +++ b/tests/unit/executor/attack/core/test_attack_result_correlation.py @@ -246,6 +246,52 @@ async def test_error_result_keeps_the_allocated_result_id_async(sqlite_instance: assert linked[1].response_error != "none" +@pytest.mark.parametrize("fail", [False, True]) +async def test_standalone_result_records_its_role_without_a_parent_async( + sqlite_instance: SQLiteMemory, fail: bool +) -> None: + attack = PromptSendingAttack(objective_target=_RecordingTarget(fail=fail)) + context = SingleTurnAttackContext(params=AttackParameters(objective="objective")) + + if fail: + with pytest.raises(RuntimeError): + await attack.execute_with_context_async(context=context) + else: + await attack.execute_with_context_async(context=context) + + [stored] = await sqlite_instance.get_attack_results_async(attack_result_ids=[context.attack_result_id]) + assert (stored.outcome == AttackOutcome.ERROR) is fail + assert stored.attribution_parent_id is None + assert stored.attribution_data == {"result_role": "target_facing"} + + +@pytest.mark.parametrize("fail", [False, True]) +async def test_standalone_sequential_results_record_their_roles_without_a_parent_async( + sqlite_instance: SQLiteMemory, fail: bool +) -> None: + target = _RecordingTarget(fail=fail) + seed_group = AttackSeedGroup(seeds=[SeedObjective(value="objective")]) + sequential = SequentialAttack( + objective_target=target, + child_attacks=[ + SequentialChildAttack(strategy=PromptSendingAttack(objective_target=target), seed_group=seed_group) + ], + ) + + if fail: + with pytest.raises(RuntimeError): + await sequential.execute_async(objective="objective") + else: + await sequential.execute_async(objective="objective") + + stored = await sqlite_instance.get_attack_results_async() + assert len(stored) == 2 + assert all((result.outcome == AttackOutcome.ERROR) is fail for result in stored) + assert all(result.attribution_parent_id is None for result in stored) + assert sorted(result.attribution_data["result_role"] for result in stored) == ["orchestration", "target_facing"] + assert all(result.attribution_data.keys() == {"result_role"} for result in stored) + + async def test_history_from_an_earlier_execution_is_copied_into_a_new_conversation_async( sqlite_instance: SQLiteMemory, ) -> None: diff --git a/tests/unit/executor/attack/core/test_attack_strategy.py b/tests/unit/executor/attack/core/test_attack_strategy.py index 43abc34d69..c84dcfb7bb 100644 --- a/tests/unit/executor/attack/core/test_attack_strategy.py +++ b/tests/unit/executor/attack/core/test_attack_strategy.py @@ -894,11 +894,11 @@ async def test_on_post_execute_stamps_scenario_attribution_when_present( "seed_group_id": "seed-a", } - async def test_on_post_execute_no_attribution_leaves_fields_none( + async def test_on_post_execute_no_attribution_records_only_result_role( self, sample_attack_context, sample_attack_result, mock_memory ): - """Outside a Scenario, _attribution is None and the attribution fields - on the persisted AttackResult must stay None.""" + """Outside a Scenario, _attribution is None, so the persisted AttackResult + records its role but no parent link.""" with patch("pyrit.memory.central_memory.CentralMemory.get_memory_instance", return_value=mock_memory): handler = _DefaultAttackStrategyEventHandler() sample_attack_context.start_time = 100.0 @@ -914,7 +914,7 @@ async def test_on_post_execute_no_attribution_leaves_fields_none( await handler.on_event_async(event_data) assert sample_attack_result.attribution_parent_id is None - assert sample_attack_result.attribution_data is None + assert sample_attack_result.attribution_data == {"result_role": "target_facing"} async def test_on_error_stamps_scenario_attribution_when_present(self, sample_attack_context, mock_memory): """Error AttackResults must also carry the attribution foreign key so From 6f53a53210a0daba9e0866a1640e5e3c8e95f848 Mon Sep 17 00:00:00 2001 From: shashank Date: Wed, 7 Oct 2026 06:39:49 +0530 Subject: [PATCH 3/3] DOC Clarify that attempt_index is relative to the immediate parent attempt_index matches the _adaptive_attempt label only for techniques that Adaptive runs directly. When a selected technique is itself a compound attack, its children are numbered under that nested parent while their label still names the outer Adaptive attempt. Qualify the statement in both paired adaptive scenario docs and add a regression that runs a nested SequentialAttack through AdaptiveTechniqueDispatcher, a real Scenario and SQLite, and pins the parent-relative index through the progress projection. --- doc/code/scenarios/3_adaptive_scenarios.ipynb | 8 ++- doc/code/scenarios/3_adaptive_scenarios.py | 8 ++- .../test_scenario_progress_read_model.py | 65 ++++++++++++++++++- 3 files changed, 75 insertions(+), 6 deletions(-) diff --git a/doc/code/scenarios/3_adaptive_scenarios.ipynb b/doc/code/scenarios/3_adaptive_scenarios.ipynb index d72ede8915..8092b162c1 100644 --- a/doc/code/scenarios/3_adaptive_scenarios.ipynb +++ b/doc/code/scenarios/3_adaptive_scenarios.ipynb @@ -776,8 +776,12 @@ " does not prove a request reached the target: an attack that ends in a preparation failure is still\n", " `target_facing`.\n", "- `child_attack_result_ids` lists an orchestration parent's children in the order they ran.\n", - "- `attempt_index` is a child's 1-based position under its parent. For Adaptive it matches the\n", - " `_adaptive_attempt` memory label.\n", + "- `attempt_index` is a child's 1-based position under its immediate parent. For a technique that\n", + " Adaptive runs directly, it matches the `_adaptive_attempt` memory label. When that technique is\n", + " itself a compound attack such as a nested `SequentialAttack`, its children are numbered under the\n", + " nested parent instead, so their `attempt_index` is not the Adaptive attempt number. Their\n", + " `_adaptive_attempt` label still names the outer attempt, but the progress response does not\n", + " include it.\n", "- Each `summary.atomic_groups` entry has a `kind`: `attack`, `baseline`, `adaptive`, or `unknown` for\n", " plans saved before kinds were recorded.\n", "\n", diff --git a/doc/code/scenarios/3_adaptive_scenarios.py b/doc/code/scenarios/3_adaptive_scenarios.py index 4b26ac8aad..dad03fff22 100644 --- a/doc/code/scenarios/3_adaptive_scenarios.py +++ b/doc/code/scenarios/3_adaptive_scenarios.py @@ -245,8 +245,12 @@ def _technique_label(result) -> str: # does not prove a request reached the target: an attack that ends in a preparation failure is still # `target_facing`. # - `child_attack_result_ids` lists an orchestration parent's children in the order they ran. -# - `attempt_index` is a child's 1-based position under its parent. For Adaptive it matches the -# `_adaptive_attempt` memory label. +# - `attempt_index` is a child's 1-based position under its immediate parent. For a technique that +# Adaptive runs directly, it matches the `_adaptive_attempt` memory label. When that technique is +# itself a compound attack such as a nested `SequentialAttack`, its children are numbered under the +# nested parent instead, so their `attempt_index` is not the Adaptive attempt number. Their +# `_adaptive_attempt` label still names the outer attempt, but the progress response does not +# include it. # - Each `summary.atomic_groups` entry has a `kind`: `attack`, `baseline`, `adaptive`, or `unknown` for # plans saved before kinds were recorded. # diff --git a/tests/unit/backend/test_scenario_progress_read_model.py b/tests/unit/backend/test_scenario_progress_read_model.py index 867b28ac14..264adcee52 100644 --- a/tests/unit/backend/test_scenario_progress_read_model.py +++ b/tests/unit/backend/test_scenario_progress_read_model.py @@ -48,7 +48,11 @@ from pyrit.scenario.core import AtomicAttack, BaselineAttackPolicy, Scenario, ScenarioTechnique from pyrit.scenario.core.attack_technique import AttackTechnique from pyrit.scenario.core.matrix_atomic_attack_builder import build_baseline_atomic_attack -from pyrit.scenario.scenarios.adaptive.dispatcher import ADAPTIVE_ATTEMPT_LABEL +from pyrit.scenario.scenarios.adaptive.dispatcher import ( + ADAPTIVE_ATTEMPT_LABEL, + AdaptiveTechniqueDispatcher, + TechniqueBundle, +) from pyrit.score import Scorer, SubStringScorer from unit.mocks import MockPromptTarget, get_mock_target_identifier, make_scenario_result @@ -581,7 +585,8 @@ def build(target: PromptTarget) -> list[AtomicAttack]: assert {r.atomic_group_id for r in (parent, *children)} == {parent.atomic_group_id} assert {r.seed_group_id for r in (parent, *children)} == {parent.seed_group_id} - # Adaptive's attempt label is kept on each stored child result and agrees with its index. + # Adaptive's attempt label is kept on each stored child result. These children sit directly + # under the Adaptive parent, so the label agrees with their parent-relative index. stored_children = await sqlite_instance.get_attack_results_async( attack_result_ids=[child.attack_result_id for child in children] ) @@ -619,6 +624,62 @@ async def test_failed_orchestration_parent_keeps_its_role_despite_having_a_conve assert parent.conversation_id != "" assert child.attempt_index == 1 + async def test_nested_compound_attempt_index_is_relative_to_its_own_parent_async( + self, sqlite_instance: MemoryInterface + ) -> None: + class _OrderedSelector: + async def select_async(self, *, technique_identifiers, objective, num_top_techniques, scenario_result_id): + return ["single", "nested"][:num_top_techniques] + + target = MockPromptTarget() + nested_technique = SequentialAttack( + objective_target=target, + child_attacks=[ + SequentialChildAttack(strategy=_prompt_sending(target=target), seed_group=_SEED_GROUP) for _ in range(2) + ], + ) + dispatcher = AdaptiveTechniqueDispatcher( + objective_target=target, + techniques={ + "single": TechniqueBundle(attack=_prompt_sending(target=target), name="single"), + "nested": TechniqueBundle(attack=nested_technique, name="nested"), + }, + selector=_OrderedSelector(), + max_attempts_per_objective=2, + ) + adaptive_group = AtomicAttack( + atomic_attack_name="adaptive_objective", + attack_technique=AttackTechnique(attack=await dispatcher.build_attack_async(seed_group=_SEED_GROUP)), + seed_groups=[_SEED_GROUP], + group_kind=ScenarioRunPlanGroupKind.ADAPTIVE, + ) + + snapshot, _, _ = await _run_and_read_progress_async( + memory=sqlite_instance, build_atomic_attacks=lambda _: [adaptive_group], target=target + ) + + by_id = {result.attack_result_id: result for result in snapshot.results} + [outer] = [ + r for r in snapshot.results if r.result_role is AttackResultRole.ORCHESTRATION and r.attempt_index is None + ] + single_id, nested_id = outer.child_attack_result_ids + single, nested = by_id[single_id], by_id[nested_id] + nested_children = [by_id[child_id] for child_id in nested.child_attack_result_ids] + + assert single.result_role is AttackResultRole.TARGET_FACING + assert nested.result_role is AttackResultRole.ORCHESTRATION + assert [child.result_role for child in nested_children] == [AttackResultRole.TARGET_FACING] * 2 + assert (single.attempt_index, nested.attempt_index) == (1, 2) + # Nested children count from 1 under their own parent, not under the Adaptive parent. + assert [child.attempt_index for child in nested_children] == [1, 2] + + # Adaptive's label still names the outer attempt, so it differs from the nested children's index. + stored = await sqlite_instance.get_attack_results_async( + attack_result_ids=[single_id, *nested.child_attack_result_ids] + ) + labels = {result.attack_result_id: result.labels[ADAPTIVE_ATTEMPT_LABEL] for result in stored} + assert labels == {single_id: "1", **dict.fromkeys(nested.child_attack_result_ids, "2")} + async def test_roles_do_not_change_progress_counts_async(self, sqlite_instance: MemoryInterface) -> None: snapshot, plan, run_id = await _run_and_read_progress_async( memory=sqlite_instance,