FEAT Add explicit Scenario progress result roles

doc/code/scenarios/3_adaptive_scenarios.ipynb

@@ -763,6 +763,59 @@
    "cell_type": "markdown",
    "id": "11",
    "metadata": {},
+   "source": [
+    "## Result roles in scenario progress\n",
+    "\n",
+    "The strategy that produces each result records what the result represents, and the scenario progress\n",
+    "API (`GET /api/scenarios/runs/{scenario_result_id}/progress`) returns it. Clients read these fields\n",
+    "instead of inferring a parent from a class name or an empty conversation ID:\n",
+    "\n",
+    "- `result_role` is `target_facing` for an attack that sends its own requests to the objective target,\n",
+    "  `orchestration` for a parent that only runs other attacks (such as the per-objective\n",
+    "  `SequentialAttack`), and `unknown` for rows saved before roles were recorded. A `target_facing` role\n",
+    "  does not prove a request reached the target: an attack that ends in a preparation failure is still\n",
+    "  `target_facing`.\n",
+    "- `child_attack_result_ids` lists an orchestration parent's children in the order they ran.\n",
+    "- `attempt_index` is a child's 1-based position under its immediate parent. For a technique that\n",
+    "  Adaptive runs directly, it matches the `_adaptive_attempt` memory label. When that technique is\n",
+    "  itself a compound attack such as a nested `SequentialAttack`, its children are numbered under the\n",
+    "  nested parent instead, so their `attempt_index` is not the Adaptive attempt number. Their\n",
+    "  `_adaptive_attempt` label still names the outer attempt, but the progress response does not\n",
+    "  include it.\n",
+    "- Each `summary.atomic_groups` entry has a `kind`: `attack`, `baseline`, `adaptive`, or `unknown` for\n",
+    "  plans saved before kinds were recorded.\n",
+    "\n",
+    "Roles describe results without changing how progress is counted: a parent and its children still\n",
+    "belong to one planned unit.\n",
+    "\n",
+    "This excerpt of a progress response shows one Adaptive objective whose first attempt failed and whose\n",
+    "second succeeded (other fields omitted):\n",
+    "\n",
+    "```json\n",
+    "{\n",
+    "  \"results\": [\n",
+    "    {\"attack_result_id\": \"child-1\", \"conversation_id\": \"conversation-1\", \"outcome\": \"failure\",\n",
+    "     \"result_role\": \"target_facing\", \"child_attack_result_ids\": [], \"attempt_index\": 1},\n",
+    "    {\"attack_result_id\": \"child-2\", \"conversation_id\": \"conversation-2\", \"outcome\": \"success\",\n",
+    "     \"result_role\": \"target_facing\", \"child_attack_result_ids\": [], \"attempt_index\": 2},\n",
+    "    {\"attack_result_id\": \"parent\", \"conversation_id\": \"\", \"outcome\": \"success\",\n",
+    "     \"result_role\": \"orchestration\", \"child_attack_result_ids\": [\"child-1\", \"child-2\"],\n",
+    "     \"attempt_index\": null}\n",
+    "  ],\n",
+    "  \"summary\": {\n",
+    "    \"atomic_groups\": [\n",
+    "      {\"atomic_attack_name\": \"baseline\", \"kind\": \"baseline\", \"completed\": 1, \"planned\": 1},\n",
+    "      {\"atomic_attack_name\": \"adaptive_airt_hate::4f0c...\", \"kind\": \"adaptive\", \"completed\": 1, \"planned\": 1}\n",
+    "    ]\n",
+    "  }\n",
+    "}\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "12",
+   "metadata": {},
    "source": [
     "## Running from the scanner CLI\n",
     "\n",

doc/code/scenarios/3_adaptive_scenarios.py

@@ -232,6 +232,54 @@ def _technique_label(result) -> str:
 for technique, n in total_picks.most_common():
     print(f"{technique:40s}  {total_wins[technique]:>4} / {n:<4}   {total_wins[technique] / n:.0%}")
 
+# %% [markdown]
+# ## Result roles in scenario progress
+#
+# The strategy that produces each result records what the result represents, and the scenario progress
+# API (`GET /api/scenarios/runs/{scenario_result_id}/progress`) returns it. Clients read these fields
+# instead of inferring a parent from a class name or an empty conversation ID:
+#
+# - `result_role` is `target_facing` for an attack that sends its own requests to the objective target,
+#   `orchestration` for a parent that only runs other attacks (such as the per-objective
+#   `SequentialAttack`), and `unknown` for rows saved before roles were recorded. A `target_facing` role
+#   does not prove a request reached the target: an attack that ends in a preparation failure is still
+#   `target_facing`.
+# - `child_attack_result_ids` lists an orchestration parent's children in the order they ran.
+# - `attempt_index` is a child's 1-based position under its immediate parent. For a technique that
+#   Adaptive runs directly, it matches the `_adaptive_attempt` memory label. When that technique is
+#   itself a compound attack such as a nested `SequentialAttack`, its children are numbered under the
+#   nested parent instead, so their `attempt_index` is not the Adaptive attempt number. Their
+#   `_adaptive_attempt` label still names the outer attempt, but the progress response does not
+#   include it.
+# - Each `summary.atomic_groups` entry has a `kind`: `attack`, `baseline`, `adaptive`, or `unknown` for
+#   plans saved before kinds were recorded.
+#
+# Roles describe results without changing how progress is counted: a parent and its children still
+# belong to one planned unit.
+#
+# This excerpt of a progress response shows one Adaptive objective whose first attempt failed and whose
+# second succeeded (other fields omitted):
+#
+# ```json
+# {
+#   "results": [
+#     {"attack_result_id": "child-1", "conversation_id": "conversation-1", "outcome": "failure",
+#      "result_role": "target_facing", "child_attack_result_ids": [], "attempt_index": 1},
+#     {"attack_result_id": "child-2", "conversation_id": "conversation-2", "outcome": "success",
+#      "result_role": "target_facing", "child_attack_result_ids": [], "attempt_index": 2},
+#     {"attack_result_id": "parent", "conversation_id": "", "outcome": "success",
+#      "result_role": "orchestration", "child_attack_result_ids": ["child-1", "child-2"],
+#      "attempt_index": null}
+#   ],
+#   "summary": {
+#     "atomic_groups": [
+#       {"atomic_attack_name": "baseline", "kind": "baseline", "completed": 1, "planned": 1},
+#       {"atomic_attack_name": "adaptive_airt_hate::4f0c...", "kind": "adaptive", "completed": 1, "planned": 1}
+#     ]
+#   }
+# }
+# ```
+
 # %% [markdown]
 # ## Running from the scanner CLI
 #

pyrit/backend/services/scenario_progress_read_model.py

@@ -9,7 +9,7 @@
 from collections.abc import Iterable, Sequence
 from dataclasses import dataclass, field
 from datetime import UTC, datetime
-from typing import Literal
+from typing import Any, Literal
 
 from pyrit.common.utils import to_sha256
 from pyrit.memory import AttackResultKeysetCursor
@@ -18,6 +18,7 @@
     AtomicAttackIdentifier,
     AttackOutcome,
     AttackResult,
+    AttackResultRole,
     AttackTechniqueIdentifier,
     ComponentIdentifier,
     ScenarioAtomicGroupProgress,
@@ -33,6 +34,7 @@
     ScenarioResult,
     ScenarioRunPlan,
     ScenarioRunPlanAtomicGroup,
+    ScenarioRunPlanGroupKind,
     ScenarioRunPlanSeedGroup,
     ScenarioScorerIdentity,
     ScenarioSeedGroupProgress,
@@ -551,6 +553,7 @@ def aggregate(*, units: Sequence[ResultUnitIdentity], planned: int | None) -> Sc
                     display_group=group.display_group,
                     status=group_status,
                     technique_details=technique_details_by_group.get(group.id),
+                    kind=group.kind or ScenarioRunPlanGroupKind.UNKNOWN,
                     **counts.model_dump(),
                 )
             )
@@ -841,8 +844,55 @@ def _map_progress_delta(
             error_type=delta.error_type,
             error_message=delta.error_message,
             score=delta.score,
+            result_role=ScenarioProgressReadModel._read_result_role(attribution_data=delta.attribution_data),
+            child_attack_result_ids=ScenarioProgressReadModel._read_child_attack_result_ids(
+                attack_metadata=delta.attack_metadata
+            ),
+            attempt_index=ScenarioProgressReadModel._read_attempt_index(attribution_data=delta.attribution_data),
         )
 
+    @staticmethod
+    def _read_result_role(*, attribution_data: dict[str, Any]) -> AttackResultRole:
+        """
+        Read the role recorded by the producing strategy.
+
+        A row without a recognized role is ``UNKNOWN``. Nothing is inferred from other
+        fields, such as an empty conversation ID.
+
+        Returns:
+            AttackResultRole: The recorded role, or ``UNKNOWN``.
+        """
+        try:
+            return AttackResultRole(attribution_data.get("result_role"))
+        except ValueError:
+            return AttackResultRole.UNKNOWN
+
+    @staticmethod
+    def _read_child_attack_result_ids(*, attack_metadata: dict[str, Any]) -> list[str]:
+        """
+        Read the ordered child result IDs that ``SequentialAttack`` stores in its metadata.
+
+        Returns:
+            list[str]: The child IDs in stored order, or an empty list when none are recorded.
+        """
+        child_ids = attack_metadata.get("child_attack_result_ids")
+        if isinstance(child_ids, list) and all(isinstance(child_id, str) for child_id in child_ids):
+            return list(child_ids)
+        return []
+
+    @staticmethod
+    def _read_attempt_index(*, attribution_data: dict[str, Any]) -> int | None:
+        """
+        Read a child result's 1-based position under its orchestration parent.
+
+        Returns:
+            int | None: The recorded position, or None when absent or invalid.
+        """
+        attempt_index = attribution_data.get("attempt_index")
+        if isinstance(attempt_index, int) and not isinstance(attempt_index, bool) and attempt_index >= 1:
+            return attempt_index
+        return None
+
     @staticmethod
     def _synthesize_legacy_plan(*, deltas: list[ScenarioAttackResultDelta]) -> ScenarioRunPlan:
         """

pyrit/executor/attack/compound/sequential_attack.py

@@ -23,7 +23,7 @@
 
 import logging
 import uuid
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
 from datetime import UTC, datetime
 from enum import Enum
 from typing import TYPE_CHECKING, Any, ClassVar
@@ -33,7 +33,7 @@
 from pyrit.executor.attack.core.attack_executor import AttackExecutor
 from pyrit.executor.attack.core.attack_parameters import AttackParameters
 from pyrit.executor.attack.core.attack_strategy import AttackContext, AttackStrategy
-from pyrit.models import AttackOutcome, AttackResult, AttackSeedGroup, ScoringExpectation
+from pyrit.models import AttackOutcome, AttackResult, AttackResultRole, AttackSeedGroup, ScoringExpectation
 
 if TYPE_CHECKING:
     from collections.abc import Mapping, Sequence
@@ -191,6 +191,8 @@ class SequentialAttack(AttackStrategy[AttackContext[AttackParameters], Sequentia
 
     DELEGATES_SCORING: ClassVar[bool] = True
 
+    RESULT_ROLE: ClassVar[AttackResultRole] = AttackResultRole.ORCHESTRATION
+
     CHILD_ATTACK_RESULT_IDS_KEY: str = "child_attack_result_ids"
     """Metadata key under which the per-child-attack result IDs are stored."""
 
@@ -249,12 +251,14 @@ async def _teardown_async(self, *, context: AttackContext[AttackParameters]) ->
     async def _perform_async(self, *, context: AttackContext[AttackParameters]) -> SequentialAttackResult:
         results: list[AttackResult] = []
 
-        for child_attack in self._child_attacks:
+        for attempt_index, child_attack in enumerate(self._child_attacks, start=1):
             labels = {**context.memory_labels, **dict(child_attack.memory_labels)}
+            # Each child shares the parent's attribution plus its own position.
+            attribution = replace(context._attribution, attempt_index=attempt_index) if context._attribution else None
             result = await self._run_child_attack_async(
                 child_attack=child_attack,
                 memory_labels=labels,
-                attribution=context._attribution,
+                attribution=attribution,
                 expectation=context.params.expectation,
             )
             results.append(result)

pyrit/executor/attack/core/attack_result_attribution.py

@@ -46,9 +46,14 @@ class AttackResultAttribution:
             ``AtomicAttackEvaluationIdentifier``).
         seed_group_id (str | None): Optional logical seed-group fingerprint for
             per-task progress attribution.
+        attempt_index (int | None): Optional 1-based position of this result among the
+            children of an orchestration parent. Persisted into
+            ``AttackResultEntry.attribution_data``. ``SequentialAttack`` sets it for each
+            child attack it dispatches, e.g. ``2`` for the second child.
     """
 
     parent_id: str
     parent_collection: str
     parent_eval_hash: str | None = None
     seed_group_id: str | None = None
+    attempt_index: int | None = None

pyrit/executor/attack/core/attack_strategy.py

@@ -33,6 +33,7 @@
     AttackIdentifier,
     AttackOutcome,
     AttackResult,
+    AttackResultRole,
     ComponentIdentifier,
     ConversationReference,
     ConverterIdentifier,
@@ -206,6 +207,12 @@ class AttackContext(StrategyContext, ABC, Generic[AttackParamsT]):
     # ID of the AttackResult this execution produces. Allocated when execution starts.
     _attack_result_id: str | None = field(default=None, init=False, repr=False, compare=False)
 
+    # Role of the AttackResult this execution produces. Copied from the strategy's
+    # ``RESULT_ROLE`` when execution starts, so success and error results both carry it.
+    _result_role: AttackResultRole = field(
+        default=AttackResultRole.TARGET_FACING, init=False, repr=False, compare=False
+    )
+
     _expectation: ScoringExpectation = field(init=False, repr=False)
 
     def __post_init__(self) -> None:
@@ -411,7 +418,7 @@ async def _on_post_execute_async(
 
         # Stamp attribution onto the result before persistence so the
         # AttackResultEntry row records its lineage. Outside an orchestrator
-        # _attribution is None and both attribution fields stay None.
+        # _attribution is None, so only the result role is recorded.
         event_data.result.related_conversations.update(event_data.context.related_conversations)
         self._apply_attribution(context=event_data.context, result=event_data.result)
         self._apply_targeted_harm_categories(context=event_data.context, result=event_data.result)
@@ -438,27 +445,32 @@ def _apply_attribution(
         """
         Copy attribution from the AttackContext onto the AttackResult.
 
-        Reads ``context._attribution`` (an ``AttackResultAttribution`` set by
-        the AttackExecutor when an upstream orchestrator supplied a factory).
-        When present, writes ``attribution_parent_id`` and a fixed-schema
-        ``attribution_data`` dict onto the result so they round-trip into
-        ``AttackResultEntry``.
+        Always writes a fixed-schema ``attribution_data`` dict recording
+        ``result_role``, the producing strategy's ``RESULT_ROLE``, so standalone
+        results are classified too. When ``context._attribution`` (an
+        ``AttackResultAttribution`` set by the AttackExecutor when an upstream
+        orchestrator supplied a factory) is present, also writes
+        ``attribution_parent_id`` and the parent linkage fields so they
+        round-trip into ``AttackResultEntry``. Without it,
+        ``attribution_parent_id`` stays None.
 
         Args:
             context: The per-task AttackContext.
             result: The AttackResult that is about to be persisted.
         """
+        attribution_data: dict[str, Any] = {"result_role": context._result_role.value}
         attribution = context._attribution
         if attribution is None:
+            result.attribution_data = attribution_data
             return
         result.attribution_parent_id = attribution.parent_id
-        attribution_data: dict[str, Any] = {
-            "parent_collection": attribution.parent_collection,
-        }
+        attribution_data["parent_collection"] = attribution.parent_collection
         if attribution.parent_eval_hash is not None:
             attribution_data["parent_eval_hash"] = attribution.parent_eval_hash
         if attribution.seed_group_id is not None:
             attribution_data["seed_group_id"] = attribution.seed_group_id
+        if attribution.attempt_index is not None:
+            attribution_data["attempt_index"] = attribution.attempt_index
         result.attribution_data = attribution_data
 
     @staticmethod
@@ -588,6 +600,10 @@ class AttackStrategy(Strategy[AttackStrategyContextT, AttackStrategyResultT], Id
     #: No scoring configuration alone does not imply delegation.
     DELEGATES_SCORING: ClassVar[bool] = False
 
+    #: What this strategy's persisted results represent. Compound attacks that only coordinate
+    #: child attacks, and never call the objective target themselves, set ``ORCHESTRATION``.
+    RESULT_ROLE: ClassVar[AttackResultRole] = AttackResultRole.TARGET_FACING
+
     def __init_subclass__(cls, **kwargs: Any) -> None:
         """
         Enforce the keyword-only constructor contract on subclasses.
@@ -865,6 +881,7 @@ async def execute_with_context_async(self, *, context: AttackStrategyContextT) -
         self._validate_scoring_expectation(context=context)
         context._error_result_persistence_error = None
         context._attack_result_id = str(uuid.uuid4())
+        context._result_role = self.RESULT_ROLE
         lifecycle = _ObjectiveTargetConversationLifecycle(
             objective_target=self._objective_target,
             logger=self._logger,

pyrit/memory/memory_interface.py

@@ -6177,6 +6177,7 @@ def _execute_get_scenario_attack_result_deltas(
                 AttackResultEntry.error_type,
                 AttackResultEntry.error_message,
                 AttackResultEntry.attribution_data,
+                AttackResultEntry.attack_metadata,
                 ScoreEntry.id.label("score_id"),
                 ScoreEntry.score_value,
                 ScoreEntry.score_type,
@@ -6236,6 +6237,7 @@ def _execute_get_scenario_attack_result_deltas(
                     error_type=row.error_type,
                     error_message=row.error_message,
                     attribution_data=row.attribution_data or {},
+                    attack_metadata=row.attack_metadata or {},
                     score=score,
                 )
             )