Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ Detects (never rewrites) injection signals and records them as evidence:
- **Obfuscation normalization** (for matching only): undoes Greek and Cyrillic look-alikes, leetspeak, words joined by `-` `_` `.`, spaced-out letters and control characters. Tier 2 scores the same normalized text; the returned content is never normalized
- **Role markers**: `SYSTEM:`, `ASSISTANT:`, `<system>`, `[INST]`, etc.
- **Instruction-override patterns**: phrases like "ignore previous instructions"
- **Encoding detection**: decode-then-detect: a decoded payload is escalated only if it trips a real attack pattern. Tier 2 scores the decoded text too
- **Encoding detection**: decode-then-detect in every string field, including UTF-8 text in base64 (an email body in `payload.body.data`, for example): a decoded payload is escalated only if it trips a real attack pattern. Tier 2 scores the decoded text too, and Tier 3 reads it in place of the encoded value. The returned content is never decoded
- **Boundary annotation (opt-in)**: `[UD-{id}]…[/UD-{id}]` wrappers when `annotate_boundary=True` (npm: `annotateBoundary`). Use `generate_boundary_instructions` from the package root in prompts when you enable wrapping.

When Tier 2 is on, Tier 1 can't block alone. Its detections count only when Tier 2's undamped score (`tier2_raw_score`) is at least 0.3, or when Tier 2 didn't read what they matched: an object key, a field longer than Tier 2's 10,000-character cap, text Tier 2 cut or set aside as unreadable, or a matched phrase that no single Tier 2 chunk holds whole (split across chunks or short lines, or made of hidden characters Tier 2 removes). When a phrase appears only after Tier 1's normalization, Tier 2 also scores that normalized copy. Otherwise `detections` still lists them, `tier1_gated` is `True` and Tier 1's risk is capped at `medium`. With Tier 2 off or unavailable, Tier 1 blocks alone. The 0.3 bar is set for the bundled model.
Expand Down
13 changes: 10 additions & 3 deletions src/stackone_defender/core/prompt_defense.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,13 @@
from ..utils.redact import describe_error, redact_error_text
from ..utils.text import phrase_key, truncate
from .sentence_cleaner import clean_high_risk_content
from .tier3_input import DEFAULT_MAX_CHUNKS, chunk_tier3_input, content_budget, format_records_for_tier3
from .tier3_input import (
DEFAULT_MAX_CHUNKS,
chunk_tier3_input,
content_budget,
decode_for_review,
format_records_for_tier3,
)
from .tier3_policy import MISSING_PROVIDER, Tier3Policy
from .tool_result_sanitizer import ToolResultSanitizer, create_tool_result_sanitizer

Expand Down Expand Up @@ -601,7 +607,8 @@ async def _maybe_tier3_cascade(
review = await self._review_whole_result(value, provider, tool_name, "cascade", review_flag, caller)
elif provider is not None:
t3.warn_if_risky_override(provider)
bounded = truncate(tier2.max_sentence, self._tier3_max_text_length)
# The chunk can be an encoded copy Tier 2 also scored; the reviewer reads it decoded.
bounded = truncate(decode_for_review(tier2.max_sentence), self._tier3_max_text_length)
outcome = await t3.call(provider, bounded, self._tier3_ctx(tool_name, "cascade", caller))
else:
if not self._tier3_missing_provider_warned:
Expand Down Expand Up @@ -1476,7 +1483,7 @@ def _evaluate_tier2(
origin: list[int] = []
for i in range(len(strings)):
origin.append(i)
head = strings[i][:DEFAULT_MAX_FIELD_ANALYSIS_LENGTH]
head = truncate(strings[i], DEFAULT_MAX_FIELD_ANALYSIS_LENGTH) # UTF-16 units, as in TS
decoded, levels = decode_all_levels(head)
if levels > 0 and decoded != head:
strings.append(decoded)
Expand Down
50 changes: 40 additions & 10 deletions src/stackone_defender/core/tier3_input.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,9 @@
from decimal import Decimal
from typing import Any

from ..config import MAX_TRAVERSAL_DEPTH
from ..utils.text import JS_WS_CHARS, from_utf16, safe_end, safe_start, to_utf16
from ..config import DEFAULT_MAX_FIELD_ANALYSIS_LENGTH, MAX_TRAVERSAL_DEPTH
from ..sanitizers.encoding_detector import decode_all_levels
from ..utils.text import JS_WS_CHARS, from_utf16, safe_end, safe_start, to_utf16, truncate
from ..utils.text import utf16_len as _utf16_len

# Line breaks (incl. U+2028/U+2029/NEL): flattened in keys, split in values, so neither can
Expand Down Expand Up @@ -310,15 +311,26 @@ def add(self, v: Any) -> bool:
return True


def _sum_string_content(v: Any, depth: int, meter: _Meter) -> int:
def decode_for_review(s: str) -> str:
"""A string as Tier 3 reads it: encoded spans (base64, hex, URL, entities..., chained) decoded, from
the same head Tier 1 and Tier 2 decode (UTF-16 units). ROT13 and ROT47 are left alone: they rewrite
the whole string, so plain text with one ROT13 word would reach the reviewer scrambled. Tier 1 and
Tier 2 still decode them. Only the reviewer's copy changes."""
head = truncate(s, DEFAULT_MAX_FIELD_ANALYSIS_LENGTH)
text, levels = decode_all_levels(head, 5, {"decode_rot13": False, "decode_rot47": False})
return text + s[len(head) :] if levels > 0 and text != head else s


def _sum_string_content(v: Any, depth: int, meter: _Meter, text: Callable[[str], str]) -> int:
"""String-leaf content as the serializer emits it (UTF-16 units, line breaks excluded)."""
if depth > MAX_TRAVERSAL_DEPTH:
return 0
v = _normalize(v, meter.limit)
if not meter.add(v):
return 0
if isinstance(v, str):
return _utf16_len(v) - len(_LINE_BREAKS.findall(v))
t = text(v)
return _utf16_len(t) - len(_LINE_BREAKS.findall(t))
if _is_array(v):
items = v
elif isinstance(v, dict):
Expand All @@ -327,15 +339,23 @@ def _sum_string_content(v: Any, depth: int, meter: _Meter) -> int:
return 0
total = 0
for x in items:
total += _sum_string_content(x, depth + 1, meter)
total += _sum_string_content(x, depth + 1, meter, text)
if meter.hit:
break
return total


class _Serializer:
def __init__(self, depth_flag: dict[str, Any], max_chars: int, max_size: int, drop_shape_only: bool = False):
def __init__(
self,
depth_flag: dict[str, Any],
max_chars: int,
max_size: int,
drop_shape_only: bool = False,
text: Callable[[str], str] = decode_for_review,
):
self.depth_flag = depth_flag
self.text = text
self.drop_shape_only = drop_shape_only
self.max_chars = max_chars
self.meter = _Meter(max_size)
Expand Down Expand Up @@ -444,7 +464,7 @@ def _children(

def _scalar(self, v: Any, prefix: str, lines: list[str], keyed: bool, under_key: bool) -> None:
is_str = isinstance(v, str)
s = to_utf16(v if is_str else _js_string(v))
s = to_utf16(self.text(v) if is_str else _js_string(v))
limit = self.cap if is_str else min(self.cap, self.non_string_cap)
if not prefix and not keyed:
# Top-level scalar record: bare lines, blank lines dropped.
Expand Down Expand Up @@ -550,14 +570,24 @@ def format_records_for_tier3(
value = _normalize(value, max_size)
top_is_array = _is_array(value)
records = list(value) if top_is_array else [value]
ser = _Serializer(depth_flag, max_chars, max_size, drop_shape_only)
# The reviewer reads encoded strings decoded (ENG-4342). Cached: the serializer may run twice
# and the coverage count reads every string again.
cache: dict[str, str] = {}

def text(s: str) -> str:
t = cache.get(s)
if t is None:
t = cache[s] = decode_for_review(s)
return t

ser = _Serializer(depth_flag, max_chars, max_size, drop_shape_only, text)
size_limit_hit = False

blocks = ser.run_records(records, top_is_array) # greedy
size_limit_hit |= ser.meter.hit
if ser.budget_truncated:
# Something didn't fit at full budget: redo with a per-sibling reserve and spread sampling.
fresh = _Serializer(depth_flag, max_chars, max_size, drop_shape_only)
fresh = _Serializer(depth_flag, max_chars, max_size, drop_shape_only, text)
fresh.reserve_mode = True
depth_flag.pop("coverage_degraded", None)
ser = fresh
Expand All @@ -567,7 +597,7 @@ def format_records_for_tier3(
meter = _Meter(max_size)
input_chars = 0
for r in records:
input_chars += _sum_string_content(r, 0, meter)
input_chars += _sum_string_content(r, 0, meter, text)
if meter.hit:
break
size_limit_hit |= meter.hit
Expand Down
108 changes: 74 additions & 34 deletions src/stackone_defender/core/tool_result_sanitizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
should_continue_traversal,
update_size_metrics,
)
from ..utils.text import phrase_key, to_utf16
from ..utils.text import phrase_key, to_utf16, truncate

# Risk levels in ascending order, for monotonic max comparison.
RISK_ORDER: list[RiskLevel] = ["low", "medium", "high", "critical"]
Expand Down Expand Up @@ -189,6 +189,7 @@ def _sanitize_value(
if self._is_field_risky(context.field_name, context.tool_name):
return self._sanitize_string_field(value, context, metadata, detect)
update_size_metrics(metadata.size_metrics, value)
self._scan_encoded_field(value, context, metadata, detect)
return value
update_size_metrics(metadata.size_metrics, value)
if not should_continue_traversal(metadata.size_metrics, depth, self._traversal.max_size, self._traversal.max_depth):
Expand Down Expand Up @@ -346,8 +347,9 @@ def _sanitize_string_field(

if self._use_tier1:
cap = self._max_field_analysis_length
analysis_value = value[:cap] if len(value) > cap else value
if len(value) > cap:
# The cap counts UTF-16 units, as in TS.
analysis_value = truncate(value, cap)
if analysis_value != value:
metadata.analysis_truncated = True

result = self._pattern_detector.analyze(analysis_value)
Expand All @@ -359,38 +361,76 @@ def _sanitize_string_field(
if context.cumulative_risk and result.matches:
self._update_cumulative_risk(context.cumulative_risk, result.suggested_risk, tier1_patterns)

# Evidence-driven encoding escalation: decode chained layers, then run
# the REAL pattern detector on the decoded text. Escalate only when the
# decoded content trips an actual attack pattern (a benign base64 body
# containing the word "ignore" is NOT a false positive).
decoded, levels = decode_all_levels(analysis_value)
if levels > 0 and decoded != analysis_value:
enc = self._pattern_detector.analyze(decoded)
if enc.has_detections and enc.matches:
enc_patterns = [m.pattern for m in enc.matches]
tier1_patterns = list(dict.fromkeys(tier1_patterns + enc_patterns))
escalated_from_encoding = True
self._record_tier1_match(metadata, decoded, enc.matches)
risk_level = _risk_ladder(risk_level, enc.suggested_risk)
if context.cumulative_risk:
self._update_cumulative_risk(context.cumulative_risk, enc.suggested_risk, enc_patterns)

# Fold this field's risk into the overall (monotonic; no-op for benign low).
self._raise_overall_risk(metadata, risk_level)

if tier1_patterns or escalated_from_encoding:
metadata.fields_sanitized.append(context.path)
methods: list = []
if tier1_patterns:
methods.append("pattern_removal")
if escalated_from_encoding:
methods.append("encoding_detection")
metadata.methods_by_field[context.path] = methods
if tier1_patterns:
metadata.patterns_removed_by_field[context.path] = tier1_patterns
enc = self._scan_decoded(analysis_value, context, metadata)
if enc is not None:
enc_patterns, enc_risk = enc
tier1_patterns = list(dict.fromkeys(tier1_patterns + enc_patterns))
escalated_from_encoding = True
risk_level = _risk_ladder(risk_level, enc_risk)

self._record_field_detections(metadata, context, risk_level, tier1_patterns, escalated_from_encoding)
return self._maybe_wrap(value, context)

def _scan_decoded(
self,
analysis_value: str,
context: SanitizationContext,
metadata: SanitizationMetadata,
cumulative: bool = True,
) -> tuple[list[str], RiskLevel] | None:
"""Decode an encoded value (base64, hex, ROT13..., chained) and run the pattern detector on the
decoded text. Evidence-driven: a decode counts only when it trips a real attack pattern, so a
benign base64 body that merely contains the word "ignore" isn't flagged."""
decoded, levels = decode_all_levels(analysis_value)
if levels == 0 or decoded == analysis_value:
return None
enc = self._pattern_detector.analyze(decoded)
if not enc.has_detections or not enc.matches:
return None
patterns = list(dict.fromkeys(m.pattern for m in enc.matches))
self._record_tier1_match(metadata, decoded, enc.matches)
if cumulative and context.cumulative_risk:
self._update_cumulative_risk(context.cumulative_risk, enc.suggested_risk, patterns)
return patterns, enc.suggested_risk

def _record_field_detections(
self,
metadata: SanitizationMetadata,
context: SanitizationContext,
risk_level: RiskLevel,
patterns: list[str],
from_encoding: bool,
) -> None:
"""Raise the overall risk and record which patterns and methods fired on this field."""
self._raise_overall_risk(metadata, risk_level)
if not patterns and not from_encoding:
return
metadata.fields_sanitized.append(context.path)
methods: list = []
if patterns:
methods.append("pattern_removal")
if from_encoding:
methods.append("encoding_detection")
metadata.methods_by_field[context.path] = methods
if patterns:
metadata.patterns_removed_by_field[context.path] = patterns

def _scan_encoded_field(
self, value: str, context: SanitizationContext, metadata: SanitizationMetadata, detect: bool
) -> None:
"""A field that isn't risky by name isn't pattern-scanned, but an encoded value in it is decoded
and scanned (ENG-4342): an attack can hide in any field as base64, e.g. a Gmail body in
``payload.body.data``."""
if not detect or not self._use_tier1:
return
cap = self._max_field_analysis_length
# Only this field escalates: these fields aren't counted as processed in the cumulative tracker,
# so feeding it their matches would skew its fractions.
enc = self._scan_decoded(truncate(value, cap), context, metadata, cumulative=False)
if enc is not None:
patterns, risk = enc
self._record_field_detections(metadata, context, _risk_ladder(context.risk_level, risk), patterns, True)

@staticmethod
def _record_tier1_match(metadata: SanitizationMetadata, text: str, matches: list[PatternMatch]) -> None:
"""Record what Tier 1 matched, for the Tier 2 gate. Tier 1's normalised copy (leet, split or
Expand All @@ -416,8 +456,8 @@ def _detect_in_key(self, key: str, key_path: str, context: SanitizationContext,
if not self._use_tier1 or len(key) < 3:
return
cap = self._max_field_analysis_length
analysis_key = key[:cap] if len(key) > cap else key
if len(key) > cap:
analysis_key = truncate(key, cap)
if analysis_key != key:
metadata.analysis_truncated = True
result = self._pattern_detector.analyze(analysis_key)
if not result.has_detections or not result.matches:
Expand Down
35 changes: 23 additions & 12 deletions src/stackone_defender/sanitizers/encoding_detector.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,28 +98,37 @@ def detect_encoding(
return result


def _b64decode_url_safe(candidate: str) -> str | None:
"""Decode standard or URL-safe base64; None when it isn't valid base64."""
_NOT_TEXT = object() # valid base64 of bytes that aren't UTF-8 text


def _b64decode_url_safe(candidate: str) -> str | object | None:
"""Decode standard or URL-safe base64 to UTF-8 text. ``_NOT_TEXT`` when it is valid base64 but not
valid UTF-8 (binary data); None when it isn't valid base64."""
try:
# Pad to a multiple of 4; b64decode rejects unpadded input (JS atob tolerates it).
unpadded = candidate.replace("-", "+").replace("_", "/").rstrip("=")
decoded_bytes = base64.b64decode(unpadded + "=" * (-len(unpadded) % 4), validate=False)
except Exception:
return None
try:
return decoded_bytes.decode("ascii")
# Strict UTF-8; a leading BOM is kept, as in TS (TextDecoder with ignoreBOM). Control
# characters other than JavaScript's \s ones become spaces, so a NUL appended to an encoded
# instruction can't keep it encoded.
return _CONTROL_CHARS.sub(" ", decoded_bytes.decode("utf-8"))
except UnicodeDecodeError:
return decoded_bytes.decode("latin-1", errors="replace")
return _NOT_TEXT


# Whitespace JavaScript's \s matches, so decodes are kept exactly as in the TS package.
_JS_WHITESPACE = frozenset(
"\t\n\v\f\r \u00a0\u1680\u2028\u2029\u202f\u205f\u3000\ufeff" + "".join(chr(c) for c in range(0x2000, 0x200B))
)
# Control characters other than JavaScript's \s ones (tab, LF, VT, FF, CR), as in TS.
_CONTROL_CHARS = re.compile("[\x00-\x08\x0e-\x1f\x7f-\x9f]")
# JavaScript's \S: anything but its whitespace set.
_JS_NON_SPACE = re.compile("[^\t\n\x0b\x0c\r \xa0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]")


def _is_printable_text(s: str | None) -> bool:
return bool(s) and all(0x20 <= ord(c) <= 0x7E or c in _JS_WHITESPACE for c in s)
def _is_printable_text(s: object) -> bool:
"""Text, not binary: valid UTF-8 (``_b64decode_url_safe`` returns ``_NOT_TEXT`` otherwise) with
something other than whitespace in it, as in TS. Any Unicode character counts."""
return isinstance(s, str) and _JS_NON_SPACE.search(s) is not None
Comment thread
hiskudin marked this conversation as resolved.
Resolved


# Retries per candidate when a word joined by - or _ makes the whole run undecodable.
Expand Down Expand Up @@ -589,7 +598,7 @@ def decode_all_encoding(text: str) -> str:
return result.processed_text or text


def decode_all_levels(text: str, max_iterations: int = 5) -> tuple[str, int]:
def decode_all_levels(text: str, max_iterations: int = 5, options: dict[str, bool] | None = None) -> tuple[str, int]:
"""Iteratively decode chained encodings until the output stabilises.

A single call to ``decode_all_encoding`` only unwraps one layer. Chained
Expand All @@ -602,6 +611,8 @@ def decode_all_levels(text: str, max_iterations: int = 5) -> tuple[str, int]:
- Aborts if decoded text exceeds 10x the original length (decompression
bomb protection).

``options`` passes detector flags (e.g. ``{"decode_rot13": False}``) to every pass.

Returns ``(decoded_text, levels)`` where ``levels`` is the number of
decode passes applied (0 if the input had no detected encodings).
"""
Expand All @@ -612,7 +623,7 @@ def decode_all_levels(text: str, max_iterations: int = 5) -> tuple[str, int]:
current = text
levels = 0
for _ in range(max_iterations):
result = detect_encoding(current, action="decode")
result = detect_encoding(current, action="decode", **(options or {}))
if not result.processed_text or result.processed_text == current:
break
if len(result.processed_text) > max_length:
Expand Down
Loading
Loading