Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -414,7 +414,11 @@ RealTimeParameters(
Use `pcm_s16le` (default) + `sample_rate=16000` for microphone capture.

`agent_context` (like `prompt`) can also be updated mid-stream after each agent
turn: `client.set_params(RealTimeSessionParameters(agent_context="..."))`.
turn: `client.set_params(RealTimeSessionParameters(agent_context="..."))`, or the
keyword form `client.update_configuration(agent_context="...")`. `mode` (including
`"manual"`) and `max_turn_duration` are updatable the same way.
`RealTimeSessionParameters` rejects unknown fields with a validation error;
`RealTimeParameters` (connect) drops them with a logged warning.

**Error codes**: `RealTimeErrorCodes` is a `dict[int, str]` mapping wire codes to human messages. Use `.get(...)` for lookup — not enum-style attribute access:
```python
Expand Down Expand Up @@ -459,7 +463,7 @@ async with AsyncRealTimeTranscriber(RealTimeTranscriberOptions(token=token_from_

**Other gotchas**:
- Don't feed `AsyncRealTimeTranscriber.stream()` from a sync generator that blocks (e.g. `time.sleep` pacing) — it starves the read task. Use an `async def` generator with `await asyncio.sleep(...)` instead.
- `format_turns=True` enables punctuation/casing on confirmed end-of-turns. Toggle mid-session via `client.set_params(RealTimeSessionParameters(format_turns=True))`.
- `format_turns=True` enables punctuation/casing on end-of-turns. Set it at connect: the server ignores `format_turns` and `filter_profanity` in a mid-session `UpdateConfiguration`, and `set_params` logs a warning when either is set.
- `AsyncRealTimeTranscriber` used as `async with` calls `disconnect(terminate=True)` on normal block exit and `disconnect(terminate=False)` on exception — no explicit `disconnect()` needed inside the block.

## Text-to-Speech
Expand Down
14 changes: 9 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2003,16 +2003,20 @@ Common codes: `4001` Not Authorized, `4002` Insufficient Funds, `4029` Client se
<details>
<summary>Change settings mid-session</summary>

`set_params` updates an active session. Typical use: enable turn formatting (punctuation, casing) only on confirmed end-of-turn so partial transcripts stay raw:
`set_params` updates an active session. Typical use: switch a voice agent to manual turn detection and raise the silence cap mid-session:

```python
from assemblyai.streaming.v3 import RealTimeSessionParameters
from assemblyai.streaming.v3 import RealTimeSessionParameters, StreamingMode

def on_turn(client, event):
if event.end_of_turn and not event.turn_is_formatted:
client.set_params(RealTimeSessionParameters(format_turns=True))
client.set_params(
RealTimeSessionParameters(mode=StreamingMode.manual, max_turn_silence=1500)
)
# Keyword form of the same call:
client.update_configuration(mode="manual", max_turn_silence=1500)
```

Updatable mid-session: `mode`, `max_turn_duration`, `min_turn_silence`, `max_turn_silence`, `end_of_turn_confidence_threshold`, `vad_threshold`, `turn_left_pad_ms`, `keyterms_prompt`, `prompt`, `agent_context`, `interruption_delay`, `language_codes`, `continuous_partials`, `session_heartbeat`, and `acknowledge_silence`. An unknown field raises a validation error. `format_turns` and `filter_profanity` take effect only at connect; `set_params` sends them but logs a warning, and the server ignores them.

For voice agents, `force_endpoint()` flushes the current turn — useful when an external signal (UI button, barge-in detection) determines the user has stopped speaking before VAD does:

```python
Expand Down
2 changes: 1 addition & 1 deletion assemblyai/__version__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "1.10.10"
__version__ = "1.10.11"
11 changes: 11 additions & 0 deletions assemblyai/streaming/v3/_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,17 @@ def _normalize_min_turn_silence(params_dict: dict) -> dict:
return params_dict


def _warn_connect_only_session_params(params_dict: dict) -> None:
"""Warn for fields `set_params` sends that the server applies only at
connect. The fields are still sent."""
for key in ("format_turns", "filter_profanity"):
if params_dict.get(key) is not None:
logger.warning(
f"`{key}` only takes effect at connect; the server ignores it "
"in a mid-session UpdateConfiguration."
)


def _normalize_voice_focus(params_dict: dict) -> dict:
"""Collapse `noise_suppression_model` / `noise_suppression_threshold` into
`voice_focus` / `voice_focus_threshold` so only the new wire keys are sent.
Expand Down
7 changes: 7 additions & 0 deletions assemblyai/streaming/v3/async_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
_normalize_min_turn_silence,
_resolve_options,
_user_agent,
_warn_connect_only_session_params,
)
from .models import (
ErrorEvent,
Expand Down Expand Up @@ -305,9 +306,15 @@ async def set_params(self, params: RealTimeSessionParameters) -> None:
if stop_event.is_set():
return
message_dict = _normalize_min_turn_silence(_dump_model(params))
_warn_connect_only_session_params(message_dict)
message = UpdateConfiguration(**message_dict)
await write_queue.put(message)

async def update_configuration(self, **kwargs: Any) -> None:
"""Keyword form of `set_params`: `update_configuration(mode="manual")`
equals `set_params(RealTimeSessionParameters(mode="manual"))`."""
await self.set_params(RealTimeSessionParameters(**kwargs))

async def force_endpoint(self) -> None:
write_queue, stop_event = self._ensure_connected("force_endpoint")
if stop_event.is_set():
Expand Down
7 changes: 7 additions & 0 deletions assemblyai/streaming/v3/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
_normalize_min_turn_silence,
_resolve_options,
_user_agent,
_warn_connect_only_session_params,
)
from ._tls import _create_tls_context
from .models import (
Expand Down Expand Up @@ -236,9 +237,15 @@ def stream(

def set_params(self, params: RealTimeSessionParameters):
message_dict = _normalize_min_turn_silence(_dump_model(params))
_warn_connect_only_session_params(message_dict)
message = UpdateConfiguration(**message_dict)
self._write_queue.put(message)

def update_configuration(self, **kwargs: Any) -> None:
"""Keyword form of `set_params`: `update_configuration(mode="manual")`
equals `set_params(RealTimeSessionParameters(mode="manual"))`."""
self.set_params(RealTimeSessionParameters(**kwargs))

def force_endpoint(self):
message = ForceEndpoint()
self._write_queue.put(message)
Expand Down
80 changes: 67 additions & 13 deletions assemblyai/streaming/v3/models.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import logging
from datetime import datetime
from enum import Enum
from typing import Any, List, Literal, Optional, Union

try:
# pydantic v2 import
from pydantic import BaseModel, Field, model_validator
from pydantic import BaseModel, ConfigDict, Field, model_validator

pydantic_v2 = True
except ImportError:
Expand All @@ -13,6 +14,8 @@

pydantic_v2 = False

logger = logging.getLogger(__name__)


class LLMGatewayMessage(BaseModel):
role: str
Expand Down Expand Up @@ -168,7 +171,34 @@ class KeepAlive(BaseModel):
type: Literal["KeepAlive"] = "KeepAlive"


class StreamingMode(str, Enum):
max_accuracy = "max_accuracy"
min_latency = "min_latency"
balanced = "balanced"
manual = "manual"

def __str__(self):
return self.value


class RealTimeSessionParameters(BaseModel):
"""Session settings that `set_params` sends mid-session as an
`UpdateConfiguration` message. `RealTimeParameters` extends this model with
the connect-only settings.

Unknown fields raise a validation error instead of being dropped.
`format_turns` and `filter_profanity` take effect only at connect; the
server ignores them mid-session.
"""

if pydantic_v2:
model_config = ConfigDict(extra="forbid")
else:

class Config:
extra = "forbid"

mode: Optional[StreamingMode] = None
end_of_turn_confidence_threshold: Optional[float] = None
min_end_of_turn_silence_when_confident: Optional[int] = (
None # Deprecated: Use min_turn_silence instead
Expand All @@ -183,8 +213,11 @@ class RealTimeSessionParameters(BaseModel):
agent_context: Optional[str] = None
interruption_delay: Optional[int] = None
turn_left_pad_ms: Optional[int] = None
# Server-validated range: 16000-128000 ms.
max_turn_duration: Optional[int] = None
language_codes: Optional[List[str]] = None
session_heartbeat: Optional[bool] = None
continuous_partials: Optional[bool] = None
# Opt in to Silence messages during silence (U3Pro only). When True, the
# server emits a Silence message roughly once per second while no speech is
# being transcribed, carrying `start_ms` and `end_ms`, the audio positions
Expand Down Expand Up @@ -247,16 +280,6 @@ def __str__(self):
return self.value


class StreamingMode(str, Enum):
max_accuracy = "max_accuracy"
min_latency = "min_latency"
balanced = "balanced"
manual = "manual"

def __str__(self):
return self.value


class StreamingPiiSubstitution(str, Enum):
hash = "hash"
entity_name = "entity_name"
Expand Down Expand Up @@ -336,7 +359,40 @@ def __str__(self):
return self.value


def _warn_unknown_connect_params(model_cls: Any, values: Any) -> None:
if not isinstance(values, dict):
return
fields = model_cls.model_fields if pydantic_v2 else model_cls.__fields__
for key in values:
if key not in fields:
logger.warning(
f"Ignoring unknown streaming connection parameter `{key}`; "
"it is not sent to the server."
)


class RealTimeParameters(RealTimeSessionParameters):
"""Connection parameters. Unknown fields are dropped with a warning."""

if pydantic_v2:
model_config = ConfigDict(extra="ignore")

@model_validator(mode="before")
@classmethod
def _warn_unknown_fields(cls, values: Any) -> Any:
_warn_unknown_connect_params(cls, values)
return values

else:

class Config:
extra = "ignore"

@root_validator(pre=True)
def _warn_unknown_fields(cls, values):
_warn_unknown_connect_params(cls, values)
return values

# Required for PCM encodings. May be omitted for Opus encodings
# (opus, ogg_opus) — the stream is self-describing and the server
# ignores the value.
Expand All @@ -361,13 +417,11 @@ class RealTimeParameters(RealTimeSessionParameters):
# Deprecated: use voice_focus / voice_focus_threshold instead.
noise_suppression_model: Optional[NoiseSuppressionModel] = None
noise_suppression_threshold: Optional[float] = None
continuous_partials: Optional[bool] = None
customer_support_audio_capture: Optional[bool] = None
include_partial_turns: Optional[bool] = None
redact_pii: Optional[bool] = None
redact_pii_policies: Optional[List[StreamingPiiPolicy]] = None
redact_pii_sub: Optional[StreamingPiiSubstitution] = None
mode: Optional[StreamingMode] = None

if pydantic_v2:

Expand Down
Loading
Loading