diff --git a/CHANGELOG.md b/CHANGELOG.md index 059466b..ad701ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [4.44.0] + +- Add the Diarization API: `client.diarization.diarize({ audio, transcript?, config?, contentType? }, { timeout?, signal? })` posts audio, and optionally a transcript with word timings in ms, to `/v1/diarize` and returns `DiarizationResponse` (`words` and `utterances` with speaker labels, `segments`, `speakers`, `audio_duration_ms`, `session_id`, `request_time_ms`). A `SyncTranscriptResponse` can be passed as the transcript; without one only `segments` and `speakers` are filled. The audio content type is inferred from the file extension (`audio/wav`, `audio/mpeg`, `audio/flac`, `audio/ogg`, `audio/mp4`, `audio/webm`, raw `audio/pcm` with `config.sample_rate` and `channels`). Failures throw `DiarizationError` (`status`, `errorCode`, `retryAfter`). New client option `diarizationBaseUrl` (default `https://diarization.assemblyai.com`; `https://diarization.us.assemblyai.com` and `https://diarization.eu.assemblyai.com` for the data zones) + ## [4.43.4] - Add `assemblyai-stt-3-7-realtime` (note taking) and `assemblyai-stt-3-7-realtime-agent` (voice agents) to the streaming `speechModel` options (`StreamingSpeechModel`) diff --git a/CLAUDE.md b/CLAUDE.md index df00c04..5efd223 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,6 +48,7 @@ const client = new AssemblyAI({ - `client.dictation.transcribeLive(audio, config?, options?)` — Dictation: upload a short spoken note as it is spoken and get one finished transcript back, optionally rewritten by an LLM - `client.dictation.openLive(config?, options?)` — Open a dictation upload that audio is pushed into (`write()` / `close()` / `result()`), for callback-driven sources - `client.dictation.warm()` — Pre-open the connection to the dictation API so the next request skips the handshake +- `client.diarization.diarize({ audio, transcript?, config?, contentType? }, { timeout?, signal? })` — Speaker labels for audio: with a transcript (e.g. a sync response requested with `timestamps: true`) every word gets a speaker and `utterances` are filled; without one only `segments` and `speakers` come back. Throws `DiarizationError`. Host: `diarizationBaseUrl` (`https://diarization.assemblyai.com`; data zones `https://diarization.us.assemblyai.com`, `https://diarization.eu.assemblyai.com`) - `client.streaming.transcriber(params)` — Create a real-time streaming session - `client.llmGateway.chatCompletions(request)` — OpenAI-compatible chat completions - `client.llmGateway.listModels()` — List models available on the LLM Gateway diff --git a/README.md b/README.md index 8942e4a..1220cb8 100644 --- a/README.md +++ b/README.md @@ -757,6 +757,57 @@ can be correlated with the request. +### Diarize speakers + +`client.diarization` labels who spoke when. Post the audio together with a +transcript that has word timings, such as a sync transcript requested with +`timestamps: true`, and every word comes back with a speaker (`A`, `B`, ...), +grouped into utterances. Requests go to `diarizationBaseUrl`, +`https://diarization.assemblyai.com` by default; pass +`https://diarization.us.assemblyai.com` or +`https://diarization.eu.assemblyai.com` to keep them in the US or EU data zone. + +```typescript +const transcript = await client.sync.transcribe("./call.wav", { + timestamps: true, +}); +const { utterances } = await client.diarization.diarize({ + audio: "./call.wav", + transcript, // only `words` is sent +}); +for (const u of utterances) console.log(`${u.speaker}: ${u.text}`); +``` + +Leave out `transcript` to diarize the audio alone, for example in parallel with +transcription: `words` and `utterances` come back empty, and `segments` (speaker +turns with `start`/`end` in ms) and `speakers` are filled. + +
+Audio formats, config and errors + +`audio` takes a local file path, a data URL, raw bytes, a Blob/File or a +readable stream (read whole before sending), but not a URL. Its content type +picks the decoder and is inferred from the file extension: `audio/wav` (also +the default), `audio/mpeg`, `audio/flac`, `audio/ogg`, `audio/mp4`, +`audio/webm`, or raw `audio/pcm`, which needs `sample_rate` and `channels` in +`config`. Pass `contentType` to override it. + +```typescript +const result = await client.diarization.diarize( + { + audio: pcmBytes, + config: { sample_rate: 16_000, channels: 1, language_code: "en" }, + }, + { timeout: 120_000 }, +); +``` + +Failures throw `DiarizationError` with `.status` (401 for a missing or invalid +key, 400 for a validation failure), `.errorCode` (the snake_cased +problem-details title, e.g. `unauthorized`) and `.retryAfter`. + +
+ ### Transcribe streaming audio Refer to [AssemblyAI's streaming documentation](https://www.assemblyai.com/docs/streaming/getting-started/transcribe-streaming-audio) for full code examples. diff --git a/package.json b/package.json index 392e231..ab2fae2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "assemblyai", - "version": "4.43.4", + "version": "4.44.0", "description": "The AssemblyAI JavaScript SDK provides an easy-to-use interface for interacting with the AssemblyAI API, which supports async and real-time transcription and text-to-speech.", "engines": { "node": ">=18" diff --git a/src/services/diarization/index.ts b/src/services/diarization/index.ts new file mode 100644 index 0000000..6261f89 --- /dev/null +++ b/src/services/diarization/index.ts @@ -0,0 +1 @@ +export * from "./service"; diff --git a/src/services/diarization/service.ts b/src/services/diarization/service.ts new file mode 100644 index 0000000..de8116e --- /dev/null +++ b/src/services/diarization/service.ts @@ -0,0 +1,180 @@ +import { readFile } from "#fs"; +import { BaseService } from "../base"; +import { + BaseServiceParams, + DiarizationAudioInput, + DiarizationRequest, + DiarizationRequestOptions, + DiarizationResponse, +} from "../.."; +import { DiarizationError } from "../../utils/errors/diarization"; +import { getPath } from "../../utils/path"; +import { + basename, + concatChunks, + dataUrlToBytes, + deadlineSignal, + describe, + errorFromResponse, + extname, + isIterable, + isWebReadableStream, + liveChunks, + multipartBoundary, + multipartClosing, + multipartHead, + readAsyncIterable, + resolveFormat, +} from "../../utils/live"; + +const diarizeEndpoint = "/v1/diarize"; +// The extension picks the content type, which picks the server's decoder. +// `.pcm`/`.raw` map to `audio/pcm` inside resolveFormat; unknown extensions +// fall back to `audio/wav`. +const contentTypes: Record = { + ".wav": "audio/wav", + ".mp3": "audio/mpeg", + ".flac": "audio/flac", + ".ogg": "audio/ogg", + ".opus": "audio/ogg", + ".m4a": "audio/mp4", + ".mp4": "audio/mp4", + ".webm": "audio/webm", +}; +const encoder = new TextEncoder(); + +/** + * The Diarization API: audio in, speaker labels out. + * + * Posts the audio, and optionally a transcript with word timings, to the + * client's `diarizationBaseUrl` (`https://diarization.assemblyai.com` by + * default; `https://diarization.us.assemblyai.com` and + * `https://diarization.eu.assemblyai.com` keep requests in the US or EU). + * + * @example + * ```ts + * const transcript = await client.sync.transcribe("./call.wav", { timestamps: true }); + * const { utterances } = await client.diarization.diarize({ audio: "./call.wav", transcript }); + * ``` + */ +export class DiarizationService extends BaseService { + constructor(params: BaseServiceParams) { + super(params); + } + + /** + * Label the speakers in `request.audio`. + * + * With `request.transcript`, every word gets a speaker and the response + * groups them into utterances. Without it, only the model runs: + * `segments` and `speakers` are filled and `words` and `utterances` are + * empty. + * @param request - The audio, and optionally the transcript, config and + * audio content type. + * @param options - Client-side options: a deadline and an abort signal. + * @returns A promise that resolves to the speaker labels. + * @throws Error when `audio` is a URL, or when raw PCM is missing + * `sample_rate` or `channels`. + * @throws DiarizationError when the API refuses the request. + */ + async diarize( + request: DiarizationRequest, + options: DiarizationRequestOptions = {}, + ): Promise { + const config = request.config ?? {}; + const { bytes, filename, suffix } = await readAudio(request.audio); + const format = request.contentType + ? { filename: filename ?? "audio", contentType: request.contentType } + : resolveFormat(config, suffix, filename, contentTypes, "config"); + if ( + format.contentType === "audio/pcm" && + (config.sample_rate === undefined || config.channels === undefined) + ) { + throw new Error( + "raw PCM audio requires both sample_rate and channels in config", + ); + } + + const boundary = multipartBoundary(); + const transcriptPart = request.transcript + ? `--${boundary}\r\n` + + 'Content-Disposition: form-data; name="transcript"\r\n' + + "Content-Type: application/json\r\n\r\n" + + // Only `words` is sent: a sync response carries much else. + JSON.stringify({ words: request.transcript.words }) + + "\r\n" + : ""; + // One buffered body rather than a stream: the API requires + // Content-Length (411 otherwise). + const body = concatChunks([ + encoder.encode(transcriptPart), + multipartHead({ boundary, config, ...format }), + bytes, + multipartClosing(boundary), + ]); + + const response = await this.fetchResponse(diarizeEndpoint, { + method: "POST", + body, + headers: { "Content-Type": `multipart/form-data; boundary=${boundary}` }, + signal: + options.timeout !== undefined + ? deadlineSignal(options.timeout, options.signal) + : options.signal, + }); + if (response.status !== 200) { + throw await errorFromResponse(response, DiarizationError, "diarization"); + } + return (await response.json()) as DiarizationResponse; + } +} + +/** + * Read any accepted audio input into bytes, with the file name and extension + * it carries, if any. URLs are rejected: the API has no URL ingestion. + */ +async function readAudio( + input: DiarizationAudioInput, +): Promise<{ bytes: Uint8Array; filename?: string; suffix: string }> { + if (typeof input === "string") { + if (/^https?:\/\//i.test(input)) { + throw new Error( + "DiarizationService does not accept URLs. Pass a local file path, " + + "audio bytes, a Blob or a stream.", + ); + } + if (input.startsWith("data:")) { + return { bytes: dataUrlToBytes(input), suffix: "" }; + } + const path = getPath(input) ?? input; + return named( + await readAsyncIterable(liveChunks(await readFile(path))), + path, + ); + } + if (input instanceof Uint8Array) return { bytes: input, suffix: "" }; + if (input instanceof ArrayBuffer) { + return { bytes: new Uint8Array(input), suffix: "" }; + } + if (input instanceof Blob) { + // File instances carry a name; the File global itself needs Node >= 20. + return named( + new Uint8Array(await input.arrayBuffer()), + (input as { name?: string }).name, + ); + } + if (isWebReadableStream(input) || isIterable(input)) { + // fs.ReadStream carries the path it was opened from. + const path = (input as { path?: unknown }).path; + return named( + await readAsyncIterable(liveChunks(input)), + typeof path === "string" ? path : undefined, + ); + } + throw new TypeError(`unsupported audio input type: ${describe(input)}`); +} + +function named(bytes: Uint8Array, path: string | undefined) { + const filename = path ? basename(path) : undefined; + return { bytes, filename, suffix: filename ? extname(filename) : "" }; +} diff --git a/src/services/index.ts b/src/services/index.ts index b69813b..606ec88 100644 --- a/src/services/index.ts +++ b/src/services/index.ts @@ -1,6 +1,7 @@ import { BaseServiceParams } from ".."; import { SyncTranscriber, SyncLiveSession } from "./sync"; import { DictationTranscriber, DictationLiveSession } from "./dictation"; +import { DiarizationService } from "./diarization"; import { LlmGatewayService } from "./llm-gateway"; import { TextToSpeechService, @@ -11,6 +12,7 @@ import { TextToSpeechError } from "../utils/errors/tts"; import { SyncTranscriptError, DictationError, + DiarizationError, LlmGatewayError, } from "../utils/errors"; import { @@ -38,6 +40,7 @@ const defaultBaseUrl = "https://api.assemblyai.com"; const defaultStreamingUrl = "https://streaming.assemblyai.com"; const defaultSyncUrl = "https://sync.assemblyai.com"; const defaultDictationUrl = "https://dictation.assemblyai.com"; +const defaultDiarizationUrl = "https://diarization.assemblyai.com"; const defaultLlmGatewayUrl = "https://llm-gateway.assemblyai.com"; const defaultTtsUrl = "https://tts.us.assemblyai.com"; @@ -72,6 +75,11 @@ class AssemblyAI { */ public dictation: DictationTranscriber; + /** + * The Diarization service: speaker labels for audio, with or without a transcript. + */ + public diarization: DiarizationService; + /** * The LLM Gateway service. */ @@ -119,6 +127,15 @@ class AssemblyAI { baseUrl: dictationBaseUrl, }); + let diarizationBaseUrl = params.diarizationBaseUrl || defaultDiarizationUrl; + if (diarizationBaseUrl.endsWith("/")) { + diarizationBaseUrl = diarizationBaseUrl.slice(0, -1); + } + this.diarization = new DiarizationService({ + ...params, + baseUrl: diarizationBaseUrl, + }); + let llmGatewayBaseUrl = params.llmGatewayBaseUrl || defaultLlmGatewayUrl; if (llmGatewayBaseUrl.endsWith("/")) { llmGatewayBaseUrl = llmGatewayBaseUrl.slice(0, -1); @@ -147,6 +164,8 @@ export { DictationTranscriber, DictationLiveSession, DictationError, + DiarizationService, + DiarizationError, LlmGatewayService, LlmGatewayError, TextToSpeechService, diff --git a/src/types/diarization/index.ts b/src/types/diarization/index.ts new file mode 100644 index 0000000..ac1519a --- /dev/null +++ b/src/types/diarization/index.ts @@ -0,0 +1,170 @@ +/** + * Audio for a diarization request: a local file path or data URL (file + * system access requires Node.js, Bun, or Deno), raw audio bytes, a + * Blob/File, or a readable stream. The audio is read whole before the request + * is sent, since the API needs its length up front. + * + * URLs are not accepted — the diarization API has no URL ingestion. + */ +export type DiarizationAudioInput = + | string + | Uint8Array + | ArrayBuffer + | Blob + | ReadableStream + | NodeJS.ReadableStream; + +/** + * One word of the transcript to label. Times are in milliseconds. + * + * A `SyncWord` from `client.sync.transcribe()` fits as is; request the sync + * transcript with `timestamps: true` so its words carry timings. + */ +export type DiarizationTranscriptWord = { + /** The text of the word. */ + text: string; + /** + * The start time of the word in milliseconds. A word without timings takes + * them from the words next to it; if no word has timings the request fails. + */ + start?: number; + /** The end time of the word in milliseconds. */ + end?: number; + /** The confidence score of the word, in the range 0-1. */ + confidence?: number; +}; + +/** + * The transcript whose words get speaker labels. Only `words` is sent, so a + * `SyncTranscriptResponse` from `client.sync.transcribe()` can be passed + * straight through. + */ +export type DiarizationTranscript = { + /** + * The words to label. An empty list skips the model and returns empty + * lists. + */ + words: DiarizationTranscriptWord[]; +}; + +/** + * Options for a diarization request. These are the only fields the API + * accepts; any other field is rejected with a 400. + */ +export type DiarizationConfig = { + /** + * The source sample rate in Hz. Setting either this or `channels` marks + * the audio as raw 16-bit PCM, and both are then required. Leave both + * unset for WAV and compressed formats, which carry them in their headers. + */ + sample_rate?: number; + /** + * The channel count. Setting either this or `sample_rate` marks the audio + * as raw 16-bit PCM, and both are then required. + */ + channels?: number; + /** + * A bare language code such as `"en"` or `"es"` (not `"en-US"`). It + * decides whether the words of one sentence are voted onto one speaker. + * Defaults to `"en"`; unused without a transcript. + */ + language_code?: string; +}; + +/** + * A diarization request. + */ +export type DiarizationRequest = { + /** The audio to diarize. */ + audio: DiarizationAudioInput; + /** + * The transcript to label, with word timings in milliseconds. Leave it out + * to diarize the audio alone: the response then has empty `words` and + * `utterances`, and `segments` and `speakers` filled. + */ + transcript?: DiarizationTranscript; + /** Options for the request. */ + config?: DiarizationConfig; + /** + * The audio part's content type, which picks the server's decoder: + * `audio/wav`, `audio/pcm` (needs `config.sample_rate` and + * `config.channels`), `audio/mpeg`, `audio/flac`, `audio/ogg`, `audio/mp4` + * or `audio/webm`. Inferred from the file extension when unset, and + * `audio/wav` when there is none. + */ + contentType?: string; +}; + +/** + * Client-side options for a diarization request. These are not sent to the + * server. + */ +export type DiarizationRequestOptions = { + /** + * The request deadline in milliseconds, spanning the upload and the + * diarization. No deadline when unset. + */ + timeout?: number; + /** An `AbortSignal` that drops the request when aborted. */ + signal?: AbortSignal; +}; + +/** A transcript word with its speaker. Times are in milliseconds. */ +export type DiarizationWord = { + text: string; + start: number; + end: number; + confidence?: number; + /** The speaker label: `A`, `B`, and so on. */ + speaker: string; + /** The calibrated confidence of the speaker label; absent when unavailable. */ + speaker_confidence?: number; +}; + +/** A run of consecutive words by one speaker. */ +export type DiarizationUtterance = { + speaker: string; + start: number; + end: number; + /** The words' text, joined. */ + text: string; + /** The mean word confidence. */ + confidence: number; + /** The mean word `speaker_confidence`; absent unless every word has one. */ + speaker_confidence?: number; + words: DiarizationWord[]; +}; + +/** One of the model's speaker turns. Times are in milliseconds. */ +export type DiarizationSegment = { + start: number; + end: number; + speaker: string; + /** The segment's calibrated confidence; absent when unavailable. */ + speaker_confidence?: number; +}; + +/** + * The result of a diarization request. + * + * A label means the same speaker in `words` and `segments`. `speakers` lists + * only the speakers words went to (all speakers, in order of first + * appearance, without a transcript), so a segment can carry a label that is + * not in it. + */ +export type DiarizationResponse = { + /** The transcript's words with speakers; empty without a transcript. */ + words: DiarizationWord[]; + /** Runs of consecutive words by one speaker; empty without a transcript. */ + utterances: DiarizationUtterance[]; + /** The speaker labels, sorted. */ + speakers: string[]; + /** The model's speaker turns, sorted by start. */ + segments: DiarizationSegment[]; + /** The audio duration in milliseconds. */ + audio_duration_ms: number; + /** The server-generated id for this request; record it for support. */ + session_id: string; + /** The server-side request time in milliseconds. */ + request_time_ms: number; +}; diff --git a/src/types/index.ts b/src/types/index.ts index de0373f..26ea611 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -1,5 +1,6 @@ export * from "./sync"; export * from "./dictation"; +export * from "./diarization"; export * from "./llm-gateway"; export * from "./files"; export * from "./transcripts"; diff --git a/src/types/services/index.ts b/src/types/services/index.ts index 3e76c69..49e9321 100644 --- a/src/types/services/index.ts +++ b/src/types/services/index.ts @@ -6,6 +6,8 @@ type BaseServiceParams = { streamingBaseUrl?: string; syncBaseUrl?: string; dictationBaseUrl?: string; + /** The Diarization API base URL. Defaults to `https://diarization.assemblyai.com`; `https://diarization.us.assemblyai.com` and `https://diarization.eu.assemblyai.com` keep requests in the US or EU. */ + diarizationBaseUrl?: string; llmGatewayBaseUrl?: string; /** The Text-to-Speech API base URL. Defaults to `https://tts.us.assemblyai.com`; `https://tts.eu.assemblyai.com` keeps requests in the EU. */ ttsBaseUrl?: string; diff --git a/src/utils/errors/diarization.ts b/src/utils/errors/diarization.ts new file mode 100644 index 0000000..2254445 --- /dev/null +++ b/src/utils/errors/diarization.ts @@ -0,0 +1,27 @@ +/** + * Error thrown when a diarization request fails. + */ +export class DiarizationError extends Error { + override name = "DiarizationError"; + + /** + * Create a new DiarizationError. + * @param message - The human-readable error message: the problem-details + * `detail` from the server. + * @param status - The HTTP status code of the failed request (401 for a + * missing or invalid key, 400 for a validation failure, 413, 415, 5xx). + * @param errorCode - Machine-readable code — the snake_cased + * problem-details `title` from the server (e.g. `unauthorized`, + * `bad_request`). + * @param retryAfter - Seconds to wait before retrying, from the + * `Retry-After` header on 429/503 responses. + */ + constructor( + message: string, + public readonly status?: number, + public readonly errorCode?: string, + public readonly retryAfter?: number, + ) { + super(message); + } +} diff --git a/src/utils/errors/index.ts b/src/utils/errors/index.ts index 757de3b..5d90187 100644 --- a/src/utils/errors/index.ts +++ b/src/utils/errors/index.ts @@ -1,5 +1,6 @@ export { SyncTranscriptError } from "./sync"; export { DictationError } from "./dictation"; +export { DiarizationError } from "./diarization"; export { LlmGatewayError } from "./llm-gateway"; export { TextToSpeechError } from "./tts"; export type { TextToSpeechErrorFields } from "./tts"; diff --git a/tests/unit/diarization.test.ts b/tests/unit/diarization.test.ts new file mode 100644 index 0000000..1b5f62d --- /dev/null +++ b/tests/unit/diarization.test.ts @@ -0,0 +1,328 @@ +import { mkdtempSync, rmSync, writeFileSync } from "fs"; +import fetchMock from "jest-fetch-mock"; +import { tmpdir } from "os"; +import path from "path"; +import { Readable } from "stream"; +import { + AssemblyAI, + DiarizationError, + DiarizationResponse, + SyncTranscriptResponse, +} from "../../src"; +import { defaultApiKey, defaultBaseUrl, requestMatches } from "./utils"; + +fetchMock.enableMocks(); + +const assembly = new AssemblyAI({ + baseUrl: defaultBaseUrl, + diarizationBaseUrl: defaultBaseUrl, + apiKey: defaultApiKey, +}); + +const diarizeUrl = "/v1/diarize"; + +const segmentsOnly: DiarizationResponse = { + words: [], + utterances: [], + segments: [ + { start: 0, end: 4200, speaker: "A", speaker_confidence: 0.93 }, + { start: 4350, end: 9100, speaker: "B", speaker_confidence: 0.88 }, + ], + speakers: ["A", "B"], + audio_duration_ms: 9500, + session_id: "eb92c4ff-4bbb-429f-9b99-7279d7fe738f", + request_time_ms: 312.4, +}; + +const labelled: DiarizationResponse = { + ...segmentsOnly, + words: [ + { text: "hi", start: 0, end: 300, confidence: 0.9, speaker: "A" }, + { text: "hey", start: 4400, end: 4700, speaker: "B" }, + ], + utterances: [ + { + speaker: "A", + start: 0, + end: 300, + text: "hi", + confidence: 0.9, + words: [{ text: "hi", start: 0, end: 300, speaker: "A" }], + }, + ], +}; + +function mockResponse(body: object, status = 200, headers = {}) { + fetchMock.doMockOnceIf( + requestMatches({ url: diarizeUrl, method: "POST" }), + JSON.stringify(body), + { status, headers }, + ); +} + +function requestInit(): RequestInit { + return fetchMock.mock.calls[0][1] as RequestInit; +} + +function requestHeaders(): Record { + return requestInit().headers as Record; +} + +type Part = { name: string; filename?: string; type?: string; body: string }; + +/** Parses the buffered multipart request body into its parts, in order. */ +function requestParts(): Part[] { + const body = requestInit().body; + expect(body).toBeInstanceOf(Uint8Array); + const boundary = /boundary=([^;]+)/.exec( + requestHeaders()["Content-Type"], + )![1]; + const text = new TextDecoder("latin1").decode(body as Uint8Array); + const parts: Part[] = []; + for (const raw of text.split(`--${boundary}`)) { + if (raw === "" || raw.startsWith("--")) continue; + const [headerText, ...rest] = raw.replace(/^\r\n/, "").split("\r\n\r\n"); + parts.push({ + name: /name="([^"]*)"/.exec(headerText)![1], + filename: /filename="([^"]*)"/.exec(headerText)?.[1], + type: /Content-Type: ([^\r\n]+)/.exec(headerText)?.[1], + body: rest.join("\r\n\r\n").replace(/\r\n$/, ""), + }); + } + return parts; +} + +function part(name: string): Part | undefined { + return requestParts().find((p) => p.name === name); +} + +const bytes = (s: string) => new TextEncoder().encode(s); + +beforeEach(() => { + fetchMock.resetMocks(); + fetchMock.doMock(); +}); + +describe("diarization", () => { + it("should diarize audio alone, without a transcript part", async () => { + mockResponse(segmentsOnly); + const result = await assembly.diarization.diarize({ + audio: bytes("RIFFfake"), + }); + + expect(result).toEqual(segmentsOnly); + expect(fetchMock.mock.calls[0][0]).toBe(defaultBaseUrl + diarizeUrl); + expect(requestInit().method).toBe("POST"); + expect(requestHeaders()["Authorization"]).toBe(defaultApiKey); + expect(requestHeaders()["Content-Type"]).toMatch( + /^multipart\/form-data; boundary=/, + ); + expect(requestParts().map((p) => p.name)).toEqual(["config", "audio"]); + expect(JSON.parse(part("config")!.body)).toEqual({}); + expect(part("audio")).toMatchObject({ + filename: "audio.wav", + type: "audio/wav", + body: "RIFFfake", + }); + }); + + it("should send the transcript's words as a JSON part", async () => { + mockResponse(labelled); + const transcript = { + words: [ + { text: "hi", start: 0, end: 300, confidence: 0.9 }, + { text: "hey", start: 4400, end: 4700 }, + ], + }; + const result = await assembly.diarization.diarize({ + audio: bytes("RIFFfake"), + transcript, + config: { language_code: "es" }, + }); + + expect(result.utterances[0].speaker).toBe("A"); + expect(requestParts().map((p) => p.name)).toEqual([ + "transcript", + "config", + "audio", + ]); + expect(part("transcript")!.type).toBe("application/json"); + expect(JSON.parse(part("transcript")!.body)).toEqual(transcript); + expect(JSON.parse(part("config")!.body)).toEqual({ language_code: "es" }); + }); + + it("should send an explicit empty words list", async () => { + mockResponse(segmentsOnly); + await assembly.diarization.diarize({ + audio: bytes("RIFF"), + transcript: { words: [] }, + }); + expect(JSON.parse(part("transcript")!.body)).toEqual({ words: [] }); + }); + + it("should take a sync transcription response as the transcript", async () => { + mockResponse(labelled); + const sync: SyncTranscriptResponse = { + text: "hi hey", + words: [ + { text: "hi", start: 0, end: 300, confidence: 0.9 }, + { text: "hey", start: 4400, end: 4700, confidence: 0.8 }, + ], + confidence: 0.85, + audio_duration_ms: 9500, + session_id: "s", + request_time_ms: 100, + }; + await assembly.diarization.diarize({ + audio: bytes("RIFF"), + transcript: sync, + }); + expect(JSON.parse(part("transcript")!.body)).toEqual({ + words: sync.words, + }); + }); + + it("should send raw PCM as audio/pcm with its config", async () => { + mockResponse(segmentsOnly); + await assembly.diarization.diarize({ + audio: bytes("\x00\x01"), + config: { sample_rate: 16000, channels: 1 }, + }); + expect(JSON.parse(part("config")!.body)).toEqual({ + sample_rate: 16000, + channels: 1, + }); + expect(part("audio")).toMatchObject({ + filename: "audio.pcm", + type: "audio/pcm", + }); + }); + + it("should reject raw PCM missing a field before sending", async () => { + await expect( + assembly.diarization.diarize({ + audio: bytes("\x00"), + config: { sample_rate: 16000 }, + }), + ).rejects.toThrow("requires both sample_rate and channels"); + await expect( + assembly.diarization.diarize({ + audio: bytes("\x00"), + contentType: "audio/pcm", + }), + ).rejects.toThrow("requires both sample_rate and channels"); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("should infer the content type from the file extension, unless overridden", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "aai-diarization-")); + try { + const file = path.join(dir, "call.mp3"); + writeFileSync(file, "ID3fake"); + + mockResponse(segmentsOnly); + await assembly.diarization.diarize({ audio: file }); + expect(part("audio")).toMatchObject({ + filename: "call.mp3", + type: "audio/mpeg", + body: "ID3fake", + }); + + fetchMock.resetMocks(); + mockResponse(segmentsOnly); + await assembly.diarization.diarize({ + audio: file, + contentType: "audio/flac", + }); + expect(part("audio")!.type).toBe("audio/flac"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("should read a stream whole and name it from its path", async () => { + mockResponse(segmentsOnly); + const stream = Readable.from([bytes("Ogg"), bytes("S")]); + (stream as unknown as { path: string }).path = "/tmp/x/note.webm"; + await assembly.diarization.diarize({ audio: stream }); + expect(part("audio")).toMatchObject({ + filename: "note.webm", + type: "audio/webm", + body: "OggS", + }); + }); + + it("should reject a URL", async () => { + await expect( + assembly.diarization.diarize({ audio: "https://example.com/a.wav" }), + ).rejects.toThrow("does not accept URLs"); + }); + + it("should map a 401 problem+json to a DiarizationError", async () => { + mockResponse( + { status: 401, title: "Unauthorized", detail: "Missing API key" }, + 401, + { "Content-Type": "application/problem+json" }, + ); + const error = await assembly.diarization + .diarize({ audio: bytes("RIFF") }) + .catch((e) => e); + expect(error).toBeInstanceOf(DiarizationError); + expect(error.name).toBe("DiarizationError"); + expect(error.status).toBe(401); + expect(error.errorCode).toBe("unauthorized"); + expect(error.message).toBe("Missing API key"); + }); + + it("should map a 400 validation error", async () => { + mockResponse( + { + status: 400, + title: "Bad Request", + detail: "transcript: words: Field required", + }, + 400, + ); + const error = await assembly.diarization + .diarize({ audio: bytes("RIFF"), transcript: {} as never }) + .catch((e) => e); + expect(error).toBeInstanceOf(DiarizationError); + expect(error.status).toBe(400); + expect(error.errorCode).toBe("bad_request"); + expect(error.message).toBe("transcript: words: Field required"); + }); + + it("should carry Retry-After on a 503", async () => { + mockResponse( + { status: 503, title: "Service Unavailable", detail: "busy" }, + 503, + { "Retry-After": "3" }, + ); + const error = await assembly.diarization + .diarize({ audio: bytes("RIFF") }) + .catch((e) => e); + expect(error.status).toBe(503); + expect(error.retryAfter).toBe(3); + }); + + it("should post to a custom diarizationBaseUrl", async () => { + const eu = new AssemblyAI({ + apiKey: defaultApiKey, + diarizationBaseUrl: "https://diarization.eu.assemblyai.com/", + }); + fetchMock.mockResponseOnce(JSON.stringify(segmentsOnly)); + await eu.diarization.diarize({ audio: bytes("RIFF") }); + expect(fetchMock.mock.calls[0][0]).toBe( + "https://diarization.eu.assemblyai.com/v1/diarize", + ); + }); + + it("should default to the global host", async () => { + const client = new AssemblyAI({ apiKey: defaultApiKey }); + fetchMock.mockResponseOnce(JSON.stringify(segmentsOnly)); + await client.diarization.diarize({ audio: bytes("RIFF") }); + expect(fetchMock.mock.calls[0][0]).toBe( + "https://diarization.assemblyai.com/v1/diarize", + ); + }); +});