From a2f8af7d4276f860736cc57953b3ad707d6158a1 Mon Sep 17 00:00:00 2001 From: Mufrad Hossain Date: Sat, 31 Oct 2026 04:16:09 +0600 Subject: [PATCH] qwen3_asr: scale decode budget for dense-script languages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared transcript estimate (12 tokens/s of audio) is calibrated on Latin-script tokenizers: English realizes ~1.2 tokens/word (~2.6 tok/s of speech), leaving generous headroom. Qwen3-ASR's tokenizer fragments dense scripts — measured 6.0 tokens/word for Bengali and 4.6 for Hindi on identical text — so fast speech in those scripts reaches ~21 tok/s and continuous Bengali dictation truncates at ~25 s (observed: 1307 tokens budgeted for a 105 s clip that needed ~1457). Give predict_transcript_tokens a per-family rate, default unchanged at 12, and pass 45 from qwen3_asr: the fastest measured speech with 2x margin, still clamped by the decoder context. Validated on a 105 s Bengali clip: rate 12 truncates with a warning, rate 45 decodes the full transcript with no slowdown (13.9 s vs 16.5 s wall clock). --- src/arch/qwen3_asr/model.cpp | 13 +++++++++++-- src/transcribe-decode-budget.h | 12 +++++++++--- tests/decode_budget_unit.cpp | 19 +++++++++++++++++++ 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/arch/qwen3_asr/model.cpp b/src/arch/qwen3_asr/model.cpp index db14e80f9..22a894b21 100644 --- a/src/arch/qwen3_asr/model.cpp +++ b/src/arch/qwen3_asr/model.cpp @@ -77,6 +77,13 @@ constexpr const char k_default_variant[] = "qwen3-asr"; // Generation reserve: what the input gate keeps free, and the decode-budget floor. constexpr int k_gen_reserve = 256; +// Qwen3-ASR's tokenizer fragments dense-script languages: measured 6.0 +// tokens/word for Bengali and 4.6 for Hindi, vs 1.2 for English. Fast speech +// in those scripts reaches ~21 tokens/s, so the shared 12 tok/s estimate +// truncates mid-clip (~25 s of continuous Bengali). 45 covers the fastest +// measured speech with 2x margin; the context ceiling still clamps runaway +// decodes. +constexpr int k_transcript_tokens_per_sec = 45; // Effective decoder context ceiling, in tokens: the model's trained maximum, // optionally lowered — never raised — by the caller's session n_ctx knob. @@ -853,7 +860,8 @@ transcribe_status run(transcribe_session * session, } const int max_new = transcribe::pick_decode_budget( - transcribe::predict_transcript_tokens(T_enc, cm->limits.ms_per_audio_token), k_gen_reserve, T_prompt, ceiling); + transcribe::predict_transcript_tokens(T_enc, cm->limits.ms_per_audio_token, k_transcript_tokens_per_sec), + k_gen_reserve, T_prompt, ceiling); // KV cache init (grow-to-fit, clamped to the context ceiling). Size to // hold prompt + decode budget, rounded up to a power of two (the step @@ -1739,7 +1747,8 @@ transcribe_status run_batch(transcribe_session * session, return TRANSCRIBE_OK; } const int max_new = - transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(max_T_enc, cm->limits.ms_per_audio_token), + transcribe::pick_decode_budget(transcribe::predict_transcript_tokens(max_T_enc, cm->limits.ms_per_audio_token, + k_transcript_tokens_per_sec), k_gen_reserve, max_T_prompt, ceiling); int max_n_kv = 1024; while (max_n_kv < max_T_prompt + max_new) { diff --git a/src/transcribe-decode-budget.h b/src/transcribe-decode-budget.h index 87ee28cf9..bfa061662 100644 --- a/src/transcribe-decode-budget.h +++ b/src/transcribe-decode-budget.h @@ -6,11 +6,17 @@ namespace transcribe { -// Conservative multilingual transcript estimate, in tokens per second. +// Default transcript estimate, in tokens per second. Calibrated on +// Latin-script languages: English realizes ~1.2 tokens per word (~2.6 tok/s +// of speech), so 12 leaves generous headroom. Dense-script languages +// fragment far more in multilingual tokenizers (measured on Qwen3-ASR's +// tokenizer: Bengali 6.0 tokens/word, Hindi 4.6, vs English 1.2), so +// families that transcribe them pass a higher rate. constexpr int k_transcript_tokens_per_sec = 12; // Fall back to the encoder-token count when its duration is unknown. -inline int predict_transcript_tokens(int audio_tokens, double ms_per_audio_token) { +inline int predict_transcript_tokens(int audio_tokens, double ms_per_audio_token, + int tokens_per_sec = k_transcript_tokens_per_sec) { if (audio_tokens <= 0) { return 0; } @@ -18,7 +24,7 @@ inline int predict_transcript_tokens(int audio_tokens, double ms_per_audio_token return audio_tokens; } const double seconds = static_cast(audio_tokens) * ms_per_audio_token / 1000.0; - const double predicted = seconds * k_transcript_tokens_per_sec; + const double predicted = seconds * tokens_per_sec; if (predicted <= 0.0) { return 0; } diff --git a/tests/decode_budget_unit.cpp b/tests/decode_budget_unit.cpp index 45e692726..5321a3bfe 100644 --- a/tests/decode_budget_unit.cpp +++ b/tests/decode_budget_unit.cpp @@ -19,6 +19,16 @@ void check_predict(const char * what, int audio_tokens, double ms_per_audio_toke } } +void check_predict_rate(const char * what, int audio_tokens, double ms_per_audio_token, int tokens_per_sec, + int expected) { + const int got = transcribe::predict_transcript_tokens(audio_tokens, ms_per_audio_token, tokens_per_sec); + if (got != expected) { + std::fprintf(stderr, "FAIL %s: predict_transcript_tokens(%d, %.3f, %d) = %d, expected %d\n", what, audio_tokens, + ms_per_audio_token, tokens_per_sec, got, expected); + ++g_failures; + } +} + void check_budget(const char * what, int predicted, int floor_tokens, int t_prompt, int ceiling, int expected) { const int got = transcribe::pick_decode_budget(predicted, floor_tokens, t_prompt, ceiling); if (got != expected) { @@ -37,6 +47,15 @@ int main(void) { check_predict("480 ms LFR encoder, same 197 s", /*audio_tokens=*/410, /*ms_per_audio_token=*/480.0, /*expected=*/2361); + // Families that transcribe dense-script languages pass a higher rate: + // Qwen3-ASR's tokenizer realizes ~6 tokens/word for Bengali and ~4.6 for + // Hindi vs ~1.2 for English, so fast speech in those scripts (~21 tok/s) + // truncates at the default 12 tok/s estimate (~25 s of continuous + // Bengali). 45 covers the fastest measured speech with 2x margin. + check_predict_rate("dense-script rate, 197 s", 2463, 80.0, 45, 8866); + check_predict_rate("26 s Bengali at the default rate truncates", 329, 80.0, 12, 315); + check_predict_rate("26 s Bengali at the dense-script rate completes", 329, 80.0, 45, 1184); + // An unpublished rate falls back to the audio-token count. check_predict("unknown rate falls back", 2463, 0.0, 2463); check_predict("negative rate falls back", 2463, -1.0, 2463);