From b7e5703981c427e70af2938e0bb3f6470e242f71 Mon Sep 17 00:00:00 2001 From: TachyonicBytes Date: Tue, 27 Oct 2026 21:27:27 +0200 Subject: [PATCH] feat: Allow text-only instruct for Voxtral models --- docs/models/voxtral-mini-3b-2507.md | 2 +- docs/models/voxtral-small-24b-2507.md | 2 +- docs/prompting.md | 2 + include/transcribe.h | 8 ++- src/arch/voxtral/model.cpp | 82 ++++++++++++++++----------- src/transcribe-asr.cpp | 16 +++++- tests/run_dispatch_unit.cpp | 11 ++++ 7 files changed, 82 insertions(+), 41 deletions(-) diff --git a/docs/models/voxtral-mini-3b-2507.md b/docs/models/voxtral-mini-3b-2507.md index 567e78938..a8fd33596 100644 --- a/docs/models/voxtral-mini-3b-2507.md +++ b/docs/models/voxtral-mini-3b-2507.md @@ -25,7 +25,7 @@ mono WAV and produces a transcript via greedy decoding. mistral-common instruct template ("Translate this to {Language}.") to translate non-English speech into the target language's text. -**Prompting:** `--task instruct --prompt ""` replaces the transcription request with a free-text instruction (summaries, questions, reformatting); the output is free text. +**Prompting:** `--task instruct --prompt ""` replaces the transcription request with a free-text instruction (summaries, questions, reformatting); the output is free text. With no audio (`n_samples == 0`), the instruction runs on its own as a text-only chat turn. See Mistral's [model card](https://huggingface.co/mistralai/Voxtral-Mini-3B-2507) for training data, intended use, and upstream evaluation. diff --git a/docs/models/voxtral-small-24b-2507.md b/docs/models/voxtral-small-24b-2507.md index e6e3614c6..6330e0f7a 100644 --- a/docs/models/voxtral-small-24b-2507.md +++ b/docs/models/voxtral-small-24b-2507.md @@ -23,7 +23,7 @@ mono WAV and produces a transcript via greedy decoding. mistral-common instruct template ("Translate this to {Language}.") to translate non-English speech into the target language's text. -**Prompting:** `--task instruct --prompt ""` replaces the transcription request with a free-text instruction (summaries, questions, reformatting); the output is free text. +**Prompting:** `--task instruct --prompt ""` replaces the transcription request with a free-text instruction (summaries, questions, reformatting); the output is free text. With no audio (`n_samples == 0`), the instruction runs on its own as a text-only chat turn. See Mistral's [model card](https://huggingface.co/mistralai/Voxtral-Small-24B-2507) for training data, intended use, and upstream evaluation. diff --git a/docs/prompting.md b/docs/prompting.md index d908e5c2d..e69646506 100644 --- a/docs/prompting.md +++ b/docs/prompting.md @@ -30,6 +30,8 @@ note under [`models/`](models/). `prompt` keeps its most recent text, both with a WARN. An INSTRUCT prompt that does not fit is an error. - INSTRUCT requires `target_language == NULL` and timestamps NONE or AUTO. +- INSTRUCT also runs with no audio: `transcribe_run` with `n_samples == 0` + sends the prompt alone, i.e. for post-processing. - `prefix`: the audio must contain the prefix's speech, and long-form models apply it to the first window only. - A feature bit covers plain transcription. Under another task or output mode diff --git a/include/transcribe.h b/include/transcribe.h index ec4ed5474..c4421ee15 100644 --- a/include/transcribe.h +++ b/include/transcribe.h @@ -469,7 +469,9 @@ TRANSCRIBE_API void transcribe_log_set(transcribe_log_callback cb, void * userda /* ----------------------------------------------------------------------- */ /* INSTRUCT: transcribe_run_params::prompt is the instruction and the output - * is free text (only full_text / raw_text are guaranteed). Offline only. */ + * is free text (only full_text / raw_text are guaranteed). Offline only. + * transcribe_run with n_samples == 0 (pcm may be NULL) runs the instruction + * on its own, with no audio. */ typedef enum { TRANSCRIBE_TASK_TRANSCRIBE = 0, TRANSCRIBE_TASK_TRANSLATE = 1, @@ -1690,7 +1692,9 @@ TRANSCRIBE_API const struct transcribe_model * transcribe_get_model(const struct * in place. Silence is valid. * n_samples: number of samples in pcm. Must be strictly positive; * a non-positive count returns TRANSCRIBE_ERR_INVALID_ARG - * (same rule as transcribe_stream_feed). + * (same rule as transcribe_stream_feed). The exception is + * TRANSCRIBE_TASK_INSTRUCT: 0 (pcm may then be NULL) is a + * text-only run on the prompt alone. * params: run params, or NULL for defaults. * * v1 supports only 16 kHz mono float32 PCM and does not link a diff --git a/src/arch/voxtral/model.cpp b/src/arch/voxtral/model.cpp index 577f8e883..aa55c2a4e 100644 --- a/src/arch/voxtral/model.cpp +++ b/src/arch/voxtral/model.cpp @@ -6,9 +6,12 @@ // // transcription : [BOS][INST][BEGIN_AUDIO][AUDIO]*N[/INST](lang:)?[TRANSCRIBE] // instruct : [BOS][INST][BEGIN_AUDIO][AUDIO]*N BPE(instruction)[/INST] +// text instruct : [BOS][INST] BPE(instruction)[/INST] // // Instruct mode covers translation (task TRANSLATE synthesizes "Translate this -// to {Language}.") and free-text prompting. +// to {Language}.") and free-text prompting. A TRANSCRIBE_TASK_INSTRUCT run +// with no audio (n_samples == 0) skips the encoder and runs the text-only +// mistral-common chat template. #include "causal_lm/causal_lm.h" #include "decoder.h" @@ -343,7 +346,8 @@ bool instruct_instruction(const transcribe_run_params * params, std::string & in return false; } -// Build the instruct prompt: audio + BPE(instruction) + [/INST]. +// Build the instruct prompt: audio + BPE(instruction) + [/INST]. With no +// audio (n_audio == 0) the [BEGIN_AUDIO] marker is left out too. transcribe_status build_instruct_prompt(const VoxtralModel & m, const std::string & instruction, int n_audio, @@ -354,8 +358,10 @@ transcribe_status build_instruct_prompt(const VoxtralModel & m, out_ids.clear(); out_ids.push_back(s.bos); out_ids.push_back(s.inst); - out_ids.push_back(s.begin_audio); - prefix_len = static_cast(out_ids.size()); // 3 + if (n_audio > 0) { + out_ids.push_back(s.begin_audio); + } + prefix_len = static_cast(out_ids.size()); // 3, or 2 without audio for (int i = 0; i < n_audio; ++i) { out_ids.push_back(m.hparams.audio_token_id); } @@ -583,7 +589,9 @@ transcribe_status run(transcribe_session * session, const float * pcm, int n_samples, const transcribe_run_params * params) { - if (session == nullptr || pcm == nullptr || n_samples <= 0) { + // No audio is only valid as a text-only INSTRUCT run. + const bool text_only = n_samples == 0 && params != nullptr && params->task == TRANSCRIBE_TASK_INSTRUCT; + if (session == nullptr || (!text_only && (pcm == nullptr || n_samples <= 0))) { return TRANSCRIBE_ERR_INVALID_ARG; } auto * cc = static_cast(session); @@ -616,38 +624,42 @@ transcribe_status run(transcribe_session * session, samples_per_chunk / cm->hparams.fe_hop_length; const int audio_per_chunk = cm->hparams.audio_tokens_per_chunk(); - const int n_chunks = std::max(1, (n_samples + samples_per_chunk - 1) / samples_per_chunk); + // Text-only: no chunks, so the mel and encoder steps below do nothing. + const int n_chunks = text_only ? 0 : std::max(1, (n_samples + samples_per_chunk - 1) / samples_per_chunk); const size_t padded = static_cast(n_chunks) * samples_per_chunk; - std::vector pcm_padded(padded, 0.0f); - std::memcpy(pcm_padded.data(), pcm, static_cast(n_samples) * sizeof(float)); + int mel_n_mels = 0, mel_n_frames = 0; + if (!text_only) { + std::vector pcm_padded(padded, 0.0f); + std::memcpy(pcm_padded.data(), pcm, static_cast(n_samples) * sizeof(float)); - const int64_t t_mel_start = ggml_time_us(); - int mel_n_mels = 0, mel_n_frames = 0; - if (const transcribe_status mst = - cm->mel->compute(pcm_padded.data(), padded, cc->mel_buf, mel_n_mels, mel_n_frames, cc->n_threads); - mst != TRANSCRIBE_OK) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "voxtral run: MelFrontend::compute failed (%s)", - transcribe_status_string(mst)); - return mst; - } - cc->t_mel_us = ggml_time_us() - t_mel_start; + const int64_t t_mel_start = ggml_time_us(); + if (const transcribe_status mst = + cm->mel->compute(pcm_padded.data(), padded, cc->mel_buf, mel_n_mels, mel_n_frames, cc->n_threads); + mst != TRANSCRIBE_OK) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "voxtral run: MelFrontend::compute failed (%s)", + transcribe_status_string(mst)); + return mst; + } + cc->t_mel_us = ggml_time_us() - t_mel_start; - if (mel_n_mels != cm->hparams.enc_num_mel_bins) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "voxtral run: mel bins %d != %d", mel_n_mels, cm->hparams.enc_num_mel_bins); - return TRANSCRIBE_ERR_GGUF; - } - if (mel_n_frames < n_chunks * frames_per_chunk) { - log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "voxtral run: mel frames %d < %d*%d", mel_n_frames, n_chunks, - frames_per_chunk); - return TRANSCRIBE_ERR_GGUF; - } + if (mel_n_mels != cm->hparams.enc_num_mel_bins) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "voxtral run: mel bins %d != %d", mel_n_mels, + cm->hparams.enc_num_mel_bins); + return TRANSCRIBE_ERR_GGUF; + } + if (mel_n_frames < n_chunks * frames_per_chunk) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "voxtral run: mel frames %d < %d*%d", mel_n_frames, n_chunks, + frames_per_chunk); + return TRANSCRIBE_ERR_GGUF; + } - // enc.mel.in dump: mel-major [n_mels, n_frames], matching the reference. - if (transcribe::debug::enabled()) { - const long long shape[2] = { mel_n_mels, mel_n_frames }; - transcribe::debug::dump_host_f32("enc.mel.in", cc->mel_buf.data(), static_cast(cc->mel_buf.size()), - shape, 2, "frontend.mel.norm"); + // enc.mel.in dump: mel-major [n_mels, n_frames], matching the reference. + if (transcribe::debug::enabled()) { + const long long shape[2] = { mel_n_mels, mel_n_frames }; + transcribe::debug::dump_host_f32("enc.mel.in", cc->mel_buf.data(), + static_cast(cc->mel_buf.size()), shape, 2, "frontend.mel.norm"); + } } // ----- Encoder + projector, per 30 s chunk ----- @@ -821,11 +833,13 @@ transcribe_status run(transcribe_session * session, const int chunk_size = causal_lm::prefill_chunk_size(); std::vector logits(vocab); - if (T_prompt > chunk_size) { + // A text-only prompt also goes through the chunked path: it already + // handles chunks with no audio rows, which the single-shot graph does not. + if (T_prompt > chunk_size || n_audio_total == 0) { // Long prompt: chunk it. A single flash-attention call is capped at // 65535 query rows on Metal, and the [T, T] mask is O(T^2) on both // host and device — see decoder.h. - if (dumps_on) { + if (dumps_on && T_prompt > chunk_size) { transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "voxtral run: T_prompt=%d exceeds the %d-token prefill chunk — only " "dec.logits_raw is dumped (per-chunk hidden states have no single-shot " diff --git a/src/transcribe-asr.cpp b/src/transcribe-asr.cpp index ffbfa0c2a..357bf3ac0 100644 --- a/src/transcribe-asr.cpp +++ b/src/transcribe-asr.cpp @@ -1324,6 +1324,15 @@ extern "C" transcribe_status transcribe_stream_get_text(const struct transcribe_ // `committed` (optional) is set true once the call passes the pre-clear // gates and commits to replacing the result. The caller uses it to decide // whether compute scratch needs releasing. +// A text-only TRANSCRIBE_TASK_INSTRUCT run: no audio (n_samples == 0, pcm may +// be NULL), with the prompt being the whole input. Reads only the caller's params, +// so it is safe before the session is dereferenced. +static bool is_text_only_run(int n_samples, const struct transcribe_run_params * params) { + return n_samples == 0 && params != nullptr && + params->struct_size >= TRANSCRIBE_FIELD_END(transcribe_run_params, task) && + enum_field_raw(¶ms->task) == TRANSCRIBE_TASK_INSTRUCT; +} + static transcribe_status run_one_inner(struct transcribe_session * session, const float * pcm, int n_samples, @@ -1338,13 +1347,14 @@ static transcribe_status run_one_inner(struct transcribe_session * sess // replacement happens when the call is well-formed enough that // the dispatcher is willing to dereference session. A call that // isn't well-formed isn't considered to have "run". - if (session == nullptr || pcm == nullptr) { + if (session == nullptr) { return TRANSCRIBE_ERR_INVALID_ARG; } // n_samples must be strictly positive, matching transcribe_stream_feed. // There is no meaningful "transcribe zero samples" operation; a // zero-length batch is treated as a caller error, not an empty run. - if (n_samples <= 0) { + // The one exception is a text-only INSTRUCT run (see is_text_only_run). + if (!is_text_only_run(n_samples, params) && (pcm == nullptr || n_samples <= 0)) { return TRANSCRIBE_ERR_INVALID_ARG; } // NULL params means "all defaults" (transcribe vs translate, no @@ -1488,7 +1498,7 @@ static transcribe_status transcribe_run_impl(struct transcribe_session * if (pcm != nullptr && n_samples > 0 && !pcm_is_finite(pcm, n_samples)) { return TRANSCRIBE_ERR_INVALID_ARG; } - if (session != nullptr && pcm != nullptr && n_samples > 0) { + if (session != nullptr && ((pcm != nullptr && n_samples > 0) || is_text_only_run(n_samples, params))) { session->batch_results.clear(); } // run_one_inner arms the guard at its commit point, so a pre-clear diff --git a/tests/run_dispatch_unit.cpp b/tests/run_dispatch_unit.cpp index 4ee967f1c..9bcebb425 100644 --- a/tests/run_dispatch_unit.cpp +++ b/tests/run_dispatch_unit.cpp @@ -700,6 +700,17 @@ void test_prompting_validation() { p.timestamps = TRANSCRIBE_TIMESTAMPS_NONE; CHECK(prompt_run(model, p) == TRANSCRIBE_OK); + // No audio: INSTRUCT runs on the prompt alone; any other task is malformed. + { + transcribe_session session; + session.model = &model; + CHECK(transcribe_run(&session, nullptr, 0, &p) == TRANSCRIBE_OK); + CHECK(g_seen_params.task == TRANSCRIBE_TASK_INSTRUCT); + p.task = TRANSCRIBE_TASK_TRANSCRIBE; + CHECK(transcribe_run(&session, nullptr, 0, &p) == TRANSCRIBE_ERR_INVALID_ARG); + p.task = TRANSCRIBE_TASK_INSTRUCT; + } + // Prefix: hard gate, and never under INSTRUCT. p.prefix = "Good morning"; CHECK(prompt_run(model, p) == TRANSCRIBE_ERR_INVALID_ARG);