From 2cd4ca49b4802cdc09c11874454c5dd30a0baa07 Mon Sep 17 00:00:00 2001 From: StuBehan Date: Wed, 28 Oct 2026 13:07:44 +0000 Subject: [PATCH 1/3] fix: stop short mlx chunks stalling playback Split over-long sentences at commas, semicolons, colons and dashes before falling back to word boundaries, and fill a chunk under 60 characters from the next sentence instead of sending it alone. A short chunk ahead of a long one left a pause of several seconds while the long one generated. --- README.md | 8 +++- stackvox/mlx_backend.py | 82 ++++++++++++++++++++++++++++----------- tests/test_mlx_backend.py | 35 +++++++++++++++++ 3 files changed, 100 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 9013a5e..da958ee 100644 --- a/README.md +++ b/README.md @@ -241,8 +241,12 @@ the model. They are rejected with an error unless the `mlx` backend is in use, both by `speak` and by a daemon running Kokoro. `--voice` and `--lang` apply to Kokoro only. Playback streams in chunks of about two sentences (up to ~170 characters), so the first audio arrives once the first chunk is generated; -short chunks also stop models drifting off-script on long text. Models slower -than real time will pause between chunks. +short chunks also stop models drifting off-script on long text. A sentence too +long for one chunk is split at a comma, semicolon, colon or dash where it has +one, and a very short sentence is joined to the start of the next rather than +sent alone, since a short chunk ahead of a long one leaves a pause while the +long one generates. Models slower than real time will still pause between +chunks. ### Model adapters diff --git a/stackvox/mlx_backend.py b/stackvox/mlx_backend.py index dd3bb7b..6d10a2a 100644 --- a/stackvox/mlx_backend.py +++ b/stackvox/mlx_backend.py @@ -31,6 +31,14 @@ # bring the first streamed audio forward. CHUNK_CHARS = 170 _SENTENCE_BOUNDARY = re.compile(r"(?<=[.!?])\s+|\n+") +# Over-long sentences are cut here before falling back to word boundaries, so +# the pause between chunks lands where a speaker would take a breath. +_CLAUSE_BOUNDARY = re.compile(r"(?<=[,;:—–])\s+") +# Streaming leaves a gap whenever a chunk takes longer to generate than the one +# before it takes to play, so a short sentence sent alone ahead of a long one +# ("Yes." then 170 characters) is the worst case for models near real time. +# A chunk shorter than this is filled from the next sentence instead. +_MIN_CHUNK_CHARS = 60 # MLX keeps freed GPU buffers in its own cache for reuse, and by default lets it # grow to the device's recommended working set (tens of GB on a large Mac). A @@ -80,33 +88,61 @@ def swapped(owner: type, name: str, value: Any) -> Iterator[None]: delattr(owner, name) +def _split_words(text: str, limit: int) -> list[str]: + pieces: list[str] = [] + piece = "" + for word in text.split(): + candidate = f"{piece} {word}".strip() + if piece and len(candidate) > limit: + pieces.append(piece) + piece = word + else: + piece = candidate + if piece: + pieces.append(piece) + return pieces + + +def _sentence_pieces(sentence: str, limit: int) -> list[str]: + if len(sentence) <= limit: + return [sentence] + pieces: list[str] = [] + for clause in (part.strip() for part in _CLAUSE_BOUNDARY.split(sentence)): + if clause: + pieces.extend([clause] if len(clause) <= limit else _split_words(clause, limit)) + return pieces + + +def _top_up(chunk: str, piece: str, limit: int) -> tuple[str, str]: + """Move as many of ``piece``'s opening words onto ``chunk`` as ``limit`` allows.""" + words = piece.split() + taken = 0 + while taken < len(words) and len(" ".join([chunk, *words[: taken + 1]])) <= limit: + taken += 1 + return " ".join([chunk, *words[:taken]]), " ".join(words[taken:]) + + def split_long_text(text: str, limit: int = CHUNK_CHARS) -> list[str]: - """Pack prose into sentence-aware chunks no longer than ``limit``.""" + """Pack prose into chunks no longer than ``limit``. + + Chunks break between sentences where they can. A sentence longer than + ``limit`` breaks at clause punctuation, and only a clause still longer than + that breaks between words. A chunk under ``_MIN_CHUNK_CHARS`` that the next + piece won't fit beside takes that piece's opening words rather than going + out alone. + """ sentences = [part.strip() for part in _SENTENCE_BOUNDARY.split(text) if part.strip()] chunks: list[str] = [] current = "" - for sentence in sentences: - pieces = [sentence] - if len(sentence) > limit: - words = sentence.split() - pieces = [] - piece = "" - for word in words: - candidate = f"{piece} {word}".strip() - if piece and len(candidate) > limit: - pieces.append(piece) - piece = word - else: - piece = candidate - if piece: - pieces.append(piece) - for piece in pieces: - candidate = f"{current} {piece}".strip() - if current and len(candidate) > limit: - chunks.append(current) - current = piece - else: - current = candidate + for piece in (piece for sentence in sentences for piece in _sentence_pieces(sentence, limit)): + candidate = f"{current} {piece}".strip() + if not current or len(candidate) <= limit: + current = candidate + continue + if len(current) < _MIN_CHUNK_CHARS: + current, piece = _top_up(current, piece, limit) + chunks.append(current) + current = piece if current: chunks.append(current) return chunks diff --git a/tests/test_mlx_backend.py b/tests/test_mlx_backend.py index dec05e0..4f4dc96 100644 --- a/tests/test_mlx_backend.py +++ b/tests/test_mlx_backend.py @@ -347,6 +347,41 @@ def test_overlong_sentence_splits_on_words(self): assert all(len(chunk) <= 50 for chunk in chunks) assert " ".join(chunks).split() == ["word"] * 200 + def test_overlong_sentence_splits_at_clauses_before_words(self): + sentence = ( + ", ".join(f"this is clause number {index} of one long sentence" for index in range(8)) + "." + ) + chunks = mlx_backend.split_long_text(sentence) + assert len(chunks) > 1 + assert all(len(chunk) <= mlx_backend.CHUNK_CHARS for chunk in chunks) + assert all(chunk.endswith((",", ".")) for chunk in chunks) + assert " ".join(chunks) == sentence + + def test_short_sentence_takes_words_from_a_long_one_rather_than_going_alone(self): + long_sentence = " ".join(["alpha"] * 28) + "." + chunks = mlx_backend.split_long_text(f"Yes. {long_sentence}") + assert chunks[0].startswith("Yes. alpha") + assert len(chunks[0]) <= mlx_backend.CHUNK_CHARS + assert " ".join(chunks) == f"Yes. {long_sentence}" + + def test_chunks_long_enough_to_stand_alone_are_not_topped_up(self): + first = "The first sentence is comfortably longer than the minimum chunk length, so it stands." + second = ( + "The second " + " ".join(["sentence"] * 10) + " is long enough that the two cannot share a chunk." + ) + assert mlx_backend.split_long_text(f"{first} {second}") == [first, second] + + def test_mixed_paragraph_breaks_only_at_punctuation(self): + text = ( + "Switched to the four bit model. The daemon is back under launchd, and this paragraph is long enough " + "to stream as more than one chunk, so you should hear it carry on into the second chunk without the " + "pause you were getting before. If there is still a gap between the sentences, let me know and we " + "can look at buffering ahead." + ) + chunks = mlx_backend.split_long_text(text) + assert all(chunk.endswith((",", ".")) for chunk in chunks) + assert " ".join(chunks) == text + class TestMlxThread: def test_load_and_generation_share_one_thread_across_callers(self, mocker, tmp_path): From 1c4f2fa31bebbd867756f65ce76bc15a868c27fd Mon Sep 17 00:00:00 2001 From: StuBehan Date: Wed, 28 Oct 2026 22:31:13 +0000 Subject: [PATCH 2/3] fix: top up short chunks at clause boundaries --- README.md | 10 ++++++---- stackvox/mlx_backend.py | 33 +++++++++++++++++++++++---------- tests/test_mlx_backend.py | 31 ++++++++++++++++++++++++++++++- 3 files changed, 59 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index da958ee..952623a 100644 --- a/README.md +++ b/README.md @@ -243,10 +243,12 @@ Kokoro only. Playback streams in chunks of about two sentences (up to ~170 characters), so the first audio arrives once the first chunk is generated; short chunks also stop models drifting off-script on long text. A sentence too long for one chunk is split at a comma, semicolon, colon or dash where it has -one, and a very short sentence is joined to the start of the next rather than -sent alone, since a short chunk ahead of a long one leaves a pause while the -long one generates. Models slower than real time will still pause between -chunks. +one. A chunk under 60 characters is filled up to the limit from the start of +the next sentence rather than sent alone, at a clause boundary where one fits +and between words otherwise, since a short chunk ahead of a long one leaves a +pause while the long one generates. That means a reply opening on a short +sentence waits for a full-size first chunk. Models slower than real time will +still pause between chunks. ### Model adapters diff --git a/stackvox/mlx_backend.py b/stackvox/mlx_backend.py index 6d10a2a..287850b 100644 --- a/stackvox/mlx_backend.py +++ b/stackvox/mlx_backend.py @@ -32,12 +32,14 @@ CHUNK_CHARS = 170 _SENTENCE_BOUNDARY = re.compile(r"(?<=[.!?])\s+|\n+") # Over-long sentences are cut here before falling back to word boundaries, so -# the pause between chunks lands where a speaker would take a breath. -_CLAUSE_BOUNDARY = re.compile(r"(?<=[,;:—–])\s+") +# the pause between chunks lands where a speaker would take a breath. A typed +# dash counts only with space either side, so hyphenated words stay whole. +_CLAUSE_BOUNDARY = re.compile(r"(?<=[,;:—–])\s+|(?<=\s-)\s+|(?<=\s--)\s+") # Streaming leaves a gap whenever a chunk takes longer to generate than the one # before it takes to play, so a short sentence sent alone ahead of a long one # ("Yes." then 170 characters) is the worst case for models near real time. -# A chunk shorter than this is filled from the next sentence instead. +# A chunk shorter than this that the next piece won't fit beside takes the +# opening of that piece, up to the chunk limit, instead. _MIN_CHUNK_CHARS = 60 # MLX keeps freed GPU buffers in its own cache for reuse, and by default lets it @@ -113,13 +115,24 @@ def _sentence_pieces(sentence: str, limit: int) -> list[str]: return pieces -def _top_up(chunk: str, piece: str, limit: int) -> tuple[str, str]: - """Move as many of ``piece``'s opening words onto ``chunk`` as ``limit`` allows.""" - words = piece.split() +def _fill(chunk: str, parts: list[str], limit: int) -> tuple[str, str]: taken = 0 - while taken < len(words) and len(" ".join([chunk, *words[: taken + 1]])) <= limit: + while taken < len(parts) and len(" ".join([chunk, *parts[: taken + 1]])) <= limit: taken += 1 - return " ".join([chunk, *words[:taken]]), " ".join(words[taken:]) + return " ".join([chunk, *parts[:taken]]), " ".join(parts[taken:]) + + +def _top_up(chunk: str, piece: str, limit: int) -> tuple[str, str]: + """Move as much of ``piece``'s opening onto ``chunk`` as ``limit`` allows. + + Whole clauses are moved when they bring ``chunk`` up to ``_MIN_CHUNK_CHARS``, + so the cut lands at punctuation; otherwise words are, and the cut lands + mid clause. + """ + topped, rest = _fill(chunk, _CLAUSE_BOUNDARY.split(piece), limit) + if len(topped) >= _MIN_CHUNK_CHARS: + return topped, rest + return _fill(chunk, piece.split(), limit) def split_long_text(text: str, limit: int = CHUNK_CHARS) -> list[str]: @@ -128,8 +141,8 @@ def split_long_text(text: str, limit: int = CHUNK_CHARS) -> list[str]: Chunks break between sentences where they can. A sentence longer than ``limit`` breaks at clause punctuation, and only a clause still longer than that breaks between words. A chunk under ``_MIN_CHUNK_CHARS`` that the next - piece won't fit beside takes that piece's opening words rather than going - out alone. + piece won't fit beside takes that piece's opening clauses, or words where no + clause fits, rather than going out alone. """ sentences = [part.strip() for part in _SENTENCE_BOUNDARY.split(text) if part.strip()] chunks: list[str] = [] diff --git a/tests/test_mlx_backend.py b/tests/test_mlx_backend.py index 4f4dc96..acc95fa 100644 --- a/tests/test_mlx_backend.py +++ b/tests/test_mlx_backend.py @@ -347,6 +347,16 @@ def test_overlong_sentence_splits_on_words(self): assert all(len(chunk) <= 50 for chunk in chunks) assert " ".join(chunks).split() == ["word"] * 200 + @pytest.mark.parametrize("separator", [",", ";", ":", " —", " –", " -", " --"]) + def test_overlong_sentence_splits_at_each_clause_separator(self, separator): + first = " ".join(["alpha"] * 20) + separator + second = " ".join(["beta"] * 20) + "." + assert mlx_backend.split_long_text(f"{first} {second}") == [first, second] + + def test_hyphenated_words_are_not_clause_boundaries(self): + sentence = " ".join(["well-known"] * 20) + "." + assert all(chunk.endswith(("well-known", ".")) for chunk in mlx_backend.split_long_text(sentence)) + def test_overlong_sentence_splits_at_clauses_before_words(self): sentence = ( ", ".join(f"this is clause number {index} of one long sentence" for index in range(8)) + "." @@ -364,6 +374,25 @@ def test_short_sentence_takes_words_from_a_long_one_rather_than_going_alone(self assert len(chunks[0]) <= mlx_backend.CHUNK_CHARS assert " ".join(chunks) == f"Yes. {long_sentence}" + def test_short_sentence_takes_whole_clauses_when_they_fill_it(self): + clause = " ".join(["alpha"] * 12) + "," + long_sentence = f"{clause} " + " ".join(["beta"] * 20) + "." + chunks = mlx_backend.split_long_text(f"Yes. {long_sentence}") + assert chunks[0] == f"Yes. {clause}" + + def test_short_sentence_takes_words_when_the_opening_clause_is_too_short(self): + long_sentence = "Well, " + " ".join(["alpha"] * 28) + "." + chunks = mlx_backend.split_long_text(f"Yes. {long_sentence}") + assert chunks[0].startswith("Yes. Well, alpha") + assert len(chunks[0]) >= mlx_backend._MIN_CHUNK_CHARS + + @pytest.mark.parametrize(("first_length", "topped_up"), [(59, True), (60, False)]) + def test_top_up_applies_only_below_the_minimum_chunk_length(self, first_length, topped_up): + first = "x" * (first_length - 1) + "." + second = " ".join(["alpha"] * 25) + "." + chunks = mlx_backend.split_long_text(f"{first} {second}") + assert (chunks[0] != first) is topped_up + def test_chunks_long_enough_to_stand_alone_are_not_topped_up(self): first = "The first sentence is comfortably longer than the minimum chunk length, so it stands." second = ( @@ -371,7 +400,7 @@ def test_chunks_long_enough_to_stand_alone_are_not_topped_up(self): ) assert mlx_backend.split_long_text(f"{first} {second}") == [first, second] - def test_mixed_paragraph_breaks_only_at_punctuation(self): + def test_paragraph_with_commas_breaks_only_at_punctuation(self): text = ( "Switched to the four bit model. The daemon is back under launchd, and this paragraph is long enough " "to stream as more than one chunk, so you should hear it carry on into the second chunk without the " From be24b4d11ea375baba5f90a064e33dc2a3b52218 Mon Sep 17 00:00:00 2001 From: StuBehan Date: Wed, 28 Oct 2026 22:39:22 +0000 Subject: [PATCH 3/3] test: pin clause-first top-up and dash boundaries --- README.md | 10 +++++----- stackvox/mlx_backend.py | 5 +++-- tests/test_mlx_backend.py | 13 ++++++++----- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 952623a..3022a93 100644 --- a/README.md +++ b/README.md @@ -244,11 +244,11 @@ characters), so the first audio arrives once the first chunk is generated; short chunks also stop models drifting off-script on long text. A sentence too long for one chunk is split at a comma, semicolon, colon or dash where it has one. A chunk under 60 characters is filled up to the limit from the start of -the next sentence rather than sent alone, at a clause boundary where one fits -and between words otherwise, since a short chunk ahead of a long one leaves a -pause while the long one generates. That means a reply opening on a short -sentence waits for a full-size first chunk. Models slower than real time will -still pause between chunks. +the next sentence or clause rather than sent alone, at a clause boundary where +that reaches 60 characters and between words otherwise, since a short chunk +ahead of a long one leaves a pause while the long one generates. That means a +reply opening on a short sentence waits for a full-size first chunk. Models +slower than real time will still pause between chunks. ### Model adapters diff --git a/stackvox/mlx_backend.py b/stackvox/mlx_backend.py index 287850b..54b780e 100644 --- a/stackvox/mlx_backend.py +++ b/stackvox/mlx_backend.py @@ -31,8 +31,9 @@ # bring the first streamed audio forward. CHUNK_CHARS = 170 _SENTENCE_BOUNDARY = re.compile(r"(?<=[.!?])\s+|\n+") -# Over-long sentences are cut here before falling back to word boundaries, so -# the pause between chunks lands where a speaker would take a breath. A typed +# Over-long sentences, and the next sentence when a short chunk is topped up, +# are cut here before falling back to word boundaries, so the pause between +# chunks lands where a speaker would take a breath. A typed # dash counts only with space either side, so hyphenated words stay whole. _CLAUSE_BOUNDARY = re.compile(r"(?<=[,;:—–])\s+|(?<=\s-)\s+|(?<=\s--)\s+") # Streaming leaves a gap whenever a chunk takes longer to generate than the one diff --git a/tests/test_mlx_backend.py b/tests/test_mlx_backend.py index acc95fa..c8c8d79 100644 --- a/tests/test_mlx_backend.py +++ b/tests/test_mlx_backend.py @@ -353,9 +353,10 @@ def test_overlong_sentence_splits_at_each_clause_separator(self, separator): second = " ".join(["beta"] * 20) + "." assert mlx_backend.split_long_text(f"{first} {second}") == [first, second] - def test_hyphenated_words_are_not_clause_boundaries(self): - sentence = " ".join(["well-known"] * 20) + "." - assert all(chunk.endswith(("well-known", ".")) for chunk in mlx_backend.split_long_text(sentence)) + def test_hyphen_without_a_space_before_it_is_not_a_clause_boundary(self): + clause = " ".join(["alpha"] * 20) + " pre- and post-war " + " ".join(["beta"] * 20) + "," + chunks = mlx_backend.split_long_text(f"{clause} gamma.") + assert chunks[0].split()[-1] != "pre-" def test_overlong_sentence_splits_at_clauses_before_words(self): sentence = ( @@ -376,12 +377,14 @@ def test_short_sentence_takes_words_from_a_long_one_rather_than_going_alone(self def test_short_sentence_takes_whole_clauses_when_they_fill_it(self): clause = " ".join(["alpha"] * 12) + "," - long_sentence = f"{clause} " + " ".join(["beta"] * 20) + "." + long_sentence = f"{clause} " + " ".join(["beta"] * 19) + "." + assert len(long_sentence) <= mlx_backend.CHUNK_CHARS < len(f"Yes. {long_sentence}") chunks = mlx_backend.split_long_text(f"Yes. {long_sentence}") assert chunks[0] == f"Yes. {clause}" def test_short_sentence_takes_words_when_the_opening_clause_is_too_short(self): - long_sentence = "Well, " + " ".join(["alpha"] * 28) + "." + long_sentence = "Well, " + " ".join(["alpha"] * 27) + "." + assert len(long_sentence) <= mlx_backend.CHUNK_CHARS < len(f"Yes. {long_sentence}") chunks = mlx_backend.split_long_text(f"Yes. {long_sentence}") assert chunks[0].startswith("Yes. Well, alpha") assert len(chunks[0]) >= mlx_backend._MIN_CHUNK_CHARS