diff --git a/README.md b/README.md index 9013a5e..3022a93 100644 --- a/README.md +++ b/README.md @@ -241,8 +241,14 @@ the model. They are rejected with an error unless the `mlx` backend is in use, both by `speak` and by a daemon running Kokoro. `--voice` and `--lang` apply to Kokoro only. Playback streams in chunks of about two sentences (up to ~170 characters), so the first audio arrives once the first chunk is generated; -short chunks also stop models drifting off-script on long text. Models slower -than real time will pause between chunks. +short chunks also stop models drifting off-script on long text. A sentence too +long for one chunk is split at a comma, semicolon, colon or dash where it has +one. A chunk under 60 characters is filled up to the limit from the start of +the next sentence or clause rather than sent alone, at a clause boundary where +that reaches 60 characters and between words otherwise, since a short chunk +ahead of a long one leaves a pause while the long one generates. That means a +reply opening on a short sentence waits for a full-size first chunk. Models +slower than real time will still pause between chunks. ### Model adapters diff --git a/stackvox/mlx_backend.py b/stackvox/mlx_backend.py index dd3bb7b..54b780e 100644 --- a/stackvox/mlx_backend.py +++ b/stackvox/mlx_backend.py @@ -31,6 +31,17 @@ # bring the first streamed audio forward. CHUNK_CHARS = 170 _SENTENCE_BOUNDARY = re.compile(r"(?<=[.!?])\s+|\n+") +# Over-long sentences, and the next sentence when a short chunk is topped up, +# are cut here before falling back to word boundaries, so the pause between +# chunks lands where a speaker would take a breath. A typed +# dash counts only with space either side, so hyphenated words stay whole. +_CLAUSE_BOUNDARY = re.compile(r"(?<=[,;:—–])\s+|(?<=\s-)\s+|(?<=\s--)\s+") +# Streaming leaves a gap whenever a chunk takes longer to generate than the one +# before it takes to play, so a short sentence sent alone ahead of a long one +# ("Yes." then 170 characters) is the worst case for models near real time. +# A chunk shorter than this that the next piece won't fit beside takes the +# opening of that piece, up to the chunk limit, instead. +_MIN_CHUNK_CHARS = 60 # MLX keeps freed GPU buffers in its own cache for reuse, and by default lets it # grow to the device's recommended working set (tens of GB on a large Mac). A @@ -80,33 +91,72 @@ def swapped(owner: type, name: str, value: Any) -> Iterator[None]: delattr(owner, name) +def _split_words(text: str, limit: int) -> list[str]: + pieces: list[str] = [] + piece = "" + for word in text.split(): + candidate = f"{piece} {word}".strip() + if piece and len(candidate) > limit: + pieces.append(piece) + piece = word + else: + piece = candidate + if piece: + pieces.append(piece) + return pieces + + +def _sentence_pieces(sentence: str, limit: int) -> list[str]: + if len(sentence) <= limit: + return [sentence] + pieces: list[str] = [] + for clause in (part.strip() for part in _CLAUSE_BOUNDARY.split(sentence)): + if clause: + pieces.extend([clause] if len(clause) <= limit else _split_words(clause, limit)) + return pieces + + +def _fill(chunk: str, parts: list[str], limit: int) -> tuple[str, str]: + taken = 0 + while taken < len(parts) and len(" ".join([chunk, *parts[: taken + 1]])) <= limit: + taken += 1 + return " ".join([chunk, *parts[:taken]]), " ".join(parts[taken:]) + + +def _top_up(chunk: str, piece: str, limit: int) -> tuple[str, str]: + """Move as much of ``piece``'s opening onto ``chunk`` as ``limit`` allows. + + Whole clauses are moved when they bring ``chunk`` up to ``_MIN_CHUNK_CHARS``, + so the cut lands at punctuation; otherwise words are, and the cut lands + mid clause. + """ + topped, rest = _fill(chunk, _CLAUSE_BOUNDARY.split(piece), limit) + if len(topped) >= _MIN_CHUNK_CHARS: + return topped, rest + return _fill(chunk, piece.split(), limit) + + def split_long_text(text: str, limit: int = CHUNK_CHARS) -> list[str]: - """Pack prose into sentence-aware chunks no longer than ``limit``.""" + """Pack prose into chunks no longer than ``limit``. + + Chunks break between sentences where they can. A sentence longer than + ``limit`` breaks at clause punctuation, and only a clause still longer than + that breaks between words. A chunk under ``_MIN_CHUNK_CHARS`` that the next + piece won't fit beside takes that piece's opening clauses, or words where no + clause fits, rather than going out alone. + """ sentences = [part.strip() for part in _SENTENCE_BOUNDARY.split(text) if part.strip()] chunks: list[str] = [] current = "" - for sentence in sentences: - pieces = [sentence] - if len(sentence) > limit: - words = sentence.split() - pieces = [] - piece = "" - for word in words: - candidate = f"{piece} {word}".strip() - if piece and len(candidate) > limit: - pieces.append(piece) - piece = word - else: - piece = candidate - if piece: - pieces.append(piece) - for piece in pieces: - candidate = f"{current} {piece}".strip() - if current and len(candidate) > limit: - chunks.append(current) - current = piece - else: - current = candidate + for piece in (piece for sentence in sentences for piece in _sentence_pieces(sentence, limit)): + candidate = f"{current} {piece}".strip() + if not current or len(candidate) <= limit: + current = candidate + continue + if len(current) < _MIN_CHUNK_CHARS: + current, piece = _top_up(current, piece, limit) + chunks.append(current) + current = piece if current: chunks.append(current) return chunks diff --git a/tests/test_mlx_backend.py b/tests/test_mlx_backend.py index dec05e0..c8c8d79 100644 --- a/tests/test_mlx_backend.py +++ b/tests/test_mlx_backend.py @@ -347,6 +347,73 @@ def test_overlong_sentence_splits_on_words(self): assert all(len(chunk) <= 50 for chunk in chunks) assert " ".join(chunks).split() == ["word"] * 200 + @pytest.mark.parametrize("separator", [",", ";", ":", " —", " –", " -", " --"]) + def test_overlong_sentence_splits_at_each_clause_separator(self, separator): + first = " ".join(["alpha"] * 20) + separator + second = " ".join(["beta"] * 20) + "." + assert mlx_backend.split_long_text(f"{first} {second}") == [first, second] + + def test_hyphen_without_a_space_before_it_is_not_a_clause_boundary(self): + clause = " ".join(["alpha"] * 20) + " pre- and post-war " + " ".join(["beta"] * 20) + "," + chunks = mlx_backend.split_long_text(f"{clause} gamma.") + assert chunks[0].split()[-1] != "pre-" + + def test_overlong_sentence_splits_at_clauses_before_words(self): + sentence = ( + ", ".join(f"this is clause number {index} of one long sentence" for index in range(8)) + "." + ) + chunks = mlx_backend.split_long_text(sentence) + assert len(chunks) > 1 + assert all(len(chunk) <= mlx_backend.CHUNK_CHARS for chunk in chunks) + assert all(chunk.endswith((",", ".")) for chunk in chunks) + assert " ".join(chunks) == sentence + + def test_short_sentence_takes_words_from_a_long_one_rather_than_going_alone(self): + long_sentence = " ".join(["alpha"] * 28) + "." + chunks = mlx_backend.split_long_text(f"Yes. {long_sentence}") + assert chunks[0].startswith("Yes. alpha") + assert len(chunks[0]) <= mlx_backend.CHUNK_CHARS + assert " ".join(chunks) == f"Yes. {long_sentence}" + + def test_short_sentence_takes_whole_clauses_when_they_fill_it(self): + clause = " ".join(["alpha"] * 12) + "," + long_sentence = f"{clause} " + " ".join(["beta"] * 19) + "." + assert len(long_sentence) <= mlx_backend.CHUNK_CHARS < len(f"Yes. {long_sentence}") + chunks = mlx_backend.split_long_text(f"Yes. {long_sentence}") + assert chunks[0] == f"Yes. {clause}" + + def test_short_sentence_takes_words_when_the_opening_clause_is_too_short(self): + long_sentence = "Well, " + " ".join(["alpha"] * 27) + "." + assert len(long_sentence) <= mlx_backend.CHUNK_CHARS < len(f"Yes. {long_sentence}") + chunks = mlx_backend.split_long_text(f"Yes. {long_sentence}") + assert chunks[0].startswith("Yes. Well, alpha") + assert len(chunks[0]) >= mlx_backend._MIN_CHUNK_CHARS + + @pytest.mark.parametrize(("first_length", "topped_up"), [(59, True), (60, False)]) + def test_top_up_applies_only_below_the_minimum_chunk_length(self, first_length, topped_up): + first = "x" * (first_length - 1) + "." + second = " ".join(["alpha"] * 25) + "." + chunks = mlx_backend.split_long_text(f"{first} {second}") + assert (chunks[0] != first) is topped_up + + def test_chunks_long_enough_to_stand_alone_are_not_topped_up(self): + first = "The first sentence is comfortably longer than the minimum chunk length, so it stands." + second = ( + "The second " + " ".join(["sentence"] * 10) + " is long enough that the two cannot share a chunk." + ) + assert mlx_backend.split_long_text(f"{first} {second}") == [first, second] + + def test_paragraph_with_commas_breaks_only_at_punctuation(self): + text = ( + "Switched to the four bit model. The daemon is back under launchd, and this paragraph is long enough " + "to stream as more than one chunk, so you should hear it carry on into the second chunk without the " + "pause you were getting before. If there is still a gap between the sentences, let me know and we " + "can look at buffering ahead." + ) + chunks = mlx_backend.split_long_text(text) + assert all(chunk.endswith((",", ".")) for chunk in chunks) + assert " ".join(chunks) == text + class TestMlxThread: def test_load_and_generation_share_one_thread_across_callers(self, mocker, tmp_path):