Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -241,8 +241,14 @@ the model. They are rejected with an error unless the `mlx` backend is in use,
both by `speak` and by a daemon running Kokoro. `--voice` and `--lang` apply to
Kokoro only. Playback streams in chunks of about two sentences (up to ~170
characters), so the first audio arrives once the first chunk is generated;
short chunks also stop models drifting off-script on long text. Models slower
than real time will pause between chunks.
short chunks also stop models drifting off-script on long text. A sentence too
long for one chunk is split at a comma, semicolon, colon or dash where it has
one. A chunk under 60 characters is filled up to the limit from the start of
the next sentence or clause rather than sent alone, at a clause boundary where
that reaches 60 characters and between words otherwise, since a short chunk
ahead of a long one leaves a pause while the long one generates. That means a
reply opening on a short sentence waits for a full-size first chunk. Models
slower than real time will still pause between chunks.

### Model adapters

Expand Down
96 changes: 73 additions & 23 deletions stackvox/mlx_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,17 @@
# bring the first streamed audio forward.
CHUNK_CHARS = 170
_SENTENCE_BOUNDARY = re.compile(r"(?<=[.!?])\s+|\n+")
# Over-long sentences, and the next sentence when a short chunk is topped up,
# are cut here before falling back to word boundaries, so the pause between
# chunks lands where a speaker would take a breath. A typed
# dash counts only with space either side, so hyphenated words stay whole.
_CLAUSE_BOUNDARY = re.compile(r"(?<=[,;:—–])\s+|(?<=\s-)\s+|(?<=\s--)\s+")
# Streaming leaves a gap whenever a chunk takes longer to generate than the one
# before it takes to play, so a short sentence sent alone ahead of a long one
# ("Yes." then 170 characters) is the worst case for models near real time.
# A chunk shorter than this that the next piece won't fit beside takes the
# opening of that piece, up to the chunk limit, instead.
_MIN_CHUNK_CHARS = 60

# MLX keeps freed GPU buffers in its own cache for reuse, and by default lets it
# grow to the device's recommended working set (tens of GB on a large Mac). A
Expand Down Expand Up @@ -80,33 +91,72 @@ def swapped(owner: type, name: str, value: Any) -> Iterator[None]:
delattr(owner, name)


def _split_words(text: str, limit: int) -> list[str]:
pieces: list[str] = []
piece = ""
for word in text.split():
candidate = f"{piece} {word}".strip()
if piece and len(candidate) > limit:
pieces.append(piece)
piece = word
else:
piece = candidate
if piece:
pieces.append(piece)
return pieces


def _sentence_pieces(sentence: str, limit: int) -> list[str]:
if len(sentence) <= limit:
return [sentence]
pieces: list[str] = []
for clause in (part.strip() for part in _CLAUSE_BOUNDARY.split(sentence)):
if clause:
pieces.extend([clause] if len(clause) <= limit else _split_words(clause, limit))
return pieces


def _fill(chunk: str, parts: list[str], limit: int) -> tuple[str, str]:
taken = 0
while taken < len(parts) and len(" ".join([chunk, *parts[: taken + 1]])) <= limit:
taken += 1
return " ".join([chunk, *parts[:taken]]), " ".join(parts[taken:])


def _top_up(chunk: str, piece: str, limit: int) -> tuple[str, str]:
"""Move as much of ``piece``'s opening onto ``chunk`` as ``limit`` allows.

Whole clauses are moved when they bring ``chunk`` up to ``_MIN_CHUNK_CHARS``,
so the cut lands at punctuation; otherwise words are, and the cut lands
mid clause.
"""
topped, rest = _fill(chunk, _CLAUSE_BOUNDARY.split(piece), limit)
if len(topped) >= _MIN_CHUNK_CHARS:
return topped, rest
return _fill(chunk, piece.split(), limit)


def split_long_text(text: str, limit: int = CHUNK_CHARS) -> list[str]:
"""Pack prose into sentence-aware chunks no longer than ``limit``."""
"""Pack prose into chunks no longer than ``limit``.

Chunks break between sentences where they can. A sentence longer than
``limit`` breaks at clause punctuation, and only a clause still longer than
that breaks between words. A chunk under ``_MIN_CHUNK_CHARS`` that the next
piece won't fit beside takes that piece's opening clauses, or words where no
clause fits, rather than going out alone.
"""
sentences = [part.strip() for part in _SENTENCE_BOUNDARY.split(text) if part.strip()]
chunks: list[str] = []
current = ""
for sentence in sentences:
pieces = [sentence]
if len(sentence) > limit:
words = sentence.split()
pieces = []
piece = ""
for word in words:
candidate = f"{piece} {word}".strip()
if piece and len(candidate) > limit:
pieces.append(piece)
piece = word
else:
piece = candidate
if piece:
pieces.append(piece)
for piece in pieces:
candidate = f"{current} {piece}".strip()
if current and len(candidate) > limit:
chunks.append(current)
current = piece
else:
current = candidate
for piece in (piece for sentence in sentences for piece in _sentence_pieces(sentence, limit)):
candidate = f"{current} {piece}".strip()
if not current or len(candidate) <= limit:
current = candidate
continue
if len(current) < _MIN_CHUNK_CHARS:
current, piece = _top_up(current, piece, limit)
chunks.append(current)
current = piece
if current:
chunks.append(current)
return chunks
Expand Down
67 changes: 67 additions & 0 deletions tests/test_mlx_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,73 @@ def test_overlong_sentence_splits_on_words(self):
assert all(len(chunk) <= 50 for chunk in chunks)
assert " ".join(chunks).split() == ["word"] * 200

@pytest.mark.parametrize("separator", [",", ";", ":", " —", " –", " -", " --"])
def test_overlong_sentence_splits_at_each_clause_separator(self, separator):
first = " ".join(["alpha"] * 20) + separator
second = " ".join(["beta"] * 20) + "."
assert mlx_backend.split_long_text(f"{first} {second}") == [first, second]

def test_hyphen_without_a_space_before_it_is_not_a_clause_boundary(self):
clause = " ".join(["alpha"] * 20) + " pre- and post-war " + " ".join(["beta"] * 20) + ","
chunks = mlx_backend.split_long_text(f"{clause} gamma.")
assert chunks[0].split()[-1] != "pre-"

def test_overlong_sentence_splits_at_clauses_before_words(self):
sentence = (
", ".join(f"this is clause number {index} of one long sentence" for index in range(8)) + "."
)
chunks = mlx_backend.split_long_text(sentence)
assert len(chunks) > 1
assert all(len(chunk) <= mlx_backend.CHUNK_CHARS for chunk in chunks)
assert all(chunk.endswith((",", ".")) for chunk in chunks)
assert " ".join(chunks) == sentence

def test_short_sentence_takes_words_from_a_long_one_rather_than_going_alone(self):
long_sentence = " ".join(["alpha"] * 28) + "."
chunks = mlx_backend.split_long_text(f"Yes. {long_sentence}")
assert chunks[0].startswith("Yes. alpha")
assert len(chunks[0]) <= mlx_backend.CHUNK_CHARS
assert " ".join(chunks) == f"Yes. {long_sentence}"

def test_short_sentence_takes_whole_clauses_when_they_fill_it(self):
clause = " ".join(["alpha"] * 12) + ","
long_sentence = f"{clause} " + " ".join(["beta"] * 19) + "."
assert len(long_sentence) <= mlx_backend.CHUNK_CHARS < len(f"Yes. {long_sentence}")
chunks = mlx_backend.split_long_text(f"Yes. {long_sentence}")
assert chunks[0] == f"Yes. {clause}"

def test_short_sentence_takes_words_when_the_opening_clause_is_too_short(self):
long_sentence = "Well, " + " ".join(["alpha"] * 27) + "."
assert len(long_sentence) <= mlx_backend.CHUNK_CHARS < len(f"Yes. {long_sentence}")
chunks = mlx_backend.split_long_text(f"Yes. {long_sentence}")
assert chunks[0].startswith("Yes. Well, alpha")
assert len(chunks[0]) >= mlx_backend._MIN_CHUNK_CHARS

@pytest.mark.parametrize(("first_length", "topped_up"), [(59, True), (60, False)])
def test_top_up_applies_only_below_the_minimum_chunk_length(self, first_length, topped_up):
first = "x" * (first_length - 1) + "."
second = " ".join(["alpha"] * 25) + "."
chunks = mlx_backend.split_long_text(f"{first} {second}")
assert (chunks[0] != first) is topped_up

def test_chunks_long_enough_to_stand_alone_are_not_topped_up(self):
first = "The first sentence is comfortably longer than the minimum chunk length, so it stands."
second = (
"The second " + " ".join(["sentence"] * 10) + " is long enough that the two cannot share a chunk."
)
assert mlx_backend.split_long_text(f"{first} {second}") == [first, second]

def test_paragraph_with_commas_breaks_only_at_punctuation(self):
text = (
"Switched to the four bit model. The daemon is back under launchd, and this paragraph is long enough "
"to stream as more than one chunk, so you should hear it carry on into the second chunk without the "
"pause you were getting before. If there is still a gap between the sentences, let me know and we "
"can look at buffering ahead."
)
chunks = mlx_backend.split_long_text(text)
assert all(chunk.endswith((",", ".")) for chunk in chunks)
assert " ".join(chunks) == text


class TestMlxThread:
def test_load_and_generation_share_one_thread_across_callers(self, mocker, tmp_path):
Expand Down
Loading