e0dec05885
Split chapter text into ~1000-char sentence-boundary chunks before sending to kokoro-fastapi. Each chunk is generated individually and the raw MP3 bytes are concatenated. This prevents the EOF / timeout failures that occur when the server receives a very large single request (e.g. a full PDF 'Full Text' chapter). chunkText() breaks at sentence endings (. ! ? newlines) to preserve natural speech flow.
72 lines
2.2 KiB
Go
72 lines
2.2 KiB
Go
package runner
|
|
|
|
import (
|
|
"regexp"
|
|
"strings"
|
|
)
|
|
|
|
// stripMarkdown removes common markdown syntax from src, returning plain text
|
|
// suitable for TTS. Mirrors the helper in the scraper's server package.
|
|
func stripMarkdown(src string) string {
|
|
src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "")
|
|
src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "")
|
|
src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "")
|
|
src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "")
|
|
src = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(src, "$1")
|
|
src = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`).ReplaceAllString(src, "")
|
|
src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "")
|
|
src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "")
|
|
src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n")
|
|
return strings.TrimSpace(src)
|
|
}
|
|
|
|
// chunkText splits text into chunks of at most maxChars characters, breaking
|
|
// at sentence boundaries (". ", "! ", "? ", "\n") so that the TTS service
|
|
// receives natural prose fragments rather than mid-sentence cuts.
|
|
//
|
|
// If a single sentence exceeds maxChars it is included as its own chunk —
|
|
// never silently truncated.
|
|
func chunkText(text string, maxChars int) []string {
|
|
if len(text) <= maxChars {
|
|
return []string{text}
|
|
}
|
|
|
|
// Sentence-boundary delimiters — we split AFTER these sequences.
|
|
// Order matters: longer sequences first.
|
|
delimiters := []string{".\n", "!\n", "?\n", ". ", "! ", "? ", "\n\n", "\n"}
|
|
|
|
var chunks []string
|
|
remaining := text
|
|
|
|
for len(remaining) > 0 {
|
|
if len(remaining) <= maxChars {
|
|
chunks = append(chunks, strings.TrimSpace(remaining))
|
|
break
|
|
}
|
|
|
|
// Find the last sentence boundary within the maxChars window.
|
|
window := remaining[:maxChars]
|
|
cutAt := -1
|
|
for _, delim := range delimiters {
|
|
idx := strings.LastIndex(window, delim)
|
|
if idx > 0 && idx+len(delim) > cutAt {
|
|
cutAt = idx + len(delim)
|
|
}
|
|
}
|
|
|
|
if cutAt <= 0 {
|
|
// No boundary found — hard-break at maxChars to avoid infinite loop.
|
|
cutAt = maxChars
|
|
}
|
|
|
|
chunk := strings.TrimSpace(remaining[:cutAt])
|
|
if chunk != "" {
|
|
chunks = append(chunks, chunk)
|
|
}
|
|
remaining = strings.TrimSpace(remaining[cutAt:])
|
|
}
|
|
|
|
return chunks
|
|
}
|
|
|