feat: add Kokoro TTS, ranking page, direct HTTP strategy, and chapter-number fix

- Add Kokoro-FastAPI TTS integration to the chapter reader UI:
  - Browser-side MSE streaming with paragraph-level click-to-start
  - Voice selector, speed slider, auto-next with prefetch of the next chapter
  - New GET /ui/chapter-text endpoint that strips Markdown and serves plain text

- Add ranking page (novelfire /ranking scraper, WriteRanking/ReadRankingItems
  in writer, GET /ranking + POST /ranking/refresh + GET /ranking/view routes)
  with local-library annotation and one-click scrape buttons

- Add StrategyDirect (plain HTTP client) as a new browser strategy; the
  default strategy is now 'direct' for chapter fetching and 'content'
  for chapter-list URL retrieval (split via BROWSERLESS_URL_STRATEGY)

- Fix chapter numbering bug: numbers are now derived from the URL path
  (/chapter-N) rather than list position, correcting newest-first ordering

- Add 'refresh <slug>' CLI sub-command to re-scrape a book from its saved
  source_url without knowing the original URL

- Extend NovelScraper interface with RankingProvider (ScrapeRanking)

- Tune scraper timeouts: wait-for-selector reduced to 5 s, GotoOptions
  timeout set to 60 s, content/scrape client defaults raised to 90 s

- Add cover extraction fix (figure.cover > img rather than bare img.cover)

- Add AGENTS.md and .aiignore for AI tooling context

- Add integration tests for browser client and novelfire scraper (build
  tag: integration) and unit tests for chapterNumberFromURL and pagination
This commit is contained in:
Admin
2026-03-01 12:25:16 +05:00
parent e6e6f7dc4d
commit 7879a51fe3
17 changed files with 2816 additions and 88 deletions

View File

@@ -13,6 +13,7 @@ import (
"fmt"
"log/slog"
"net/http"
"strconv"
"sync"
"time"
@@ -23,23 +24,27 @@ import (
// Server wraps an HTTP mux with the scraping endpoints.
type Server struct {
addr string
oCfg orchestrator.Config
novel scraper.NovelScraper
log *slog.Logger
writer *writer.Writer
mu sync.Mutex
running bool
addr string
oCfg orchestrator.Config
novel scraper.NovelScraper
log *slog.Logger
writer *writer.Writer
mu sync.Mutex
running bool
kokoroURL string // Kokoro-FastAPI base URL, e.g. http://kokoro:8880
kokoroVoice string // default voice, e.g. af_bella
}
// New creates a new Server.
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger) *Server {
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, kokoroURL, kokoroVoice string) *Server {
return &Server{
addr: addr,
oCfg: oCfg,
novel: novel,
log: log,
writer: writer.New(oCfg.StaticRoot),
addr: addr,
oCfg: oCfg,
novel: novel,
log: log,
writer: writer.New(oCfg.StaticRoot),
kokoroURL: kokoroURL,
kokoroVoice: kokoroVoice,
}
}
@@ -52,16 +57,21 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
// UI routes
mux.HandleFunc("GET /", s.handleHome)
mux.HandleFunc("GET /ranking", s.handleRanking)
mux.HandleFunc("POST /ranking/refresh", s.handleRankingRefresh)
mux.HandleFunc("GET /ranking/view", s.handleRankingView)
mux.HandleFunc("GET /books/{slug}", s.handleBook)
mux.HandleFunc("GET /books/{slug}/chapters/{n}", s.handleChapter)
mux.HandleFunc("POST /ui/scrape/book", s.handleUIScrapeBook)
mux.HandleFunc("GET /ui/scrape/status", s.handleUIScrapeStatus)
// Plain-text chapter content for browser-side TTS
mux.HandleFunc("GET /ui/chapter-text/{slug}/{n}", s.handleChapterText)
srv := &http.Server{
Addr: s.addr,
Handler: mux,
ReadTimeout: 15 * time.Second,
WriteTimeout: 15 * time.Second,
WriteTimeout: 60 * time.Second,
IdleTimeout: 60 * time.Second,
}
@@ -85,6 +95,25 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
}
// handleChapterText returns the plain text of a chapter (markdown stripped)
// for browser-side TTS. The browser POSTs this directly to Kokoro-FastAPI.
func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 {
http.NotFound(w, r)
return
}
raw, err := s.writer.ReadChapter(slug, n)
if err != nil {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Header().Set("Cache-Control", "no-store")
fmt.Fprint(w, stripMarkdown(raw))
}
func (s *Server) handleScrapeCatalogue(w http.ResponseWriter, r *http.Request) {
cfg := s.oCfg
cfg.SingleBookURL = "" // full catalogue