kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

feat: add Kokoro TTS, ranking page, direct HTTP strategy, and chapter-number fix

Browse Source 
- Add Kokoro-FastAPI TTS integration to the chapter reader UI:
  - Browser-side MSE streaming with paragraph-level click-to-start
  - Voice selector, speed slider, auto-next with prefetch of the next chapter
  - New GET /ui/chapter-text endpoint that strips Markdown and serves plain text

- Add ranking page (novelfire /ranking scraper, WriteRanking/ReadRankingItems
  in writer, GET /ranking + POST /ranking/refresh + GET /ranking/view routes)
  with local-library annotation and one-click scrape buttons

- Add StrategyDirect (plain HTTP client) as a new browser strategy; the
  default strategy is now 'direct' for chapter fetching and 'content'
  for chapter-list URL retrieval (split via BROWSERLESS_URL_STRATEGY)

- Fix chapter numbering bug: numbers are now derived from the URL path
  (/chapter-N) rather than list position, correcting newest-first ordering

- Add 'refresh <slug>' CLI sub-command to re-scrape a book from its saved
  source_url without knowing the original URL

- Extend NovelScraper interface with RankingProvider (ScrapeRanking)

- Tune scraper timeouts: wait-for-selector reduced to 5 s, GotoOptions
  timeout set to 60 s, content/scrape client defaults raised to 90 s

- Add cover extraction fix (figure.cover > img rather than bare img.cover)

- Add AGENTS.md and .aiignore for AI tooling context

- Add integration tests for browser client and novelfire scraper (build
  tag: integration) and unit tests for chapterNumberFromURL and pagination
This commit is contained in:
Admin
2026-03-01 12:25:16 +05:00
parent e6e6f7dc4d
commit 7879a51fe3
17 changed files with 2816 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
scraper/cmd/scraper/main.go
View File

@@ -17,6 +17,8 @@
// SCRAPER_WORKERS Chapter goroutine count (default: NumCPU)
// SCRAPER_STATIC_ROOT Output directory (default: ./static/books)
// SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
// KOKORO_URL Kokoro-FastAPI base URL (default: "")
// KOKORO_VOICE Default TTS voice (default: af_bella)
// LOG_LEVEL debug | info | warn | error (default: info)
package main
@@ -30,11 +32,13 @@ import (
"strconv"
"strings"
"syscall"
"time"
"github.com/libnovel/scraper/internal/browser"
"github.com/libnovel/scraper/internal/novelfire"
"github.com/libnovel/scraper/internal/orchestrator"
"github.com/libnovel/scraper/internal/server"
"github.com/libnovel/scraper/internal/writer"
)
func main() {
@@ -73,11 +77,18 @@ func run(log *slog.Logger) error {
browserCfg.MaxConcurrent = n
}
}
if s := os.Getenv("BROWSERLESS_TIMEOUT"); s != "" {
if n, err := strconv.Atoi(s); err == nil && n > 0 {
browserCfg.Timeout = time.Duration(n) * time.Second
}
}
strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyContent))))
strategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_STRATEGY", string(browser.StrategyDirect))))
urlStrategy := browser.Strategy(strings.ToLower(envOr("BROWSERLESS_URL_STRATEGY", string(browser.StrategyContent))))
bc := newBrowserClient(strategy, browserCfg)
urlClient := newBrowserClient(urlStrategy, browserCfg)
nf := novelfire.New(bc, log)
nf := novelfire.New(bc, log, urlClient)
workers := 0
if s := os.Getenv("SCRAPER_WORKERS"); s != "" {
@@ -114,15 +125,44 @@ func run(log *slog.Logger) error {
o := orchestrator.New(oCfg, nf, log)
return o.Run(ctx)
case "refresh":
// refresh <slug> - re-scrape a book from its saved source_url
if len(args) < 2 {
return fmt.Errorf("refresh command requires a book slug argument")
}
slug := args[1]
w := writer.New(oCfg.StaticRoot)
meta, ok, err := w.ReadMetadata(slug)
if err != nil {
return fmt.Errorf("failed to read metadata for %s: %w", slug, err)
}
if !ok {
return fmt.Errorf("book %q not found in %s", slug, oCfg.StaticRoot)
}
if meta.SourceURL == "" {
return fmt.Errorf("book %q has no source_url in metadata", slug)
}
oCfg.SingleBookURL = meta.SourceURL
log.Info("refreshing book from source_url",
"slug", slug,
"source_url", meta.SourceURL,
)
o := orchestrator.New(oCfg, nf, log)
return o.Run(ctx)
case "serve":
addr := envOr("SCRAPER_HTTP_ADDR", ":8080")
kokoroURL := envOr("KOKORO_URL", "")
kokoroVoice := envOr("KOKORO_VOICE", "af_bella")
log.Info("starting HTTP server",
"addr", addr,
"strategy", strategy,
"workers", workers,
"max_concurrent", browserCfg.MaxConcurrent,
"kokoro_url", kokoroURL,
"kokoro_voice", kokoroVoice,
)
srv := server.New(addr, oCfg, nf, log)
srv := server.New(addr, oCfg, nf, log, kokoroURL, kokoroVoice)
return srv.ListenAndServe(ctx)
default:
@@ -136,6 +176,8 @@ func newBrowserClient(strategy browser.Strategy, cfg browser.Config) browser.Bro
return browser.NewScrapeClient(cfg)
case browser.StrategyCDP:
return browser.NewCDPClient(cfg)
case browser.StrategyDirect:
return browser.NewDirectHTTPClient(cfg)
default:
return browser.NewContentClient(cfg)
}
@@ -153,16 +195,21 @@ func printUsage() {
Commands:
run [--url <book-url>] One-shot: scrape full catalogue, or a single book
refresh <slug> Re-scrape a book from its saved source_url
serve Start HTTP server (POST /scrape, POST /scrape/book)
Environment variables:
BROWSERLESS_URL Browserless base URL (default: http://localhost:3000)
BROWSERLESS_TOKEN API token (default: "")
BROWSERLESS_STRATEGY content | scrape | cdp (default: content)
BROWSERLESS_STRATEGY content|scrape|cdp|direct (default: direct)
BROWSERLESS_URL_STRATEGY Strategy for URL retrieval (default: content)
BROWSERLESS_MAX_CONCURRENT Max simultaneous sessions (default: 5)
BROWSERLESS_TIMEOUT HTTP request timeout sec (default: 90)
SCRAPER_WORKERS Chapter goroutines (default: NumCPU = %d)
SCRAPER_STATIC_ROOT Output directory (default: ./static/books)
SCRAPER_HTTP_ADDR HTTP listen address (default: :8080)
KOKORO_URL Kokoro-FastAPI base URL (default: "", TTS disabled)
KOKORO_VOICE Default TTS voice (default: af_bella)
LOG_LEVEL debug|info|warn|error (default: info)
`, runtime.NumCPU())
}