feat: add Kokoro TTS, ranking page, direct HTTP strategy, and chapter-number fix
- Add Kokoro-FastAPI TTS integration to the chapter reader UI: - Browser-side MSE streaming with paragraph-level click-to-start - Voice selector, speed slider, auto-next with prefetch of the next chapter - New GET /ui/chapter-text endpoint that strips Markdown and serves plain text - Add ranking page (novelfire /ranking scraper, WriteRanking/ReadRankingItems in writer, GET /ranking + POST /ranking/refresh + GET /ranking/view routes) with local-library annotation and one-click scrape buttons - Add StrategyDirect (plain HTTP client) as a new browser strategy; the default strategy is now 'direct' for chapter fetching and 'content' for chapter-list URL retrieval (split via BROWSERLESS_URL_STRATEGY) - Fix chapter numbering bug: numbers are now derived from the URL path (/chapter-N) rather than list position, correcting newest-first ordering - Add 'refresh <slug>' CLI sub-command to re-scrape a book from its saved source_url without knowing the original URL - Extend NovelScraper interface with RankingProvider (ScrapeRanking) - Tune scraper timeouts: wait-for-selector reduced to 5 s, GotoOptions timeout set to 60 s, content/scrape client defaults raised to 90 s - Add cover extraction fix (figure.cover > img rather than bare img.cover) - Add AGENTS.md and .aiignore for AI tooling context - Add integration tests for browser client and novelfire scraper (build tag: integration) and unit tests for chapterNumberFromURL and pagination
This commit is contained in:
@@ -21,6 +21,10 @@ const (
|
||||
// DevTools Protocol). Most powerful; required for complex interactions
|
||||
// (clicking, scrolling, waiting for network idle, etc.).
|
||||
StrategyCDP Strategy = "cdp"
|
||||
|
||||
// StrategyDirect uses a plain HTTP client to fetch HTML directly.
|
||||
// Suitable for sites that don't require JavaScript rendering.
|
||||
StrategyDirect Strategy = "direct"
|
||||
)
|
||||
|
||||
// WaitForSelector describes the waitForSelector option sent to Browserless.
|
||||
@@ -29,12 +33,20 @@ type WaitForSelector struct {
|
||||
Timeout int `json:"timeout,omitempty"` // ms
|
||||
}
|
||||
|
||||
// GotoOptions controls page navigation behavior.
|
||||
type GotoOptions struct {
|
||||
Timeout int `json:"timeout,omitempty"` // ms
|
||||
WaitUntil string `json:"waitUntil,omitempty"` // e.g., "networkidle2", "load"
|
||||
}
|
||||
|
||||
// ContentRequest is the body sent to POST /content.
|
||||
type ContentRequest struct {
|
||||
URL string `json:"url"`
|
||||
WaitFor *WaitForSelector `json:"waitForSelector,omitempty"`
|
||||
WaitForTimeout int `json:"waitForTimeout,omitempty"` // ms
|
||||
RejectResourceTypes []string `json:"rejectResourceTypes,omitempty"` // e.g. ["image","stylesheet"]
|
||||
GotoOptions *GotoOptions `json:"gotoOptions,omitempty"`
|
||||
BestAttempt bool `json:"bestAttempt,omitempty"` // return partial content on timeout/error
|
||||
}
|
||||
|
||||
// ScrapeElement is one element descriptor inside a ScrapeRequest.
|
||||
@@ -45,9 +57,10 @@ type ScrapeElement struct {
|
||||
|
||||
// ScrapeRequest is the body sent to POST /scrape.
|
||||
type ScrapeRequest struct {
|
||||
URL string `json:"url"`
|
||||
Elements []ScrapeElement `json:"elements"`
|
||||
WaitFor *WaitForSelector `json:"waitForSelector,omitempty"`
|
||||
URL string `json:"url"`
|
||||
Elements []ScrapeElement `json:"elements"`
|
||||
WaitFor *WaitForSelector `json:"waitForSelector,omitempty"`
|
||||
GotoOptions *GotoOptions `json:"gotoOptions,omitempty"`
|
||||
}
|
||||
|
||||
// ScrapeResult is one entry in the response from POST /scrape.
|
||||
|
||||
Reference in New Issue
Block a user