feat: add Kokoro TTS, ranking page, direct HTTP strategy, and chapter-number fix

- Add Kokoro-FastAPI TTS integration to the chapter reader UI:
  - Browser-side MSE streaming with paragraph-level click-to-start
  - Voice selector, speed slider, auto-next with prefetch of the next chapter
  - New GET /ui/chapter-text endpoint that strips Markdown and serves plain text

- Add ranking page (novelfire /ranking scraper, WriteRanking/ReadRankingItems
  in writer, GET /ranking + POST /ranking/refresh + GET /ranking/view routes)
  with local-library annotation and one-click scrape buttons

- Add StrategyDirect (plain HTTP client) as a new browser strategy; the
  default strategy is now 'direct' for chapter fetching and 'content'
  for chapter-list URL retrieval (split via BROWSERLESS_URL_STRATEGY)

- Fix chapter numbering bug: numbers are now derived from the URL path
  (/chapter-N) rather than list position, correcting newest-first ordering

- Add 'refresh <slug>' CLI sub-command to re-scrape a book from its saved
  source_url without knowing the original URL

- Extend NovelScraper interface with RankingProvider (ScrapeRanking)

- Tune scraper timeouts: wait-for-selector reduced to 5 s, GotoOptions
  timeout set to 60 s, content/scrape client defaults raised to 90 s

- Add cover extraction fix (figure.cover > img rather than bare img.cover)

- Add AGENTS.md and .aiignore for AI tooling context

- Add integration tests for browser client and novelfire scraper (build
  tag: integration) and unit tests for chapterNumberFromURL and pagination

scraper/internal/browser/content_scrape.go

@@ -65,7 +65,7 @@ type contentClient struct {
 // NewContentClient returns a BrowserClient that uses POST /content.
 func NewContentClient(cfg Config) BrowserClient {
 	if cfg.Timeout == 0 {
-		cfg.Timeout = 60 * time.Second
+		cfg.Timeout = 90 * time.Second
 	}
 	return &contentClient{
 		cfg:  cfg,
@@ -135,7 +135,7 @@ type scrapeClient struct {
 // NewScrapeClient returns a BrowserClient that uses POST /scrape.
 func NewScrapeClient(cfg Config) BrowserClient {
 	if cfg.Timeout == 0 {
-		cfg.Timeout = 60 * time.Second
+		cfg.Timeout = 90 * time.Second
 	}
 	return &scrapeClient{
 		cfg:  cfg,

scraper/internal/browser/http.go

@@ -0,0 +1,68 @@
+package browser
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+)
+
+type httpClient struct {
+	cfg  Config
+	http *http.Client
+	sem  chan struct{}
+}
+
+func NewDirectHTTPClient(cfg Config) BrowserClient {
+	if cfg.Timeout == 0 {
+		cfg.Timeout = 30 * time.Second
+	}
+	return &httpClient{
+		cfg:  cfg,
+		http: &http.Client{Timeout: cfg.Timeout},
+		sem:  makeSem(cfg.MaxConcurrent),
+	}
+}
+
+func (c *httpClient) Strategy() Strategy { return StrategyDirect }
+
+func (c *httpClient) GetContent(ctx context.Context, req ContentRequest) (string, error) {
+	if err := acquire(ctx, c.sem); err != nil {
+		return "", fmt.Errorf("http: semaphore: %w", err)
+	}
+	defer release(c.sem)
+
+	httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, req.URL, nil)
+	if err != nil {
+		return "", fmt.Errorf("http: build request: %w", err)
+	}
+	httpReq.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
+	httpReq.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+	httpReq.Header.Set("Accept-Language", "en-US,en;q=0.5")
+
+	resp, err := c.http.Do(httpReq)
+	if err != nil {
+		return "", fmt.Errorf("http: do request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		b, _ := io.ReadAll(resp.Body)
+		return "", fmt.Errorf("http: unexpected status %d: %s", resp.StatusCode, b)
+	}
+
+	raw, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("http: read body: %w", err)
+	}
+	return string(raw), nil
+}
+
+func (c *httpClient) ScrapePage(_ context.Context, _ ScrapeRequest) (ScrapeResponse, error) {
+	return ScrapeResponse{}, fmt.Errorf("http client does not support ScrapePage; use browserless")
+}
+
+func (c *httpClient) CDPSession(_ context.Context, _ string, _ CDPSessionFunc) error {
+	return fmt.Errorf("http client does not support CDP; use browserless")
+}

scraper/internal/browser/integration_test.go

@@ -0,0 +1,152 @@
+//go:build integration
+
+// Integration tests for the Browserless /content API.
+//
+// These tests require a live Browserless instance and are gated behind the
+// "integration" build tag so they never run in normal `go test ./...` passes.
+//
+// Run them with:
+//
+//	BROWSERLESS_URL=http://localhost:3000 \
+//	BROWSERLESS_TOKEN=your-token \          # omit if auth is disabled
+//	go test -v -tags integration -timeout 120s \
+//	    github.com/libnovel/scraper/internal/browser
+package browser_test
+
+import (
+	"context"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/libnovel/scraper/internal/browser"
+)
+
+// chapterURL is the novelfire chapter used in every integration sub-test.
+const chapterURL = "https://novelfire.net/book/a-dragon-against-the-whole-world/chapter-1"
+
+// newIntegrationClient reads BROWSERLESS_URL / BROWSERLESS_TOKEN from the
+// environment and returns a configured contentClient.
+// The test is skipped when BROWSERLESS_URL is not set.
+func newIntegrationClient(t *testing.T) browser.BrowserClient {
+	t.Helper()
+	baseURL := os.Getenv("BROWSERLESS_URL")
+	if baseURL == "" {
+		t.Skip("BROWSERLESS_URL not set — skipping integration test")
+	}
+	return browser.NewContentClient(browser.Config{
+		BaseURL: baseURL,
+		Token:   os.Getenv("BROWSERLESS_TOKEN"),
+		// Use a generous per-request HTTP timeout so the wait-for-selector
+		// (75 s) doesn't get cut off by the transport layer.
+		Timeout:       120 * time.Second,
+		MaxConcurrent: 1,
+	})
+}
+
+// TestIntegration_ChapterContent_ReturnsHTML verifies that a POST /content
+// request with the production wait-for-selector settings succeeds and that the
+// returned HTML contains the #content div expected on novelfire chapter pages.
+func TestIntegration_ChapterContent_ReturnsHTML(t *testing.T) {
+	client := newIntegrationClient(t)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 110*time.Second)
+	defer cancel()
+
+	req := browser.ContentRequest{
+		URL: chapterURL,
+		WaitFor: &browser.WaitForSelector{
+			Selector: "#content",
+			Timeout:  5000,
+		},
+		RejectResourceTypes: productionRejectTypes(),
+	}
+
+	html, err := client.GetContent(ctx, req)
+	if err != nil {
+		t.Fatalf("GetContent failed: %v", err)
+	}
+
+	// The #content div must not be empty; presence of <p> tags inside it is a
+	// reliable indicator that chapter paragraphs were rendered.
+	contentIdx := strings.Index(html, `id="content"`)
+	if contentIdx == -1 {
+		t.Fatalf("id=\"content\" not found in response (%d bytes)", len(html))
+	}
+
+	// Look for <p> tags after the #content marker — the chapter text lives there.
+	afterContent := html[contentIdx:]
+	if !strings.Contains(afterContent, "<p") {
+		t.Errorf("#content section contains no <p> tags; JS rendering may have failed.\nSection preview:\n%s",
+			truncate(afterContent, 1000))
+	}
+
+	t.Logf("chapter content section starts at byte %d (total response: %d bytes)", contentIdx, len(html))
+}
+
+// TestIntegration_ChapterContent_TimeoutSurfacedCorrectly verifies that a
+// deliberately too-short timeout returns an error containing "TimeoutError" (the
+// Browserless error string seen in the failing log entry).  This ensures our
+// error-classification logic in retryGetContent matches real Browserless output.
+func TestIntegration_ChapterContent_TimeoutSurfacedCorrectly(t *testing.T) {
+	client := newIntegrationClient(t)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second)
+	defer cancel()
+
+	req := browser.ContentRequest{
+		URL: chapterURL,
+		WaitFor: &browser.WaitForSelector{
+			Selector: "#content",
+			Timeout:  500, // intentionally too short (500 ms) → Browserless will time out
+		},
+		RejectResourceTypes: productionRejectTypes(),
+	}
+
+	_, err := client.GetContent(ctx, req)
+	if err == nil {
+		t.Fatal("expected a timeout error from Browserless, but GetContent succeeded — " +
+			"the page may now load very fast; adjust the timeout threshold")
+	}
+
+	t.Logf("got expected error: %v", err)
+
+	// Browserless wraps navigation timeouts in a 500 response with
+	// "TimeoutError: Navigation timeout" in the body — this is the exact
+	// error that is triggering retries in production.
+	if !strings.Contains(err.Error(), "500") {
+		t.Errorf("expected HTTP 500 status in error, got: %v", err)
+	}
+}
+
+// ── helpers ───────────────────────────────────────────────────────────────────
+
+// productionRejectTypes returns the same resource-type block-list the
+// novelfire scraper uses in production, so integration tests exercise the
+// identical request shape.
+func productionRejectTypes() []string {
+	return []string{
+		"cspviolationreport",
+		"eventsource",
+		"fedcm",
+		"font",
+		"image",
+		"manifest",
+		"media",
+		"other",
+		"ping",
+		"signedexchange",
+		"stylesheet",
+		"texttrack",
+		"websocket",
+	}
+}
+
+// truncate returns the first n bytes of s as a string.
+func truncate(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	return s[:n] + "…"
+}

scraper/internal/browser/interface.go

@@ -21,6 +21,10 @@ const (
 	// DevTools Protocol). Most powerful; required for complex interactions
 	// (clicking, scrolling, waiting for network idle, etc.).
 	StrategyCDP Strategy = "cdp"
+
+	// StrategyDirect uses a plain HTTP client to fetch HTML directly.
+	// Suitable for sites that don't require JavaScript rendering.
+	StrategyDirect Strategy = "direct"
 )
 
 // WaitForSelector describes the waitForSelector option sent to Browserless.
@@ -29,12 +33,20 @@ type WaitForSelector struct {
 	Timeout  int    `json:"timeout,omitempty"` // ms
 }
 
+// GotoOptions controls page navigation behavior.
+type GotoOptions struct {
+	Timeout   int    `json:"timeout,omitempty"`   // ms
+	WaitUntil string `json:"waitUntil,omitempty"` // e.g., "networkidle2", "load"
+}
+
 // ContentRequest is the body sent to POST /content.
 type ContentRequest struct {
 	URL                 string           `json:"url"`
 	WaitFor             *WaitForSelector `json:"waitForSelector,omitempty"`
 	WaitForTimeout      int              `json:"waitForTimeout,omitempty"`      // ms
 	RejectResourceTypes []string         `json:"rejectResourceTypes,omitempty"` // e.g. ["image","stylesheet"]
+	GotoOptions         *GotoOptions     `json:"gotoOptions,omitempty"`
+	BestAttempt         bool             `json:"bestAttempt,omitempty"` // return partial content on timeout/error
 }
 
 // ScrapeElement is one element descriptor inside a ScrapeRequest.
@@ -45,9 +57,10 @@ type ScrapeElement struct {
 
 // ScrapeRequest is the body sent to POST /scrape.
 type ScrapeRequest struct {
-	URL      string           `json:"url"`
-	Elements []ScrapeElement  `json:"elements"`
-	WaitFor  *WaitForSelector `json:"waitForSelector,omitempty"`
+	URL         string           `json:"url"`
+	Elements    []ScrapeElement  `json:"elements"`
+	WaitFor     *WaitForSelector `json:"waitForSelector,omitempty"`
+	GotoOptions *GotoOptions     `json:"gotoOptions,omitempty"`
 }
 
 // ScrapeResult is one entry in the response from POST /scrape.