feat: add Kokoro TTS, ranking page, direct HTTP strategy, and chapter-number fix

- Add Kokoro-FastAPI TTS integration to the chapter reader UI: - Browser-side MSE streaming with paragraph-level click-to-start - Voice selector, speed slider, auto-next with prefetch of the next chapter - New GET /ui/chapter-text endpoint that strips Markdown and serves plain text - Add ranking page (novelfire /ranking scraper, WriteRanking/ReadRankingItems in writer, GET /ranking + POST /ranking/refresh + GET /ranking/view routes) with local-library annotation and one-click scrape buttons - Add StrategyDirect (plain HTTP client) as a new browser strategy; the default strategy is now 'direct' for chapter fetching and 'content' for chapter-list URL retrieval (split via BROWSERLESS_URL_STRATEGY) - Fix chapter numbering bug: numbers are now derived from the URL path (/chapter-N) rather than list position, correcting newest-first ordering - Add 'refresh <slug>' CLI sub-command to re-scrape a book from its saved source_url without knowing the original URL - Extend NovelScraper interface with RankingProvider (ScrapeRanking) - Tune scraper timeouts: wait-for-selector reduced to 5 s, GotoOptions timeout set to 60 s, content/scrape client defaults raised to 90 s - Add cover extraction fix (figure.cover > img rather than bare img.cover) - Add AGENTS.md and .aiignore for AI tooling context - Add integration tests for browser client and novelfire scraper (build tag: integration) and unit tests for chapterNumberFromURL and pagination
2026-03-01 12:25:16 +05:00
parent e6e6f7dc4d
commit 7879a51fe3
17 changed files with 2816 additions and 88 deletions
--- a/scraper/internal/novelfire/scraper.go
+++ b/scraper/internal/novelfire/scraper.go
@@ -13,6 +13,7 @@ import (
 	"fmt"
 	"log/slog"
 	"net/url"
+	"path"
 	"strconv"
 	"strings"
 	"time"
@@ -25,6 +26,7 @@ import (
 const (
 	baseURL       = "https://novelfire.net"
 	cataloguePath = "/genre-all/sort-new/status-all/all-novel"
+	rankingPath   = "/ranking"
 )

 // rejectResourceTypes lists Browserless resource types to block on every request.
@@ -40,8 +42,6 @@ var rejectResourceTypes = []string{
 	"media",
 	"other",
 	"ping",
-	"prefetch",
-	"preflight",
 	"signedexchange",
 	"stylesheet",
 	"texttrack",
@@ -51,16 +51,22 @@ var rejectResourceTypes = []string{
 // Scraper is the novelfire.net implementation of scraper.NovelScraper.
 // It uses the /content strategy by default (rendered HTML via Browserless).
 type Scraper struct {
-	client browser.BrowserClient
-	log    *slog.Logger
+	client    browser.BrowserClient
+	urlClient browser.BrowserClient // separate client for URL retrieval (uses browserless content strategy)
+	log       *slog.Logger
 }

 // New returns a new novelfire Scraper.
-func New(client browser.BrowserClient, log *slog.Logger) *Scraper {
+// client is used for content fetching, urlClient is used for URL retrieval (chapter list).
+// If urlClient is nil, client will be used for both.
+func New(client browser.BrowserClient, log *slog.Logger, urlClient browser.BrowserClient) *Scraper {
 	if log == nil {
 		log = slog.Default()
 	}
-	return &Scraper{client: client, log: log}
+	if urlClient == nil {
+		urlClient = client
+	}
+	return &Scraper{client: client, urlClient: urlClient, log: log}
 }

 // SourceName implements NovelScraper.
@@ -92,15 +98,14 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan scraper.Catalogue
 				"page", page,
 				"payload_url", pageURL,
 				"payload_wait_selector", ".novel-item",
-				"payload_wait_selector_timeout_ms", 10000,
-				"payload_wait_for_timeout_ms", 10000,
+				"payload_wait_selector_timeout_ms", 5000,
 			)

 			html, err := s.client.GetContent(ctx, browser.ContentRequest{
 				URL:                 pageURL,
-				WaitFor:             &browser.WaitForSelector{Selector: ".novel-item", Timeout: 10000},
-				WaitForTimeout:      10000,
+				WaitFor:             &browser.WaitForSelector{Selector: ".novel-item", Timeout: 5000},
 				RejectResourceTypes: rejectResourceTypes,
+				GotoOptions:         &browser.GotoOptions{Timeout: 60000},
 			})
 			if err != nil {
 				s.log.Debug("catalogue page fetch failed",
@@ -173,15 +178,14 @@ func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.B
 	s.log.Debug("metadata fetch starting",
 		"payload_url", bookURL,
 		"payload_wait_selector", ".novel-title",
-		"payload_wait_selector_timeout_ms", 10000,
-		"payload_wait_for_timeout_ms", 10000,
+		"payload_wait_selector_timeout_ms", 5000,
 	)

 	raw, err := s.client.GetContent(ctx, browser.ContentRequest{
 		URL:                 bookURL,
-		WaitFor:             &browser.WaitForSelector{Selector: ".novel-title", Timeout: 10000},
-		WaitForTimeout:      10000,
+		WaitFor:             &browser.WaitForSelector{Selector: ".novel-title", Timeout: 5000},
 		RejectResourceTypes: rejectResourceTypes,
+		GotoOptions:         &browser.GotoOptions{Timeout: 60000},
 	})
 	if err != nil {
 		s.log.Debug("metadata fetch failed", "url", bookURL, "err", err)
@@ -198,8 +202,11 @@ func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.B
 	title := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "h1", Class: "novel-title"})
 	// <span class="author"><a>Author Name</a></span>
 	author := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "author"})
-	// <img class="cover" src="...">
-	cover := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "img", Class: "cover", Attr: "src"})
+	// <figure class="cover"><img src="..."></figure>
+	var cover string
+	if figureCover := htmlutil.FindFirst(root, scraper.Selector{Tag: "figure", Class: "cover"}); figureCover != nil {
+		cover = htmlutil.ExtractFirst(figureCover, scraper.Selector{Tag: "img", Attr: "src"})
+	}
 	// <span class="status">Ongoing</span>
 	status := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "span", Class: "status"})

@@ -245,32 +252,41 @@ func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (scraper.B

 func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scraper.ChapterRef, error) {
 	var refs []scraper.ChapterRef
-	// Chapter list URL: {bookURL}/chapters
-	pageURL := strings.TrimRight(bookURL, "/") + "/chapters"
+	// Chapter list URL: {bookURL}/chapters?page=N
+	baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters"
 	page := 1

-	for pageURL != "" {
+	for {
 		select {
 		case <-ctx.Done():
 			return refs, ctx.Err()
 		default:
 		}

+		pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page)
 		s.log.Info("scraping chapter list", "page", page, "url", pageURL)

 		s.log.Debug("chapter list fetch starting",
 			"page", page,
 			"payload_url", pageURL,
 			"payload_wait_selector", ".chapter-list",
-			"payload_wait_selector_timeout_ms", 10000,
-			"payload_wait_for_timeout_ms", 10000,
+			"payload_wait_selector_timeout_ms", 15000,
+			"payload_wait_timeout_ms", 2000,
+			"strategy", s.urlClient.Strategy(),
 		)

-		raw, err := s.client.GetContent(ctx, browser.ContentRequest{
-			URL:                 pageURL,
-			WaitFor:             &browser.WaitForSelector{Selector: ".chapter-list", Timeout: 10000},
-			WaitForTimeout:      10000,
+		raw, err := s.urlClient.GetContent(ctx, browser.ContentRequest{
+			URL: pageURL,
+			// Wait up to 15 s for the chapter list container to appear in the DOM.
+			WaitFor: &browser.WaitForSelector{Selector: ".chapter-list", Timeout: 15000},
+			// After the selector is found, wait an additional 2 s for any
+			// deferred JS rendering (lazy-loaded links, infinite-scroll hydration).
+			WaitForTimeout:      2000,
 			RejectResourceTypes: rejectResourceTypes,
+			GotoOptions:         &browser.GotoOptions{Timeout: 60000},
+			// Do NOT use BestAttempt — we want a complete page or a clear error,
+			// not silently partial HTML that looks like "no more chapters".
+			BestAttempt: false,
 		})
 		if err != nil {
 			s.log.Debug("chapter list fetch failed",
@@ -293,10 +309,27 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra

 		chapterList := htmlutil.FindFirst(root, scraper.Selector{Class: "chapter-list"})
 		if chapterList == nil {
+			// No chapter list container on this page — we've gone past the last page.
+			s.log.Debug("chapter list container not found, stopping pagination", "page", page)
 			break
 		}
+
 		// Each chapter row: <li class="chapter-item"><a href="...">Title</a></li>
 		items := htmlutil.FindAll(chapterList, scraper.Selector{Tag: "li"})
+
+		s.log.Debug("chapter list page parsed",
+			"page", page,
+			"url", pageURL,
+			"chapters_on_page", len(items),
+			"total_refs_so_far", len(refs),
+		)
+
+		// Zero items on this page means we've gone past the last page.
+		if len(items) == 0 {
+			s.log.Debug("no chapters on page, stopping pagination", "page", page)
+			break
+		}
+
 		for _, item := range items {
 			linkNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "a"})
 			if linkNode == nil {
@@ -308,7 +341,15 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra
 				continue
 			}
 			chURL := resolveURL(baseURL, href)
-			num := len(refs) + 1
+			num := chapterNumberFromURL(chURL)
+			if num <= 0 {
+				// Fall back to position if the URL has no parseable number.
+				num = len(refs) + 1
+				s.log.Warn("chapter number not parseable from URL, falling back to position",
+					"url", chURL,
+					"position", num,
+				)
+			}
 			refs = append(refs, scraper.ChapterRef{
 				Number: num,
 				Title:  strings.TrimSpace(chTitle),
@@ -316,30 +357,134 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]scra
 			})
 		}

-		s.log.Debug("chapter list page parsed",
-			"page", page,
-			"url", pageURL,
-			"chapters_on_page", len(items),
-			"total_refs_so_far", len(refs),
-		)
-
-		// Next page: <a class="next" href="...">
-		nextHref := htmlutil.ExtractFirst(root, scraper.Selector{Tag: "a", Class: "next", Attr: "href"})
-		if nextHref == "" {
-			break
-		}
-		pageURL = resolveURL(baseURL, nextHref)
 		page++
 	}

 	return refs, nil
 }

+// ─── RankingProvider ───────────────────────────────────────────────────────────
+
+func (s *Scraper) ScrapeRanking(ctx context.Context) (<-chan scraper.BookMeta, <-chan error) {
+	entries := make(chan scraper.BookMeta, 64)
+	errs := make(chan error, 16)
+
+	go func() {
+		defer close(entries)
+		defer close(errs)
+
+		pageURL := baseURL + rankingPath
+		rank := 1
+
+		for pageURL != "" {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+			}
+
+			s.log.Info("scraping ranking page", "url", pageURL)
+
+			// Use WaitFor only for browser-based strategies
+			var raw string
+			var err error
+			if s.client.Strategy() == browser.StrategyDirect {
+				raw, err = s.client.GetContent(ctx, browser.ContentRequest{
+					URL:                 pageURL,
+					RejectResourceTypes: rejectResourceTypes,
+				})
+			} else {
+				raw, err = s.client.GetContent(ctx, browser.ContentRequest{
+					URL:                 pageURL,
+					WaitFor:             &browser.WaitForSelector{Selector: ".rank-novels", Timeout: 30000},
+					RejectResourceTypes: rejectResourceTypes,
+					GotoOptions:         &browser.GotoOptions{Timeout: 60000},
+					BestAttempt:         true,
+				})
+			}
+			if err != nil {
+				s.log.Debug("ranking page fetch failed", "url", pageURL, "err", err)
+				errs <- fmt.Errorf("ranking page: %w", err)
+				return
+			}
+
+			root, err := htmlutil.ParseHTML(raw)
+			if err != nil {
+				errs <- fmt.Errorf("ranking page parse: %w", err)
+				return
+			}
+
+			rankList := htmlutil.FindFirst(root, scraper.Selector{Class: "rank-novels"})
+			if rankList == nil {
+				break
+			}
+
+			items := htmlutil.FindAll(rankList, scraper.Selector{Tag: "li", Class: "novel-item"})
+			for _, item := range items {
+				// Cover: <figure class="cover"><a href="/book/slug"><img data-src="..."></a></figure>
+				var cover string
+				if fig := htmlutil.FindFirst(item, scraper.Selector{Tag: "figure", Class: "cover"}); fig != nil {
+					cover = htmlutil.ExtractFirst(fig, scraper.Selector{Tag: "img", Attr: "data-src"})
+					if cover != "" {
+						cover = baseURL + cover
+					}
+				}
+
+				// Title and URL: <h2 class="title"><a href="/book/slug">Title</a></h2>
+				titleNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "h2", Class: "title"})
+				var title, bookURL string
+				if titleNode != nil {
+					linkNode := htmlutil.FindFirst(titleNode, scraper.Selector{Tag: "a"})
+					if linkNode != nil {
+						title = htmlutil.ExtractText(linkNode, scraper.Selector{})
+						href := htmlutil.ExtractText(linkNode, scraper.Selector{Attr: "href"})
+						bookURL = resolveURL(baseURL, href)
+					}
+				}
+
+				// Status: <span class="status"> Ongoing/Completed </span>
+				status := htmlutil.ExtractFirst(item, scraper.Selector{Tag: "span", Class: "status"})
+
+				// Genres: <div class="categories"><div class="scroll"><span>Genre1</span><span>Genre2</span>...</div></div>
+				var genres []string
+				categoriesNode := htmlutil.FindFirst(item, scraper.Selector{Tag: "div", Class: "categories"})
+				if categoriesNode != nil {
+					genres = htmlutil.ExtractAll(categoriesNode, scraper.Selector{Tag: "span", Multiple: true})
+				}
+
+				slug := slugFromURL(bookURL)
+
+				meta := scraper.BookMeta{
+					Slug:      slug,
+					Title:     title,
+					Cover:     cover,
+					Status:    strings.TrimSpace(status),
+					Genres:    genres,
+					SourceURL: bookURL,
+					Ranking:   rank,
+				}
+				rank++
+
+				select {
+				case <-ctx.Done():
+					return
+				case entries <- meta:
+				}
+			}
+
+			// Next page - ranking pages use different pagination, just get first page for now
+			break
+		}
+	}()
+
+	return entries, errs
+}
+
 // ─── ChapterTextProvider ─────────────────────────────────────────────────────

 // retryGetContent calls client.GetContent up to maxAttempts times, backing off
 // exponentially between retries. Only errors that look like transient Browserless
-// 5xx responses (navigation timeouts, etc.) are retried; context cancellation and
+// failures (timeouts, 5xx responses) are retried; context cancellation and
 // permanent errors are returned immediately.
 func retryGetContent(
 	ctx context.Context,
@@ -363,11 +508,6 @@ func retryGetContent(
 			return "", err
 		}

-		// Only retry on Browserless 5xx responses.
-		if !strings.Contains(err.Error(), "unexpected status 5") {
-			return "", err
-		}
-
 		if attempt < maxAttempts {
 			log.Warn("chapter fetch failed, retrying",
 				"url", req.URL,
@@ -393,15 +533,15 @@ func (s *Scraper) ScrapeChapterText(ctx context.Context, ref scraper.ChapterRef)
 		"title", ref.Title,
 		"payload_url", ref.URL,
 		"payload_wait_selector", "#content",
-		"payload_wait_selector_timeout_ms", 75000,
-		"payload_wait_for_timeout_ms", 75000,
+		"payload_wait_selector_timeout_ms", 5000,
 	)

 	raw, err := retryGetContent(ctx, s.log, s.client, browser.ContentRequest{
 		URL:                 ref.URL,
-		WaitFor:             &browser.WaitForSelector{Selector: "#content", Timeout: 75000},
-		WaitForTimeout:      75000,
+		WaitFor:             &browser.WaitForSelector{Selector: "#content", Timeout: 5000},
 		RejectResourceTypes: rejectResourceTypes,
+		GotoOptions:         &browser.GotoOptions{Timeout: 60000},
+		BestAttempt:         true,
 	}, 9, 6*time.Second)
 	if err != nil {
 		s.log.Debug("chapter text fetch failed",
@@ -411,6 +551,18 @@ func (s *Scraper) ScrapeChapterText(ctx context.Context, ref scraper.ChapterRef)
 		)
 		return scraper.Chapter{}, fmt.Errorf("chapter %d fetch: %w", ref.Number, err)
 	}
+	if len(raw) > 0 {
+		preview := raw
+		if len(preview) > 500 {
+			preview = preview[:500]
+		}
+		s.log.Debug("chapter text fetch partial content",
+			"chapter", ref.Number,
+			"url", ref.URL,
+			"response_bytes", len(raw),
+			"preview", preview,
+		)
+	}
 	s.log.Debug("chapter text fetch completed",
 		"chapter", ref.Number,
 		"url", ref.URL,
@@ -484,3 +636,30 @@ func parseChapterCount(s string) int {
 	n, _ := strconv.Atoi(fields[0])
 	return n
 }
+
+// chapterNumberFromURL extracts the chapter number from a novelfire chapter URL.
+//
+// URL pattern: https://novelfire.net/book/{book-slug}/chapter-{N}
+// The last path segment is expected to be "chapter-{N}" or "{N}".
+// Returns 0 if no number can be parsed.
+func chapterNumberFromURL(chapterURL string) int {
+	u, err := url.Parse(chapterURL)
+	if err != nil {
+		return 0
+	}
+	seg := path.Base(u.Path) // e.g. "chapter-42" or "42"
+	// Strip a "chapter-" prefix if present.
+	seg = strings.TrimPrefix(seg, "chapter-")
+	// Also handle "chap-", "ch-" variants used by some sites.
+	seg = strings.TrimPrefix(seg, "chap-")
+	seg = strings.TrimPrefix(seg, "ch-")
+	// Take only the leading digits (handles slugs like "42-title-text").
+	digits := strings.FieldsFunc(seg, func(r rune) bool {
+		return r < '0' || r > '9'
+	})
+	if len(digits) == 0 {
+		return 0
+	}
+	n, _ := strconv.Atoi(digits[0])
+	return n
+}