diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go index 9c8fd94..454c1f6 100644 --- a/scraper/internal/server/server.go +++ b/scraper/internal/server/server.go @@ -33,6 +33,7 @@ import ( "github.com/libnovel/scraper/internal/orchestrator" "github.com/libnovel/scraper/internal/scraper" "github.com/libnovel/scraper/internal/storage" + "golang.org/x/net/html" ) // Server wraps an HTTP mux with the scraping endpoints. @@ -119,6 +120,10 @@ func (s *Server) ListenAndServe(ctx context.Context) error { mux.HandleFunc("GET /health", s.handleHealth) mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue) mux.HandleFunc("POST /scrape/book", s.handleScrapeBook) + // Browse API — fetches and parses novelfire catalogue page + mux.HandleFunc("GET /api/browse", s.handleBrowse) + // Scrape status + mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus) // Progress API mux.HandleFunc("GET /api/progress", s.handleGetProgress) mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress) @@ -637,3 +642,264 @@ func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) { } }() } + +// ─── Browse API ─────────────────────────────────────────────────────────────── + +// NovelListing represents a single novel entry from the novelfire browse page. +type NovelListing struct { + Slug string `json:"slug"` + Title string `json:"title"` + Cover string `json:"cover"` + Rank string `json:"rank"` + Rating string `json:"rating"` + Chapters string `json:"chapters"` + URL string `json:"url"` +} + +const novelFireBase = "https://novelfire.net" + +// handleBrowse handles GET /api/browse. +// Query params: +// +// page (default 1) +// genre (default "all") +// sort (default "popular") +// status (default "all") +// type (default "all-novel") +// +// Returns JSON: {"novels":[...], "page": N, "hasNext": bool} +func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + page := q.Get("page") + if page == "" { + page = "1" + } + genre := q.Get("genre") + if genre == "" { + genre = "all" + } + sortBy := q.Get("sort") + if sortBy == "" { + sortBy = "popular" + } + status := q.Get("status") + if status == "" { + status = "all" + } + novelType := q.Get("type") + if novelType == "" { + novelType = "all-novel" + } + + // Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page} + targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s", + novelFireBase, genre, sortBy, status, novelType, page) + + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil) + if err != nil { + http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError) + return + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)") + req.Header.Set("Accept", "text/html,application/xhtml+xml") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + s.log.Error("browse fetch failed", "url", targetURL, "err", err) + http.Error(w, `{"error":"failed to fetch browse page"}`, http.StatusBadGateway) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + http.Error(w, fmt.Sprintf(`{"error":"upstream returned %d"}`, resp.StatusCode), http.StatusBadGateway) + return + } + + novels, hasNext := parseBrowsePage(resp.Body) + pageNum, _ := strconv.Atoi(page) + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "public, max-age=300") + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "novels": novels, + "page": pageNum, + "hasNext": hasNext, + }) +} + +// parseBrowsePage parses the novelfire HTML and extracts novel listings. +// Returns novels and whether a "next page" link was found. +func parseBrowsePage(r io.Reader) ([]NovelListing, bool) { + doc, err := html.Parse(r) + if err != nil { + return nil, false + } + + var novels []NovelListing + hasNext := false + + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.ElementNode { + switch n.Data { + case "li": + if hasClass(n, "novel-item") { + if novel, ok := parseNovelItem(n); ok { + novels = append(novels, novel) + } + } + // pagination li with class "next" + if hasClass(n, "next") { + hasNext = true + } + case "a": + // Detect "next" pagination link + if hasClass(n, "next") || attrVal(n, "rel") == "next" { + hasNext = true + } + // Also check aria-label="Next" + if attrVal(n, "aria-label") == "Next" { + hasNext = true + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(doc) + return novels, hasNext +} + +// parseNovelItem extracts a NovelListing from a
  • node. +func parseNovelItem(li *html.Node) (NovelListing, bool) { + var novel NovelListing + + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.ElementNode { + switch n.Data { + case "a": + href := attrVal(n, "href") + if strings.HasPrefix(href, "/book/") { + slug := strings.TrimPrefix(href, "/book/") + slug = strings.TrimSuffix(slug, "/") + if novel.Slug == "" { + novel.Slug = slug + novel.URL = novelFireBase + href + } + } + case "img": + // lazy-loaded covers use data-src + src := attrVal(n, "data-src") + if src == "" { + src = attrVal(n, "src") + } + if src != "" && novel.Cover == "" { + novel.Cover = src + } + case "h4": + if hasClass(n, "novel-title") && novel.Title == "" { + novel.Title = strings.TrimSpace(textContent(n)) + } + case "span": + cls := attrVal(n, "class") + if strings.Contains(cls, "_bl") && novel.Rank == "" { + novel.Rank = strings.TrimSpace(textContent(n)) + } + if strings.Contains(cls, "_br") && novel.Rating == "" { + novel.Rating = strings.TrimSpace(textContent(n)) + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(li) + + // Extract chapter count from the novel stats text (contains "N Chapters") + novel.Chapters = extractChapters(li) + + if novel.Slug == "" || novel.Title == "" { + return novel, false + } + return novel, true +} + +// extractChapters finds the chapter count text within a novel-item node. +func extractChapters(n *html.Node) string { + var result string + var walk func(*html.Node) + walk = func(node *html.Node) { + if node.Type == html.ElementNode { + cls := attrVal(node, "class") + if strings.Contains(cls, "novel-stats") || strings.Contains(cls, "chapter") { + txt := strings.TrimSpace(textContent(node)) + if strings.Contains(txt, "Chapter") || strings.Contains(txt, "chapter") { + // Extract just the numeric part if possible + result = txt + return + } + } + } + for c := node.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(n) + return result +} + +// hasClass reports whether an HTML node has the given CSS class. +func hasClass(n *html.Node, cls string) bool { + for _, a := range n.Attr { + if a.Key == "class" { + for _, c := range strings.Fields(a.Val) { + if c == cls { + return true + } + } + } + } + return false +} + +// attrVal returns the value of an attribute on an HTML node, or "". +func attrVal(n *html.Node, key string) string { + for _, a := range n.Attr { + if a.Key == key { + return a.Val + } + } + return "" +} + +// textContent returns the concatenated text content of a node and its descendants. +func textContent(n *html.Node) string { + if n.Type == html.TextNode { + return n.Data + } + var sb strings.Builder + for c := n.FirstChild; c != nil; c = c.NextSibling { + sb.WriteString(textContent(c)) + } + return sb.String() +} + +// ─── Scrape status API ──────────────────────────────────────────────────────── + +// handleScrapeStatus handles GET /api/scrape/status. +// Returns JSON: {"running": bool} +func (s *Server) handleScrapeStatus(w http.ResponseWriter, _ *http.Request) { + s.mu.Lock() + running := s.running + s.mu.Unlock() + + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]bool{"running": running}) +}