diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go index 9c8fd94..454c1f6 100644 --- a/scraper/internal/server/server.go +++ b/scraper/internal/server/server.go @@ -33,6 +33,7 @@ import ( "github.com/libnovel/scraper/internal/orchestrator" "github.com/libnovel/scraper/internal/scraper" "github.com/libnovel/scraper/internal/storage" + "golang.org/x/net/html" ) // Server wraps an HTTP mux with the scraping endpoints. @@ -119,6 +120,10 @@ func (s *Server) ListenAndServe(ctx context.Context) error { mux.HandleFunc("GET /health", s.handleHealth) mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue) mux.HandleFunc("POST /scrape/book", s.handleScrapeBook) + // Browse API — fetches and parses novelfire catalogue page + mux.HandleFunc("GET /api/browse", s.handleBrowse) + // Scrape status + mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus) // Progress API mux.HandleFunc("GET /api/progress", s.handleGetProgress) mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress) @@ -637,3 +642,264 @@ func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) { } }() } + +// ─── Browse API ─────────────────────────────────────────────────────────────── + +// NovelListing represents a single novel entry from the novelfire browse page. +type NovelListing struct { + Slug string `json:"slug"` + Title string `json:"title"` + Cover string `json:"cover"` + Rank string `json:"rank"` + Rating string `json:"rating"` + Chapters string `json:"chapters"` + URL string `json:"url"` +} + +const novelFireBase = "https://novelfire.net" + +// handleBrowse handles GET /api/browse. +// Query params: +// +// page (default 1) +// genre (default "all") +// sort (default "popular") +// status (default "all") +// type (default "all-novel") +// +// Returns JSON: {"novels":[...], "page": N, "hasNext": bool} +func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + page := q.Get("page") + if page == "" { + page = "1" + } + genre := q.Get("genre") + if genre == "" { + genre = "all" + } + sortBy := q.Get("sort") + if sortBy == "" { + sortBy = "popular" + } + status := q.Get("status") + if status == "" { + status = "all" + } + novelType := q.Get("type") + if novelType == "" { + novelType = "all-novel" + } + + // Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page} + targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s", + novelFireBase, genre, sortBy, status, novelType, page) + + ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil) + if err != nil { + http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError) + return + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)") + req.Header.Set("Accept", "text/html,application/xhtml+xml") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + s.log.Error("browse fetch failed", "url", targetURL, "err", err) + http.Error(w, `{"error":"failed to fetch browse page"}`, http.StatusBadGateway) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + http.Error(w, fmt.Sprintf(`{"error":"upstream returned %d"}`, resp.StatusCode), http.StatusBadGateway) + return + } + + novels, hasNext := parseBrowsePage(resp.Body) + pageNum, _ := strconv.Atoi(page) + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "public, max-age=300") + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "novels": novels, + "page": pageNum, + "hasNext": hasNext, + }) +} + +// parseBrowsePage parses the novelfire HTML and extracts novel listings. +// Returns novels and whether a "next page" link was found. +func parseBrowsePage(r io.Reader) ([]NovelListing, bool) { + doc, err := html.Parse(r) + if err != nil { + return nil, false + } + + var novels []NovelListing + hasNext := false + + var walk func(*html.Node) + walk = func(n *html.Node) { + if n.Type == html.ElementNode { + switch n.Data { + case "li": + if hasClass(n, "novel-item") { + if novel, ok := parseNovelItem(n); ok { + novels = append(novels, novel) + } + } + // pagination li with class "next" + if hasClass(n, "next") { + hasNext = true + } + case "a": + // Detect "next" pagination link + if hasClass(n, "next") || attrVal(n, "rel") == "next" { + hasNext = true + } + // Also check aria-label="Next" + if attrVal(n, "aria-label") == "Next" { + hasNext = true + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c) + } + } + walk(doc) + return novels, hasNext +} + +// parseNovelItem extracts a NovelListing from a