scraper/internal/server/server.go

@@ -33,6 +33,7 @@ import (
 	"github.com/libnovel/scraper/internal/orchestrator"
 	"github.com/libnovel/scraper/internal/scraper"
 	"github.com/libnovel/scraper/internal/storage"
+	"golang.org/x/net/html"
 )
 
 // Server wraps an HTTP mux with the scraping endpoints.
@@ -119,6 +120,10 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
 	mux.HandleFunc("GET /health", s.handleHealth)
 	mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
 	mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
+	// Browse API — fetches and parses novelfire catalogue page
+	mux.HandleFunc("GET /api/browse", s.handleBrowse)
+	// Scrape status
+	mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus)
 	// Progress API
 	mux.HandleFunc("GET /api/progress", s.handleGetProgress)
 	mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
@@ -637,3 +642,264 @@ func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
 		}
 	}()
 }
+
+// ─── Browse API ───────────────────────────────────────────────────────────────
+
+// NovelListing represents a single novel entry from the novelfire browse page.
+type NovelListing struct {
+	Slug     string `json:"slug"`
+	Title    string `json:"title"`
+	Cover    string `json:"cover"`
+	Rank     string `json:"rank"`
+	Rating   string `json:"rating"`
+	Chapters string `json:"chapters"`
+	URL      string `json:"url"`
+}
+
+const novelFireBase = "https://novelfire.net"
+
+// handleBrowse handles GET /api/browse.
+// Query params:
+//
+//	page    (default 1)
+//	genre   (default "all")
+//	sort    (default "popular")
+//	status  (default "all")
+//	type    (default "all-novel")
+//
+// Returns JSON: {"novels":[...], "page": N, "hasNext": bool}
+func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
+	q := r.URL.Query()
+	page := q.Get("page")
+	if page == "" {
+		page = "1"
+	}
+	genre := q.Get("genre")
+	if genre == "" {
+		genre = "all"
+	}
+	sortBy := q.Get("sort")
+	if sortBy == "" {
+		sortBy = "popular"
+	}
+	status := q.Get("status")
+	if status == "" {
+		status = "all"
+	}
+	novelType := q.Get("type")
+	if novelType == "" {
+		novelType = "all-novel"
+	}
+
+	// Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page}
+	targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
+		novelFireBase, genre, sortBy, status, novelType, page)
+
+	ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
+	if err != nil {
+		http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError)
+		return
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
+	req.Header.Set("Accept", "text/html,application/xhtml+xml")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		s.log.Error("browse fetch failed", "url", targetURL, "err", err)
+		http.Error(w, `{"error":"failed to fetch browse page"}`, http.StatusBadGateway)
+		return
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		http.Error(w, fmt.Sprintf(`{"error":"upstream returned %d"}`, resp.StatusCode), http.StatusBadGateway)
+		return
+	}
+
+	novels, hasNext := parseBrowsePage(resp.Body)
+	pageNum, _ := strconv.Atoi(page)
+
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Cache-Control", "public, max-age=300")
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{
+		"novels":  novels,
+		"page":    pageNum,
+		"hasNext": hasNext,
+	})
+}
+
+// parseBrowsePage parses the novelfire HTML and extracts novel listings.
+// Returns novels and whether a "next page" link was found.
+func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {
+	doc, err := html.Parse(r)
+	if err != nil {
+		return nil, false
+	}
+
+	var novels []NovelListing
+	hasNext := false
+
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		if n.Type == html.ElementNode {
+			switch n.Data {
+			case "li":
+				if hasClass(n, "novel-item") {
+					if novel, ok := parseNovelItem(n); ok {
+						novels = append(novels, novel)
+					}
+				}
+				// pagination li with class "next"
+				if hasClass(n, "next") {
+					hasNext = true
+				}
+			case "a":
+				// Detect "next" pagination link
+				if hasClass(n, "next") || attrVal(n, "rel") == "next" {
+					hasNext = true
+				}
+				// Also check aria-label="Next"
+				if attrVal(n, "aria-label") == "Next" {
+					hasNext = true
+				}
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(doc)
+	return novels, hasNext
+}
+
+// parseNovelItem extracts a NovelListing from a <li class="novel-item"> node.
+func parseNovelItem(li *html.Node) (NovelListing, bool) {
+	var novel NovelListing
+
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		if n.Type == html.ElementNode {
+			switch n.Data {
+			case "a":
+				href := attrVal(n, "href")
+				if strings.HasPrefix(href, "/book/") {
+					slug := strings.TrimPrefix(href, "/book/")
+					slug = strings.TrimSuffix(slug, "/")
+					if novel.Slug == "" {
+						novel.Slug = slug
+						novel.URL = novelFireBase + href
+					}
+				}
+			case "img":
+				// lazy-loaded covers use data-src
+				src := attrVal(n, "data-src")
+				if src == "" {
+					src = attrVal(n, "src")
+				}
+				if src != "" && novel.Cover == "" {
+					novel.Cover = src
+				}
+			case "h4":
+				if hasClass(n, "novel-title") && novel.Title == "" {
+					novel.Title = strings.TrimSpace(textContent(n))
+				}
+			case "span":
+				cls := attrVal(n, "class")
+				if strings.Contains(cls, "_bl") && novel.Rank == "" {
+					novel.Rank = strings.TrimSpace(textContent(n))
+				}
+				if strings.Contains(cls, "_br") && novel.Rating == "" {
+					novel.Rating = strings.TrimSpace(textContent(n))
+				}
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(li)
+
+	// Extract chapter count from the novel stats text (contains "N Chapters")
+	novel.Chapters = extractChapters(li)
+
+	if novel.Slug == "" || novel.Title == "" {
+		return novel, false
+	}
+	return novel, true
+}
+
+// extractChapters finds the chapter count text within a novel-item node.
+func extractChapters(n *html.Node) string {
+	var result string
+	var walk func(*html.Node)
+	walk = func(node *html.Node) {
+		if node.Type == html.ElementNode {
+			cls := attrVal(node, "class")
+			if strings.Contains(cls, "novel-stats") || strings.Contains(cls, "chapter") {
+				txt := strings.TrimSpace(textContent(node))
+				if strings.Contains(txt, "Chapter") || strings.Contains(txt, "chapter") {
+					// Extract just the numeric part if possible
+					result = txt
+					return
+				}
+			}
+		}
+		for c := node.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(n)
+	return result
+}
+
+// hasClass reports whether an HTML node has the given CSS class.
+func hasClass(n *html.Node, cls string) bool {
+	for _, a := range n.Attr {
+		if a.Key == "class" {
+			for _, c := range strings.Fields(a.Val) {
+				if c == cls {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+// attrVal returns the value of an attribute on an HTML node, or "".
+func attrVal(n *html.Node, key string) string {
+	for _, a := range n.Attr {
+		if a.Key == key {
+			return a.Val
+		}
+	}
+	return ""
+}
+
+// textContent returns the concatenated text content of a node and its descendants.
+func textContent(n *html.Node) string {
+	if n.Type == html.TextNode {
+		return n.Data
+	}
+	var sb strings.Builder
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		sb.WriteString(textContent(c))
+	}
+	return sb.String()
+}
+
+// ─── Scrape status API ────────────────────────────────────────────────────────
+
+// handleScrapeStatus handles GET /api/scrape/status.
+// Returns JSON: {"running": bool}
+func (s *Server) handleScrapeStatus(w http.ResponseWriter, _ *http.Request) {
+	s.mu.Lock()
+	running := s.running
+	s.mu.Unlock()
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]bool{"running": running})
+}