feat(scraper): add GET /api/browse and GET /api/scrape/status endpoints
- GET /api/browse fetches novelfire.net catalogue page and parses it with golang.org/x/net/html;
returns JSON {novels, page, hasNext} with per-novel slug/title/cover/rank/rating/chapters/url.
Supports page, genre, sort, status query params.
- GET /api/scrape/status returns {"running": bool} for polling job state from the UI
This commit is contained in:
Admin
@@ -33,6 +33,7 @@ import (
|
||||
"github.com/libnovel/scraper/internal/orchestrator"
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
"github.com/libnovel/scraper/internal/storage"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// Server wraps an HTTP mux with the scraping endpoints.
|
||||
@@ -119,6 +120,10 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
mux.HandleFunc("GET /health", s.handleHealth)
|
||||
mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
|
||||
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
|
||||
// Browse API — fetches and parses novelfire catalogue page
|
||||
mux.HandleFunc("GET /api/browse", s.handleBrowse)
|
||||
// Scrape status
|
||||
mux.HandleFunc("GET /api/scrape/status", s.handleScrapeStatus)
|
||||
// Progress API
|
||||
mux.HandleFunc("GET /api/progress", s.handleGetProgress)
|
||||
mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
|
||||
@@ -637,3 +642,264 @@ func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// ─── Browse API ───────────────────────────────────────────────────────────────
|
||||
|
||||
// NovelListing represents a single novel entry from the novelfire browse page.
|
||||
type NovelListing struct {
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Cover string `json:"cover"`
|
||||
Rank string `json:"rank"`
|
||||
Rating string `json:"rating"`
|
||||
Chapters string `json:"chapters"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
const novelFireBase = "https://novelfire.net"
|
||||
|
||||
// handleBrowse handles GET /api/browse.
|
||||
// Query params:
|
||||
//
|
||||
// page (default 1)
|
||||
// genre (default "all")
|
||||
// sort (default "popular")
|
||||
// status (default "all")
|
||||
// type (default "all-novel")
|
||||
//
|
||||
// Returns JSON: {"novels":[...], "page": N, "hasNext": bool}
|
||||
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
page := q.Get("page")
|
||||
if page == "" {
|
||||
page = "1"
|
||||
}
|
||||
genre := q.Get("genre")
|
||||
if genre == "" {
|
||||
genre = "all"
|
||||
}
|
||||
sortBy := q.Get("sort")
|
||||
if sortBy == "" {
|
||||
sortBy = "popular"
|
||||
}
|
||||
status := q.Get("status")
|
||||
if status == "" {
|
||||
status = "all"
|
||||
}
|
||||
novelType := q.Get("type")
|
||||
if novelType == "" {
|
||||
novelType = "all-novel"
|
||||
}
|
||||
|
||||
// Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page}
|
||||
targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
|
||||
novelFireBase, genre, sortBy, status, novelType, page)
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
|
||||
if err != nil {
|
||||
http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-scraper/1.0)")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
s.log.Error("browse fetch failed", "url", targetURL, "err", err)
|
||||
http.Error(w, `{"error":"failed to fetch browse page"}`, http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
http.Error(w, fmt.Sprintf(`{"error":"upstream returned %d"}`, resp.StatusCode), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
novels, hasNext := parseBrowsePage(resp.Body)
|
||||
pageNum, _ := strconv.Atoi(page)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "public, max-age=300")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"novels": novels,
|
||||
"page": pageNum,
|
||||
"hasNext": hasNext,
|
||||
})
|
||||
}
|
||||
|
||||
// parseBrowsePage parses the novelfire HTML and extracts novel listings.
|
||||
// Returns novels and whether a "next page" link was found.
|
||||
func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {
|
||||
doc, err := html.Parse(r)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
var novels []NovelListing
|
||||
hasNext := false
|
||||
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode {
|
||||
switch n.Data {
|
||||
case "li":
|
||||
if hasClass(n, "novel-item") {
|
||||
if novel, ok := parseNovelItem(n); ok {
|
||||
novels = append(novels, novel)
|
||||
}
|
||||
}
|
||||
// pagination li with class "next"
|
||||
if hasClass(n, "next") {
|
||||
hasNext = true
|
||||
}
|
||||
case "a":
|
||||
// Detect "next" pagination link
|
||||
if hasClass(n, "next") || attrVal(n, "rel") == "next" {
|
||||
hasNext = true
|
||||
}
|
||||
// Also check aria-label="Next"
|
||||
if attrVal(n, "aria-label") == "Next" {
|
||||
hasNext = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(doc)
|
||||
return novels, hasNext
|
||||
}
|
||||
|
||||
// parseNovelItem extracts a NovelListing from a <li class="novel-item"> node.
|
||||
func parseNovelItem(li *html.Node) (NovelListing, bool) {
|
||||
var novel NovelListing
|
||||
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.ElementNode {
|
||||
switch n.Data {
|
||||
case "a":
|
||||
href := attrVal(n, "href")
|
||||
if strings.HasPrefix(href, "/book/") {
|
||||
slug := strings.TrimPrefix(href, "/book/")
|
||||
slug = strings.TrimSuffix(slug, "/")
|
||||
if novel.Slug == "" {
|
||||
novel.Slug = slug
|
||||
novel.URL = novelFireBase + href
|
||||
}
|
||||
}
|
||||
case "img":
|
||||
// lazy-loaded covers use data-src
|
||||
src := attrVal(n, "data-src")
|
||||
if src == "" {
|
||||
src = attrVal(n, "src")
|
||||
}
|
||||
if src != "" && novel.Cover == "" {
|
||||
novel.Cover = src
|
||||
}
|
||||
case "h4":
|
||||
if hasClass(n, "novel-title") && novel.Title == "" {
|
||||
novel.Title = strings.TrimSpace(textContent(n))
|
||||
}
|
||||
case "span":
|
||||
cls := attrVal(n, "class")
|
||||
if strings.Contains(cls, "_bl") && novel.Rank == "" {
|
||||
novel.Rank = strings.TrimSpace(textContent(n))
|
||||
}
|
||||
if strings.Contains(cls, "_br") && novel.Rating == "" {
|
||||
novel.Rating = strings.TrimSpace(textContent(n))
|
||||
}
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(li)
|
||||
|
||||
// Extract chapter count from the novel stats text (contains "N Chapters")
|
||||
novel.Chapters = extractChapters(li)
|
||||
|
||||
if novel.Slug == "" || novel.Title == "" {
|
||||
return novel, false
|
||||
}
|
||||
return novel, true
|
||||
}
|
||||
|
||||
// extractChapters finds the chapter count text within a novel-item node.
|
||||
func extractChapters(n *html.Node) string {
|
||||
var result string
|
||||
var walk func(*html.Node)
|
||||
walk = func(node *html.Node) {
|
||||
if node.Type == html.ElementNode {
|
||||
cls := attrVal(node, "class")
|
||||
if strings.Contains(cls, "novel-stats") || strings.Contains(cls, "chapter") {
|
||||
txt := strings.TrimSpace(textContent(node))
|
||||
if strings.Contains(txt, "Chapter") || strings.Contains(txt, "chapter") {
|
||||
// Extract just the numeric part if possible
|
||||
result = txt
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
for c := node.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(n)
|
||||
return result
|
||||
}
|
||||
|
||||
// hasClass reports whether an HTML node has the given CSS class.
|
||||
func hasClass(n *html.Node, cls string) bool {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == "class" {
|
||||
for _, c := range strings.Fields(a.Val) {
|
||||
if c == cls {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// attrVal returns the value of an attribute on an HTML node, or "".
|
||||
func attrVal(n *html.Node, key string) string {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == key {
|
||||
return a.Val
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// textContent returns the concatenated text content of a node and its descendants.
|
||||
func textContent(n *html.Node) string {
|
||||
if n.Type == html.TextNode {
|
||||
return n.Data
|
||||
}
|
||||
var sb strings.Builder
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
sb.WriteString(textContent(c))
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// ─── Scrape status API ────────────────────────────────────────────────────────
|
||||
|
||||
// handleScrapeStatus handles GET /api/scrape/status.
|
||||
// Returns JSON: {"running": bool}
|
||||
func (s *Server) handleScrapeStatus(w http.ResponseWriter, _ *http.Request) {
|
||||
s.mu.Lock()
|
||||
running := s.running
|
||||
s.mu.Unlock()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]bool{"running": running})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user