libnovel/scraper/internal/server/handlers_browse.go

package server

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/libnovel/scraper/internal/storage"
	"golang.org/x/net/html"

	"github.com/libnovel/scraper/internal/scraper/htmlutil"
)

// ─── Browse API ───────────────────────────────────────────────────────────────

// NovelListing represents a single novel entry from the novelfire browse page.
type NovelListing struct {
	Slug     string `json:"slug"`
	Title    string `json:"title"`
	Cover    string `json:"cover"`
	Rank     string `json:"rank"`
	Rating   string `json:"rating"`
	Chapters string `json:"chapters"`
	URL      string `json:"url"`
}

const novelFireBase = "https://novelfire.net"
const novelFireDomain = "novelfire.net"

// handleBrowse handles GET /api/browse.
// Query params:
//
//	page    (default 1)
//	genre   (default "all")
//	sort    (default "popular")
//	status  (default "all")
//	type    (default "all-novel")
//
// Returns JSON: {"novels":[...], "page": N, "hasNext": bool}
//
// Cache strategy: check MinIO browse bucket first (key: {domain}/html/page-N.html);
// if a snapshot exists, parse it and return structured JSON.
// On a cache miss, fetch live from novelfire.net, return the result, and
// trigger a background SingleFile snapshot + ranking population.
func (s *Server) handleBrowse(w http.ResponseWriter, r *http.Request) {
	q := r.URL.Query()
	page := q.Get("page")
	if page == "" {
		page = "1"
	}
	genre := q.Get("genre")
	if genre == "" {
		genre = "all"
	}
	sortBy := q.Get("sort")
	if sortBy == "" {
		sortBy = "popular"
	}
	status := q.Get("status")
	if status == "" {
		status = "all"
	}
	novelType := q.Get("type")
	if novelType == "" {
		novelType = "all-novel"
	}

	pageNum, _ := strconv.Atoi(page)
	if pageNum <= 0 {
		pageNum = 1
	}

	ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
	defer cancel()

	// ── Cache-first: try MinIO snapshot (new key layout) ─────────────────
	cacheKey := s.store.BrowseHTMLKey(novelFireDomain, pageNum)
	if html, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok && len(html) > 0 {
		novels, hasNext := parseBrowsePage(strings.NewReader(html))
		s.log.Debug("browse: served from cache", "key", cacheKey)
		// Still fire background ranking population in case PocketBase ranking
		// records are missing (e.g. after a schema reset / fresh deploy).
		targetURLForRanking := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
			novelFireBase, genre, sortBy, status, novelType, page)
		s.triggerDirectScrape(cacheKey, targetURLForRanking)
		w.Header().Set("Content-Type", "application/json")
		w.Header().Set("Cache-Control", "public, max-age=300")
		_ = json.NewEncoder(w).Encode(map[string]interface{}{
			"novels":  novels,
			"page":    pageNum,
			"hasNext": hasNext,
		})
		return
	}

	// ── Live fallback: direct fetch from novelfire.net ───────────────────
	// Build URL: /genre-{genre}/sort-{sort}/status-{status}/{type}?page={page}
	targetURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=%s",
		novelFireBase, genre, sortBy, status, novelType, page)

	var novels []NovelListing
	var hasNext bool
	var fetchErr error
	for attempt := 1; attempt <= 3; attempt++ {
		if attempt > 1 {
			select {
			case <-ctx.Done():
				http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
				return
			case <-time.After(time.Duration(attempt) * time.Second):
			}
		}

		var req *http.Request
		req, fetchErr = http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
		if fetchErr != nil {
			http.Error(w, `{"error":"failed to build request"}`, http.StatusInternalServerError)
			return
		}
		req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
		req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
		req.Header.Set("Accept-Language", "en-US,en;q=0.9")
		// Do NOT set Accept-Encoding manually: Go's http.Transport handles
		// transparent gzip decompression only when it adds the header itself.
		// If we set it explicitly, Transport disables auto-decompression and
		// parseBrowsePage receives raw gzip bytes instead of HTML.
		req.Header.Set("Cache-Control", "no-cache")
		req.Header.Set("Pragma", "no-cache")

		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			fetchErr = err
			s.log.Warn("browse fetch failed, retrying", "url", targetURL, "attempt", attempt, "err", err)
			continue
		}

		if resp.StatusCode != http.StatusOK {
			_, _ = io.Copy(io.Discard, resp.Body)
			resp.Body.Close()
			fetchErr = fmt.Errorf("upstream returned %d", resp.StatusCode)
			s.log.Warn("browse upstream error, retrying", "url", targetURL, "attempt", attempt, "status", resp.StatusCode)
			continue
		}

		novels, hasNext = parseBrowsePage(resp.Body)
		resp.Body.Close()
		fetchErr = nil
		break
	}
	if fetchErr != nil {
		s.log.Error("browse fetch failed after retries", "url", targetURL, "err", fetchErr)
		// ── In-memory fallback: use cached result from a prior successful fetch ──
		s.browseMemCacheMu.RLock()
		entry, memHit := s.browseMemCache[cacheKey]
		s.browseMemCacheMu.RUnlock()
		if memHit {
			s.log.Warn("browse: upstream unavailable, serving stale in-memory cache",
				"key", cacheKey, "age", time.Since(entry.cachedAt).Round(time.Second))
			w.Header().Set("Content-Type", "application/json")
			w.Header().Set("Cache-Control", "public, max-age=60")
			_ = json.NewEncoder(w).Encode(map[string]interface{}{
				"novels":  entry.novels,
				"page":    pageNum,
				"hasNext": entry.hasNext,
			})
			return
		}
		http.Error(w, fmt.Sprintf(`{"error":"%s"}`, fetchErr.Error()), http.StatusBadGateway)
		return
	}

	// ── Populate in-memory cache with the fresh upstream result ──────────
	if len(novels) > 0 {
		s.browseMemCacheMu.Lock()
		s.browseMemCache[cacheKey] = browseCacheEntry{
			novels:   novels,
			hasNext:  hasNext,
			cachedAt: time.Now(),
		}
		s.browseMemCacheMu.Unlock()
	}

	// ── Background: fetch and cache page directly from novelfire.net ─────
	// Fire-and-forget: stores raw HTML in MinIO and populates the ranking
	// collection in PocketBase (no browser/SingleFile needed).
	s.triggerDirectScrape(cacheKey, targetURL)

	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Cache-Control", "public, max-age=300")
	_ = json.NewEncoder(w).Encode(map[string]interface{}{
		"novels":  novels,
		"page":    pageNum,
		"hasNext": hasNext,
	})
}

// triggerDirectScrape fires a background goroutine that:
//  1. Fetches pageURL directly from novelfire.net using Go's HTTP client
//     (no browser/SingleFile needed — the page is server-rendered HTML).
//  2. Stores the raw HTML in MinIO at cacheKey so future requests are served
//     from cache without hitting the origin.
//  3. Parses the HTML to extract novel listings.
//  4. For each listing, upserts a ranking record in PocketBase (rank, slug,
//     title, cover key, source_url).
//  5. Fires a separate goroutine per cover image to download and store it at
//     {domain}/assets/book-covers/{slug}.jpg in MinIO.
//
// It is a no-op when a refresh for this cache key is already in progress.
// The goroutine uses a fresh context so it outlives the HTTP request.
func (s *Server) triggerDirectScrape(cacheKey, pageURL string) {
	s.browseMu.Lock()
	if _, inflight := s.browseInFlight[cacheKey]; inflight {
		s.browseMu.Unlock()
		return
	}
	s.browseInFlight[cacheKey] = struct{}{}
	s.browseMu.Unlock()

	go func() {
		defer func() {
			s.browseMu.Lock()
			delete(s.browseInFlight, cacheKey)
			s.browseMu.Unlock()
		}()

		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
		defer cancel()

		req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
		if err != nil {
			s.log.Warn("triggerDirectScrape: build request failed", "key", cacheKey, "err", err)
			return
		}
		req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
		req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
		req.Header.Set("Accept-Language", "en-US,en;q=0.9")

		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			s.log.Warn("triggerDirectScrape: fetch failed", "key", cacheKey, "err", err)
			return
		}
		defer resp.Body.Close()

		if resp.StatusCode != http.StatusOK {
			s.log.Warn("triggerDirectScrape: non-200 response", "key", cacheKey, "status", resp.StatusCode)
			return
		}

		htmlBytes, readErr := io.ReadAll(resp.Body)
		if readErr != nil {
			s.log.Warn("triggerDirectScrape: read body failed", "key", cacheKey, "err", readErr)
			return
		}
		if len(htmlBytes) == 0 {
			s.log.Warn("triggerDirectScrape: empty response body", "key", cacheKey)
			return
		}

		// Store the HTML in MinIO so subsequent requests are cache-hits.
		if putErr := s.store.SaveBrowsePage(ctx, cacheKey, string(htmlBytes)); putErr != nil {
			s.log.Warn("triggerDirectScrape: SaveBrowsePage failed", "key", cacheKey, "err", putErr)
			// Non-fatal: continue to populate PocketBase/covers even if MinIO write fails.
		} else {
			s.log.Info("triggerDirectScrape: cached browse page", "key", cacheKey, "bytes", len(htmlBytes))
		}

		// Parse to extract novel listings.
		novels, _ := parseBrowsePage(strings.NewReader(string(htmlBytes)))
		if len(novels) == 0 {
			s.log.Warn("triggerDirectScrape: no novels parsed", "key", cacheKey)
			return
		}

		// Upsert each novel into PocketBase ranking and kick off cover downloads.
		for i, novel := range novels {
			rank := i + 1
			coverKey := s.store.BrowseCoverKey(novelFireDomain, novel.Slug)

			item := storage.RankingItem{
				Rank:      rank,
				Slug:      novel.Slug,
				Title:     novel.Title,
				Cover:     coverKey, // stored as MinIO key; UI fetches via /api/cover/...
				SourceURL: novel.URL,
			}
			if werr := s.store.WriteRankingItem(ctx, item); werr != nil {
				s.log.Warn("triggerDirectScrape: WriteRankingItem failed",
					"slug", novel.Slug, "err", werr)
			}

			if novel.Cover != "" {
				go s.downloadAndStoreCover(coverKey, novel.Cover)
			}
		}

		s.log.Info("triggerDirectScrape: ranking populated", "count", len(novels), "key", cacheKey)
	}()
}

// warmBrowseCache checks whether the browse cache for page 1 is populated in
// MinIO and, if not, triggers a background direct scrape.  This is called
// once on server startup so the first user request is likely served from cache.
func (s *Server) warmBrowseCache() {
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	cacheKey := s.store.BrowseHTMLKey(novelFireDomain, 1)
	if _, ok, err := s.store.GetBrowsePage(ctx, cacheKey); err == nil && ok {
		s.log.Debug("warmBrowseCache: page 1 already cached, skipping")
		return
	}

	targetURL := fmt.Sprintf("%s/genre-all/sort-popular/status-all/all-novel?page=1", novelFireBase)
	s.log.Info("warmBrowseCache: page 1 not cached, triggering background scrape")
	s.triggerDirectScrape(cacheKey, targetURL)
}

// downloadAndStoreCover delegates to storage.DownloadAndStoreCover.
func (s *Server) downloadAndStoreCover(key, imageURL string) {
	storage.DownloadAndStoreCover(s.store, s.log, key, imageURL)
}

// parseBrowsePage parses the novelfire HTML and extracts novel listings.
// Returns novels and whether a "next page" link was found.
func parseBrowsePage(r io.Reader) ([]NovelListing, bool) {
	doc, err := html.Parse(r)
	if err != nil {
		return nil, false
	}

	var novels []NovelListing
	hasNext := false

	var walk func(*html.Node)
	walk = func(n *html.Node) {
		if n.Type == html.ElementNode {
			switch n.Data {
			case "li":
				if hasClass(n, "novel-item") {
					if novel, ok := parseNovelItem(n); ok {
						novels = append(novels, novel)
					}
				}
				// pagination li with class "next"
				if hasClass(n, "next") {
					hasNext = true
				}
			case "a":
				// Detect "next" pagination link
				if hasClass(n, "next") || attrVal(n, "rel") == "next" {
					hasNext = true
				}
				// Also check aria-label="Next"
				if attrVal(n, "aria-label") == "Next" {
					hasNext = true
				}
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(doc)
	return novels, hasNext
}

// parseNovelItem extracts a NovelListing from a <li class="novel-item"> node.
func parseNovelItem(li *html.Node) (NovelListing, bool) {
	var novel NovelListing

	var walk func(*html.Node)
	walk = func(n *html.Node) {
		if n.Type == html.ElementNode {
			switch n.Data {
			case "a":
				href := attrVal(n, "href")
				if strings.HasPrefix(href, "/book/") {
					slug := strings.TrimPrefix(href, "/book/")
					slug = strings.TrimSuffix(slug, "/")
					if novel.Slug == "" {
						novel.Slug = slug
						novel.URL = novelFireBase + href
					}
				}
			case "img":
				// lazy-loaded covers use data-src
				src := attrVal(n, "data-src")
				if src == "" {
					src = attrVal(n, "src")
				}
				if src != "" && novel.Cover == "" {
					if !strings.HasPrefix(src, "http") {
						src = novelFireBase + src
					}
					novel.Cover = src
				}
			case "h4":
				if hasClass(n, "novel-title") && novel.Title == "" {
					novel.Title = strings.TrimSpace(textContent(n))
				}
			case "span":
				cls := attrVal(n, "class")
				if strings.Contains(cls, "_bl") && novel.Rank == "" {
					novel.Rank = strings.TrimSpace(textContent(n))
				}
				if strings.Contains(cls, "_br") && novel.Rating == "" {
					novel.Rating = strings.TrimSpace(textContent(n))
				}
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(li)

	// Extract chapter count from the novel stats text (contains "N Chapters")
	novel.Chapters = extractChapters(li)

	if novel.Slug == "" || novel.Title == "" {
		return novel, false
	}
	return novel, true
}

// extractChapters finds the chapter count text within a novel-item node.
func extractChapters(n *html.Node) string {
	var result string
	var walk func(*html.Node)
	walk = func(node *html.Node) {
		if node.Type == html.ElementNode {
			cls := attrVal(node, "class")
			if strings.Contains(cls, "novel-stats") || strings.Contains(cls, "chapter") {
				txt := strings.TrimSpace(textContent(node))
				if strings.Contains(txt, "Chapter") || strings.Contains(txt, "chapter") {
					// Extract just the numeric part if possible
					result = txt
					return
				}
			}
		}
		for c := node.FirstChild; c != nil; c = c.NextSibling {
			walk(c)
		}
	}
	walk(n)
	return result
}

// hasClass reports whether an HTML node has the given CSS class.
func hasClass(n *html.Node, cls string) bool {
	for _, a := range n.Attr {
		if a.Key == "class" {
			for _, c := range strings.Fields(a.Val) {
				if c == cls {
					return true
				}
			}
		}
	}
	return false
}

// attrVal returns the value of an attribute on an HTML node, or "".
// Delegates to htmlutil.AttrVal.
func attrVal(n *html.Node, key string) string { return htmlutil.AttrVal(n, key) }

// textContent returns the concatenated text content of a node and its descendants.
// Delegates to htmlutil.TextContent.
func textContent(n *html.Node) string { return htmlutil.TextContent(n) }

// ─── Search API ───────────────────────────────────────────────────────────────

// handleSearch handles GET /api/search.
//
// Query params:
//
//	q      — search query string (required, min 2 chars)
//	source — "local" | "remote" | "all" (default: "all")
//
// When source includes "local", it searches books already in the local store
// by title substring match.  When source includes "remote", it fetches the
// novelfire.net search page and parses results.  Results from both sources
// are merged with local results first (de-duplicated by slug).
//
// Returns JSON: {"results": [...NovelListing], "local_count": N, "remote_count": N}
func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
	q := r.URL.Query().Get("q")
	if len([]rune(q)) < 2 {
		http.Error(w, `{"error":"query must be at least 2 characters"}`, http.StatusBadRequest)
		return
	}

	source := r.URL.Query().Get("source")
	if source == "" {
		source = "all"
	}

	ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second)
	defer cancel()

	var localResults []NovelListing
	var remoteResults []NovelListing

	// ── Local search (PocketBase books) ──────────────────────────────────
	if source == "local" || source == "all" {
		books, err := s.store.ListBooks(ctx)
		if err != nil {
			s.log.Warn("search: ListBooks failed", "err", err)
		} else {
			qLower := strings.ToLower(q)
			for _, b := range books {
				if strings.Contains(strings.ToLower(b.Title), qLower) ||
					strings.Contains(strings.ToLower(b.Author), qLower) {
					listing := NovelListing{
						Slug:  b.Slug,
						Title: b.Title,
						Cover: b.Cover,
						URL:   b.SourceURL,
					}
					localResults = append(localResults, listing)
				}
			}
		}
	}

	// ── Remote search (novelfire.net /search?keyword=...) ─────────────────
	if source == "remote" || source == "all" {
		searchURL := novelFireBase + "/search?keyword=" + url.QueryEscape(q)

		req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil)
		if err == nil {
			req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
			req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
			req.Header.Set("Accept-Language", "en-US,en;q=0.9")
			if resp, fetchErr := http.DefaultClient.Do(req); fetchErr == nil {
				defer resp.Body.Close()
				if resp.StatusCode == http.StatusOK {
					parsed, _ := parseBrowsePage(resp.Body)
					remoteResults = parsed
				} else {
					s.log.Warn("search: remote returned non-200", "status", resp.StatusCode, "url", searchURL)
				}
			}
		}
	}

	// ── Merge: de-duplicate remote results already in local ───────────────
	localSlugs := make(map[string]bool, len(localResults))
	for _, item := range localResults {
		localSlugs[item.Slug] = true
	}

	combined := make([]NovelListing, 0, len(localResults)+len(remoteResults))
	combined = append(combined, localResults...)
	for _, item := range remoteResults {
		if !localSlugs[item.Slug] {
			combined = append(combined, item)
		}
	}

	w.Header().Set("Content-Type", "application/json")
	_ = json.NewEncoder(w).Encode(map[string]interface{}{
		"results":      combined,
		"local_count":  len(localResults),
		"remote_count": len(remoteResults),
	})
}