libnovel/backend/internal/runner/browse_refresh.go

package runner

// browse_refresh.go — independent 6-hour loop that fetches novelfire.net
// browse page snapshots and stores them in MinIO.
//
// Design:
//   - Runs on its own ticker (BrowseRefreshInterval, default 6h) inside Run().
//   - Fetches page 1 for each combination of the standard genre/sort/status
//     filter values and stores the parsed JSON blob in MinIO via BrowseStore.
//   - The backend's handleBrowse then serves from MinIO instead of calling
//     novelfire.net live, which avoids IP-based rate-limiting on the server.

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strings"
	"time"
)

// browseNovelListing mirrors backend.NovelListing for JSON serialisation.
type browseNovelListing struct {
	Slug  string `json:"slug"`
	Title string `json:"title"`
	Cover string `json:"cover"`
	URL   string `json:"url"`
}

// browseSnapshot is the JSON structure stored in MinIO.
type browseSnapshot struct {
	Novels  []browseNovelListing `json:"novels"`
	Page    int                  `json:"page"`
	HasNext bool                 `json:"hasNext"`
	// CachedAt is the UTC time the snapshot was written (ISO 8601).
	CachedAt string `json:"cachedAt"`
}

// browseCombos lists the filter combinations to pre-fetch.
// Each entry is (genre, sort, status, novelType).
var browseCombos = []struct{ genre, sort, status, novelType string }{
	{"all", "popular", "all", "all-novel"},
	{"all", "popular", "ongoing", "all-novel"},
	{"all", "popular", "completed", "all-novel"},
	{"all", "new", "all", "all-novel"},
	{"all", "new", "ongoing", "all-novel"},
	{"all", "new", "completed", "all-novel"},
	{"all", "top-rated", "all", "all-novel"},
	{"all", "top-rated", "ongoing", "all-novel"},
	{"all", "top-rated", "completed", "all-novel"},
}

const novelFireBrowseBase = "https://novelfire.net"

// runBrowseRefresh fetches all browse combos from novelfire.net and stores
// the results in MinIO. Errors per-combo are logged but do not abort the
// whole refresh cycle.
func (r *Runner) runBrowseRefresh(ctx context.Context) {
	if r.deps.BrowseStore == nil {
		r.deps.Log.Warn("runner: browse refresh skipped — BrowseStore not configured")
		return
	}

	log := r.deps.Log.With("op", "browse_refresh")
	log.Info("runner: browse refresh starting", "combos", len(browseCombos))

	ok, fail := 0, 0
	for _, c := range browseCombos {
		if ctx.Err() != nil {
			break
		}
		novels, hasNext, err := fetchBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType)
		if err != nil {
			log.Warn("runner: browse fetch failed",
				"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
			fail++
			continue
		}

		snap := browseSnapshot{
			Novels:   novels,
			Page:     1,
			HasNext:  hasNext,
			CachedAt: time.Now().UTC().Format(time.RFC3339),
		}
		data, _ := json.Marshal(snap)
		if err := r.deps.BrowseStore.PutBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType, 1, data); err != nil {
			log.Warn("runner: browse put failed",
				"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
			fail++
			continue
		}
		ok++
	}

	log.Info("runner: browse refresh finished", "ok", ok, "failed", fail)
}

// fetchBrowsePage calls novelfire.net and returns a list of novel listings
// plus a hasNext flag. Mirrors the logic in backend/handlers.go.
func fetchBrowsePage(ctx context.Context, genre, sort, status, novelType string) ([]browseNovelListing, bool, error) {
	pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=1",
		novelFireBrowseBase, genre, sort, status, novelType)

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
	if err != nil {
		return nil, false, fmt.Errorf("build request: %w", err)
	}
	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
	req.Header.Set("Accept-Language", "en-US,en;q=0.9")

	httpClient := &http.Client{Timeout: 45 * time.Second}
	resp, err := httpClient.Do(req)
	if err != nil {
		return nil, false, fmt.Errorf("fetch %s: %w", pageURL, err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		_, _ = io.Copy(io.Discard, resp.Body)
		return nil, false, fmt.Errorf("upstream returned %d for %s", resp.StatusCode, pageURL)
	}

	return parseBrowseHTML(resp.Body)
}

// parseBrowseHTML parses a novelfire HTML response body. Mirrors parseBrowsePage
// in backend/handlers.go — kept separate to avoid coupling packages.
func parseBrowseHTML(r io.Reader) ([]browseNovelListing, bool, error) {
	data, err := io.ReadAll(r)
	if err != nil {
		return nil, false, err
	}
	body := string(data)

	hasNext := strings.Contains(body, `rel="next"`) ||
		strings.Contains(body, `aria-label="Next"`) ||
		strings.Contains(body, `class="next"`)

	slugRe := regexp.MustCompile(`href="/book/([^/"]+)"`)
	titleRe := regexp.MustCompile(`class="novel-title[^"]*"[^>]*>([^<]+)<`)
	coverRe := regexp.MustCompile(`data-src="(https?://[^"]+)"`)

	slugMatches := slugRe.FindAllStringSubmatch(body, -1)
	titleMatches := titleRe.FindAllStringSubmatch(body, -1)
	coverMatches := coverRe.FindAllStringSubmatch(body, -1)

	var novels []browseNovelListing
	seen := make(map[string]bool)
	for i, sm := range slugMatches {
		slug := sm[1]
		if seen[slug] {
			continue
		}
		seen[slug] = true

		item := browseNovelListing{
			Slug: slug,
			URL:  novelFireBrowseBase + "/book/" + slug,
		}
		if i < len(titleMatches) {
			item.Title = strings.TrimSpace(titleMatches[i][1])
		}
		if i < len(coverMatches) {
			item.Cover = coverMatches[i][1]
		}
		if item.Title != "" {
			novels = append(novels, item)
		}
	}

	return novels, hasNext, nil
}