package runner // browse_refresh.go — independent 6-hour loop that fetches novelfire.net // browse page snapshots and stores them in MinIO. // // Design: // - Runs on its own ticker (BrowseRefreshInterval, default 6h) inside Run(). // - Fetches page 1 for each combination of the standard genre/sort/status // filter values and stores the parsed JSON blob in MinIO via BrowseStore. // - The backend's handleBrowse then serves from MinIO instead of calling // novelfire.net live, which avoids IP-based rate-limiting on the server. import ( "context" "encoding/json" "fmt" "io" "net/http" "regexp" "strings" "time" ) // browseNovelListing mirrors backend.NovelListing for JSON serialisation. type browseNovelListing struct { Slug string `json:"slug"` Title string `json:"title"` Cover string `json:"cover"` URL string `json:"url"` } // browseSnapshot is the JSON structure stored in MinIO. type browseSnapshot struct { Novels []browseNovelListing `json:"novels"` Page int `json:"page"` HasNext bool `json:"hasNext"` // CachedAt is the UTC time the snapshot was written (ISO 8601). CachedAt string `json:"cachedAt"` } // browseCombos lists the filter combinations to pre-fetch. // Each entry is (genre, sort, status, novelType). var browseCombos = []struct{ genre, sort, status, novelType string }{ {"all", "popular", "all", "all-novel"}, {"all", "popular", "ongoing", "all-novel"}, {"all", "popular", "completed", "all-novel"}, {"all", "new", "all", "all-novel"}, {"all", "new", "ongoing", "all-novel"}, {"all", "new", "completed", "all-novel"}, {"all", "top-rated", "all", "all-novel"}, {"all", "top-rated", "ongoing", "all-novel"}, {"all", "top-rated", "completed", "all-novel"}, } const novelFireBrowseBase = "https://novelfire.net" // runBrowseRefresh fetches all browse combos from novelfire.net and stores // the results in MinIO. Errors per-combo are logged but do not abort the // whole refresh cycle. func (r *Runner) runBrowseRefresh(ctx context.Context) { if r.deps.BrowseStore == nil { r.deps.Log.Warn("runner: browse refresh skipped — BrowseStore not configured") return } log := r.deps.Log.With("op", "browse_refresh") log.Info("runner: browse refresh starting", "combos", len(browseCombos)) ok, fail := 0, 0 for _, c := range browseCombos { if ctx.Err() != nil { break } novels, hasNext, err := fetchBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType) if err != nil { log.Warn("runner: browse fetch failed", "genre", c.genre, "sort", c.sort, "status", c.status, "err", err) fail++ continue } snap := browseSnapshot{ Novels: novels, Page: 1, HasNext: hasNext, CachedAt: time.Now().UTC().Format(time.RFC3339), } data, _ := json.Marshal(snap) if err := r.deps.BrowseStore.PutBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType, 1, data); err != nil { log.Warn("runner: browse put failed", "genre", c.genre, "sort", c.sort, "status", c.status, "err", err) fail++ continue } ok++ } log.Info("runner: browse refresh finished", "ok", ok, "failed", fail) } // fetchBrowsePage calls novelfire.net and returns a list of novel listings // plus a hasNext flag. Mirrors the logic in backend/handlers.go. func fetchBrowsePage(ctx context.Context, genre, sort, status, novelType string) ([]browseNovelListing, bool, error) { pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=1", novelFireBrowseBase, genre, sort, status, novelType) req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil) if err != nil { return nil, false, fmt.Errorf("build request: %w", err) } req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") req.Header.Set("Accept-Language", "en-US,en;q=0.9") httpClient := &http.Client{Timeout: 45 * time.Second} resp, err := httpClient.Do(req) if err != nil { return nil, false, fmt.Errorf("fetch %s: %w", pageURL, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { _, _ = io.Copy(io.Discard, resp.Body) return nil, false, fmt.Errorf("upstream returned %d for %s", resp.StatusCode, pageURL) } return parseBrowseHTML(resp.Body) } // parseBrowseHTML parses a novelfire HTML response body. Mirrors parseBrowsePage // in backend/handlers.go — kept separate to avoid coupling packages. func parseBrowseHTML(r io.Reader) ([]browseNovelListing, bool, error) { data, err := io.ReadAll(r) if err != nil { return nil, false, err } body := string(data) hasNext := strings.Contains(body, `rel="next"`) || strings.Contains(body, `aria-label="Next"`) || strings.Contains(body, `class="next"`) slugRe := regexp.MustCompile(`href="/book/([^/"]+)"`) titleRe := regexp.MustCompile(`class="novel-title[^"]*"[^>]*>([^<]+)<`) coverRe := regexp.MustCompile(`data-src="(https?://[^"]+)"`) slugMatches := slugRe.FindAllStringSubmatch(body, -1) titleMatches := titleRe.FindAllStringSubmatch(body, -1) coverMatches := coverRe.FindAllStringSubmatch(body, -1) var novels []browseNovelListing seen := make(map[string]bool) for i, sm := range slugMatches { slug := sm[1] if seen[slug] { continue } seen[slug] = true item := browseNovelListing{ Slug: slug, URL: novelFireBrowseBase + "/book/" + slug, } if i < len(titleMatches) { item.Title = strings.TrimSpace(titleMatches[i][1]) } if i < len(coverMatches) { item.Cover = coverMatches[i][1] } if item.Title != "" { novels = append(novels, item) } } return novels, hasNext, nil }