feat(v3): add v3 stack — backend rewrite, renamed env vars, docs
- New Go backend binary (backend + runner) replacing old scraper/ - Rename SCRAPER_API_URL → BACKEND_API_URL in UI env and docker-compose - Rename scraperFetch → backendFetch across all 19 UI server files - Remove SCRAPER_PROXY env var and proxy transport from browser.Config - Add Meilisearch, Valkey, Caddy to docker-compose - Add docs/: api-endpoints.md, request-flow.mermaid.md, data-flow.mermaid.md
This commit is contained in:
Admin
176
v3/backend/internal/runner/browse_refresh.go
Normal file
176
v3/backend/internal/runner/browse_refresh.go
Normal file
@@ -0,0 +1,176 @@
|
||||
package runner
|
||||
|
||||
// browse_refresh.go — independent 6-hour loop that fetches novelfire.net
|
||||
// browse page snapshots and stores them in MinIO.
|
||||
//
|
||||
// Design:
|
||||
// - Runs on its own ticker (BrowseRefreshInterval, default 6h) inside Run().
|
||||
// - Fetches page 1 for each combination of the standard genre/sort/status
|
||||
// filter values and stores the parsed JSON blob in MinIO via BrowseStore.
|
||||
// - The backend's handleBrowse then serves from MinIO instead of calling
|
||||
// novelfire.net live, which avoids IP-based rate-limiting on the server.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// browseNovelListing mirrors backend.NovelListing for JSON serialisation.
|
||||
type browseNovelListing struct {
|
||||
Slug string `json:"slug"`
|
||||
Title string `json:"title"`
|
||||
Cover string `json:"cover"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// browseSnapshot is the JSON structure stored in MinIO.
|
||||
type browseSnapshot struct {
|
||||
Novels []browseNovelListing `json:"novels"`
|
||||
Page int `json:"page"`
|
||||
HasNext bool `json:"hasNext"`
|
||||
// CachedAt is the UTC time the snapshot was written (ISO 8601).
|
||||
CachedAt string `json:"cachedAt"`
|
||||
}
|
||||
|
||||
// browseCombos lists the filter combinations to pre-fetch.
|
||||
// Each entry is (genre, sort, status, novelType).
|
||||
var browseCombos = []struct{ genre, sort, status, novelType string }{
|
||||
{"all", "popular", "all", "all-novel"},
|
||||
{"all", "popular", "ongoing", "all-novel"},
|
||||
{"all", "popular", "completed", "all-novel"},
|
||||
{"all", "new", "all", "all-novel"},
|
||||
{"all", "new", "ongoing", "all-novel"},
|
||||
{"all", "new", "completed", "all-novel"},
|
||||
{"all", "top-rated", "all", "all-novel"},
|
||||
{"all", "top-rated", "ongoing", "all-novel"},
|
||||
{"all", "top-rated", "completed", "all-novel"},
|
||||
}
|
||||
|
||||
const novelFireBrowseBase = "https://novelfire.net"
|
||||
|
||||
// runBrowseRefresh fetches all browse combos from novelfire.net and stores
|
||||
// the results in MinIO. Errors per-combo are logged but do not abort the
|
||||
// whole refresh cycle.
|
||||
func (r *Runner) runBrowseRefresh(ctx context.Context) {
|
||||
if r.deps.BrowseStore == nil {
|
||||
r.deps.Log.Warn("runner: browse refresh skipped — BrowseStore not configured")
|
||||
return
|
||||
}
|
||||
|
||||
log := r.deps.Log.With("op", "browse_refresh")
|
||||
log.Info("runner: browse refresh starting", "combos", len(browseCombos))
|
||||
|
||||
ok, fail := 0, 0
|
||||
for _, c := range browseCombos {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
novels, hasNext, err := fetchBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType)
|
||||
if err != nil {
|
||||
log.Warn("runner: browse fetch failed",
|
||||
"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
|
||||
fail++
|
||||
continue
|
||||
}
|
||||
|
||||
snap := browseSnapshot{
|
||||
Novels: novels,
|
||||
Page: 1,
|
||||
HasNext: hasNext,
|
||||
CachedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
data, _ := json.Marshal(snap)
|
||||
if err := r.deps.BrowseStore.PutBrowsePage(ctx, c.genre, c.sort, c.status, c.novelType, 1, data); err != nil {
|
||||
log.Warn("runner: browse put failed",
|
||||
"genre", c.genre, "sort", c.sort, "status", c.status, "err", err)
|
||||
fail++
|
||||
continue
|
||||
}
|
||||
ok++
|
||||
}
|
||||
|
||||
log.Info("runner: browse refresh finished", "ok", ok, "failed", fail)
|
||||
}
|
||||
|
||||
// fetchBrowsePage calls novelfire.net and returns a list of novel listings
|
||||
// plus a hasNext flag. Mirrors the logic in backend/handlers.go.
|
||||
func fetchBrowsePage(ctx context.Context, genre, sort, status, novelType string) ([]browseNovelListing, bool, error) {
|
||||
pageURL := fmt.Sprintf("%s/genre-%s/sort-%s/status-%s/%s?page=1",
|
||||
novelFireBrowseBase, genre, sort, status, novelType)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, pageURL, nil)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("build request: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-runner/2)")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
|
||||
httpClient := &http.Client{Timeout: 45 * time.Second}
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("fetch %s: %w", pageURL, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
return nil, false, fmt.Errorf("upstream returned %d for %s", resp.StatusCode, pageURL)
|
||||
}
|
||||
|
||||
return parseBrowseHTML(resp.Body)
|
||||
}
|
||||
|
||||
// parseBrowseHTML parses a novelfire HTML response body. Mirrors parseBrowsePage
|
||||
// in backend/handlers.go — kept separate to avoid coupling packages.
|
||||
func parseBrowseHTML(r io.Reader) ([]browseNovelListing, bool, error) {
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
body := string(data)
|
||||
|
||||
hasNext := strings.Contains(body, `rel="next"`) ||
|
||||
strings.Contains(body, `aria-label="Next"`) ||
|
||||
strings.Contains(body, `class="next"`)
|
||||
|
||||
slugRe := regexp.MustCompile(`href="/book/([^/"]+)"`)
|
||||
titleRe := regexp.MustCompile(`class="novel-title[^"]*"[^>]*>([^<]+)<`)
|
||||
coverRe := regexp.MustCompile(`data-src="(https?://[^"]+)"`)
|
||||
|
||||
slugMatches := slugRe.FindAllStringSubmatch(body, -1)
|
||||
titleMatches := titleRe.FindAllStringSubmatch(body, -1)
|
||||
coverMatches := coverRe.FindAllStringSubmatch(body, -1)
|
||||
|
||||
var novels []browseNovelListing
|
||||
seen := make(map[string]bool)
|
||||
for i, sm := range slugMatches {
|
||||
slug := sm[1]
|
||||
if seen[slug] {
|
||||
continue
|
||||
}
|
||||
seen[slug] = true
|
||||
|
||||
item := browseNovelListing{
|
||||
Slug: slug,
|
||||
URL: novelFireBrowseBase + "/book/" + slug,
|
||||
}
|
||||
if i < len(titleMatches) {
|
||||
item.Title = strings.TrimSpace(titleMatches[i][1])
|
||||
}
|
||||
if i < len(coverMatches) {
|
||||
item.Cover = coverMatches[i][1]
|
||||
}
|
||||
if item.Title != "" {
|
||||
novels = append(novels, item)
|
||||
}
|
||||
}
|
||||
|
||||
return novels, hasNext, nil
|
||||
}
|
||||
Reference in New Issue
Block a user