kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

feat(v3): Caddy hardening, Watchtower, scrape fixes, docs diagrams

Browse Source 
- Caddy: custom image with caddy-ratelimit plugin, security headers
  (X-Frame-Options, HSTS, CSP-adjacent, etc.), per-IP rate limiting on
  auth/scrape/global zones, static error pages (502/503/504), fix routing
  to remove /api/scrape/* and /api/chapter-text-preview/* direct-to-backend
  (were bypassing SvelteKit auth middleware)
- docker-compose: Caddy build context + error volume, Watchtower service
  (label-enable mode, 5 min poll), watchtower labels on backend/runner/ui
- Scraper: ScrapeChapterList uses retryGet (9 attempts, Retry-After backoff)
  to fix 429-induced chapter list failures; upTo param stops pagination early
  for range scrapes
- UI: Browse→Catalogue rename (routes, API, links), admin scrape page
  Continue/Retry buttons, +error.svelte branded error page, type cleanup
  (removed dead exports, added BookPreviewMeta/BookPreviewResponse to scraper.ts)
- Meilisearch: meta_updated field, sort=update fix, facet distribution
- Docs: reorganise into docs/d2/ and docs/mermaid/ subdirectories, update
  all diagrams to reflect Caddy/Watchtower/routing changes, add
  api-routing.d2 ownership map with auth-level colour coding, regenerate SVGs
This commit is contained in:
Admin
2026-03-22 21:10:38 +05:00
parent a85636d5db
commit 5b27d501af
60 changed files with 2485 additions and 908 deletions
Download Patch File Download Diff File Expand all files Collapse all files

177
v3/backend/internal/backend/handlers.go
View File

@@ -7,8 +7,7 @@ package backend
// handleScrapeStatus, handleScrapeTasks
// handleBrowse, handleSearch
// handleGetRanking, handleGetCover
// handleBookPreview, handleChapterText, handleReindex
// handleChapterText, handleReindex
// handleBookPreview, handleChapterText, handleChapterTextPreview, handleChapterMarkdown, handleReindex
// handleAudioGenerate, handleAudioStatus, handleAudioProxy
// handleVoices
// handlePresignChapter, handlePresignAudio, handlePresignVoiceSample
@@ -29,6 +28,8 @@ package backend
// by the runner after each catalogue scrape).
// - GET /api/book-preview returns stored data when in library, or enqueues a
// scrape task and returns 202 when not. The backend never scrapes directly.
// - GET /api/chapter-text-preview scrapes a chapter live from novelfire.net
// directly (no runner task, no store writes). Used for unscraped books.
import (
"context"
@@ -45,6 +46,8 @@ import (
"github.com/libnovel/backend/internal/domain"
"github.com/libnovel/backend/internal/kokoro"
"github.com/libnovel/backend/internal/meili"
"github.com/libnovel/backend/internal/novelfire/htmlutil"
"github.com/libnovel/backend/internal/scraper"
)
const (
@@ -502,6 +505,117 @@ func (s *Server) handleChapterMarkdown(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, raw)
}
// handleChapterTextPreview handles GET /api/chapter-text-preview/{slug}/{n}.
//
// Fetches a chapter live from novelfire.net and returns its plain text without
// writing anything to PocketBase or MinIO. This is the preview path used when
// a chapter has not yet been scraped into the library.
//
// Optional query params:
//
// chapter_url — the canonical chapter URL (preferred over constructing one)
// title — hint for the chapter title (used when the page title is empty)
//
// Response: {"slug":string,"number":int,"title":string,"text":string,"url":string}
func (s *Server) handleChapterTextPreview(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 || slug == "" {
jsonError(w, http.StatusBadRequest, "invalid slug or chapter number")
return
}
// Determine the chapter URL to fetch.
chapterURL := r.URL.Query().Get("chapter_url")
if chapterURL == "" {
// Best-effort: novelfire chapter URLs follow /book/{slug}/chapter-{n}
chapterURL = fmt.Sprintf("%s/book/%s/chapter-%d", novelFireBase, slug, n)
}
titleHint := r.URL.Query().Get("title")
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
defer cancel()
// Fetch the chapter page.
req, err := http.NewRequestWithContext(ctx, http.MethodGet, chapterURL, nil)
if err != nil {
s.deps.Log.Error("chapter-text-preview: build request failed", "url", chapterURL, "err", err)
jsonError(w, http.StatusInternalServerError, "failed to build request")
return
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-backend/2)")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
resp, err := http.DefaultClient.Do(req)
if err != nil {
s.deps.Log.Warn("chapter-text-preview: fetch failed", "url", chapterURL, "err", err)
jsonError(w, http.StatusBadGateway, "failed to fetch chapter")
return
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
jsonError(w, http.StatusNotFound, "chapter not found")
return
}
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
s.deps.Log.Warn("chapter-text-preview: upstream error",
"url", chapterURL, "status", resp.StatusCode, "body_snippet", string(body))
jsonError(w, http.StatusBadGateway, fmt.Sprintf("upstream returned %d", resp.StatusCode))
return
}
bodyBytes, err := io.ReadAll(resp.Body)
if err != nil {
s.deps.Log.Error("chapter-text-preview: read body failed", "err", err)
jsonError(w, http.StatusInternalServerError, "failed to read response")
return
}
// Parse HTML and extract the #content node.
root, err := htmlutil.ParseHTML(string(bodyBytes))
if err != nil {
s.deps.Log.Error("chapter-text-preview: html parse failed", "err", err)
jsonError(w, http.StatusInternalServerError, "failed to parse chapter HTML")
return
}
container := htmlutil.FindFirst(root, scraper.Selector{ID: "content"})
if container == nil {
s.deps.Log.Warn("chapter-text-preview: #content not found", "url", chapterURL)
jsonError(w, http.StatusNotFound, "chapter content not found on page")
return
}
markdownText := htmlutil.NodeToMarkdown(container)
plainText := stripMarkdown(markdownText)
// Extract the chapter title from the page <title> or <h1> if not hinted.
chapterTitle := titleHint
if chapterTitle == "" {
// Try <h1 class="chapter-title"> first, then <h2 class="chapter-title">
for _, tag := range []string{"h1", "h2", "h3"} {
if node := htmlutil.FindFirst(root, scraper.Selector{Tag: tag, Class: "chapter-title"}); node != nil {
chapterTitle = strings.TrimSpace(htmlutil.TextContent(node))
break
}
}
}
if chapterTitle == "" {
chapterTitle = fmt.Sprintf("Chapter %d", n)
}
writeJSON(w, 0, map[string]any{
"slug": slug,
"number": n,
"title": chapterTitle,
"text": plainText,
"url": chapterURL,
})
}
// handleReindex handles POST /api/reindex/{slug}.
// Rebuilds the chapters_idx PocketBase collection for a book from MinIO objects.
func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) {
@@ -741,6 +855,59 @@ func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request
writeJSON(w, 0, map[string]string{"url": u})
}
// handleAvatarUpload handles PUT /api/avatar-upload/{userId}.
// The request body must be the raw image bytes; Content-Type must be
// image/jpeg, image/png, or image/webp.
//
// This endpoint is called by the SvelteKit server (not the browser directly),
// so MinIO credentials and internal networking are not a concern.
//
// Returns: { "key": "<objectKey>" }
func (s *Server) handleAvatarUpload(w http.ResponseWriter, r *http.Request) {
userID := r.PathValue("userId")
if userID == "" {
jsonError(w, http.StatusBadRequest, "missing userId")
return
}
ct := r.Header.Get("Content-Type")
var ext string
switch {
case strings.HasPrefix(ct, "image/jpeg"):
ext = "jpg"
case strings.HasPrefix(ct, "image/png"):
ext = "png"
case strings.HasPrefix(ct, "image/webp"):
ext = "webp"
default:
jsonError(w, http.StatusBadRequest, "unsupported content-type; use image/jpeg, image/png, or image/webp")
return
}
const maxSize = 5 << 20 // 5 MiB
data, err := io.ReadAll(io.LimitReader(r.Body, maxSize+1))
if err != nil {
jsonError(w, http.StatusBadRequest, "failed to read body")
return
}
if len(data) > maxSize {
jsonError(w, http.StatusRequestEntityTooLarge, "image too large (max 5 MiB)")
return
}
if len(data) == 0 {
jsonError(w, http.StatusBadRequest, "empty body")
return
}
key, err := s.deps.PresignStore.PutAvatar(r.Context(), userID, ext, ct, data)
if err != nil {
s.deps.Log.Error("avatar upload failed", "userId", userID, "err", err)
jsonError(w, http.StatusInternalServerError, "upload failed")
return
}
writeJSON(w, 0, map[string]string{"key": key})
}
// handlePresignAvatarUpload handles GET /api/presign/avatar-upload/{userId}.
// Query params: ext (jpg|png|webp, defaults to jpg)
func (s *Server) handlePresignAvatarUpload(w http.ResponseWriter, r *http.Request) {
@@ -912,7 +1079,7 @@ func (s *Server) handleCatalogue(w http.ResponseWriter, r *http.Request) {
Limit: limit,
}
books, total, err := s.deps.SearchIndex.Catalogue(r.Context(), cq)
books, total, facets, err := s.deps.SearchIndex.Catalogue(r.Context(), cq)
if err != nil {
s.deps.Log.Error("handleCatalogue: Catalogue query failed", "err", err)
jsonError(w, http.StatusInternalServerError, "search failed")
@@ -928,6 +1095,10 @@ func (s *Server) handleCatalogue(w http.ResponseWriter, r *http.Request) {
"limit": limit,
"total": total,
"has_next": hasNext,
"facets": map[string]any{
"genres": facets.Genres,
"statuses": facets.Statuses,
},
})
}

5
v3/backend/internal/backend/server.go
View File

@@ -144,6 +144,10 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
// Use this instead of presign+fetch to avoid SvelteKit→MinIO network path.
mux.HandleFunc("GET /api/chapter-markdown/{slug}/{n}", s.handleChapterMarkdown)
// Chapter text preview — live scrape from novelfire.net, no store writes.
// Used when the chapter is not yet in the library (preview mode).
mux.HandleFunc("GET /api/chapter-text-preview/{slug}/{n}", s.handleChapterTextPreview)
// Reindex chapters_idx from MinIO
mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex)
@@ -161,6 +165,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
mux.HandleFunc("GET /api/presign/voice-sample/{voice}", s.handlePresignVoiceSample)
mux.HandleFunc("GET /api/presign/avatar-upload/{userId}", s.handlePresignAvatarUpload)
mux.HandleFunc("GET /api/presign/avatar/{userId}", s.handlePresignAvatar)
mux.HandleFunc("PUT /api/avatar-upload/{userId}", s.handleAvatarUpload)
// Reading progress
mux.HandleFunc("GET /api/progress", s.handleGetProgress)

4
v3/backend/internal/bookstore/bookstore.go
View File

@@ -105,6 +105,10 @@ type PresignStore interface {
// Returns ("", false, nil) when no avatar exists.
PresignAvatarURL(ctx context.Context, userID string) (string, bool, error)
// PutAvatar stores raw image bytes for a user avatar directly in MinIO.
// ext should be "jpg", "png", or "webp". Returns the object key.
PutAvatar(ctx context.Context, userID, ext, contentType string, data []byte) (key string, err error)
// DeleteAvatar removes all avatar objects for a user.
DeleteAvatar(ctx context.Context, userID string) error
}

3
v3/backend/internal/bookstore/bookstore_test.go
View File

@@ -68,6 +68,9 @@ func (m *mockStore) PresignAvatarUpload(_ context.Context, _, _ string) (string,
func (m *mockStore) PresignAvatarURL(_ context.Context, _ string) (string, bool, error) {
return "", false, nil
}
func (m *mockStore) PutAvatar(_ context.Context, _, _, _ string, _ []byte) (string, error) {
return "", nil
}
func (m *mockStore) DeleteAvatar(_ context.Context, _ string) error { return nil }
// ProgressStore

24
v3/backend/internal/config/config.go
View File

@@ -101,6 +101,11 @@ type Runner struct {
// scrapes per-book metadata, downloads covers, and re-indexes in Meilisearch.
// Defaults to 24h. Set to 0 to use the default.
CatalogueRefreshInterval time.Duration
// SkipInitialCatalogueRefresh prevents the runner from running a full
// catalogue walk on startup. Useful for quick restarts where the catalogue
// is already indexed and a 24h walk would be wasteful.
// Controlled by RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true.
SkipInitialCatalogueRefresh bool
}
// Config is the top-level configuration struct consumed by both binaries.
@@ -142,7 +147,7 @@ func Load() Config {
PublicUseSSL: envBool("MINIO_PUBLIC_USE_SSL", true),
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
BucketAvatars: envOr("MINIO_BUCKET_AVATARS", "libnovel-avatars"),
BucketAvatars: envOr("MINIO_BUCKET_AVATARS", "avatars"),
BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"),
},
@@ -156,14 +161,15 @@ func Load() Config {
},
Runner: Runner{
PollInterval: envDuration("RUNNER_POLL_INTERVAL", 30*time.Second),
MaxConcurrentScrape: envInt("RUNNER_MAX_CONCURRENT_SCRAPE", 1),
MaxConcurrentAudio: envInt("RUNNER_MAX_CONCURRENT_AUDIO", 1),
WorkerID: envOr("RUNNER_WORKER_ID", workerID),
Workers: envInt("RUNNER_WORKERS", 0), // 0 → runtime.NumCPU()
Timeout: envDuration("RUNNER_TIMEOUT", 90*time.Second),
MetricsAddr: envOr("RUNNER_METRICS_ADDR", ":9091"),
CatalogueRefreshInterval: envDuration("RUNNER_CATALOGUE_REFRESH_INTERVAL", 0),
PollInterval: envDuration("RUNNER_POLL_INTERVAL", 30*time.Second),
MaxConcurrentScrape: envInt("RUNNER_MAX_CONCURRENT_SCRAPE", 1),
MaxConcurrentAudio: envInt("RUNNER_MAX_CONCURRENT_AUDIO", 1),
WorkerID: envOr("RUNNER_WORKER_ID", workerID),
Workers: envInt("RUNNER_WORKERS", 0), // 0 → runtime.NumCPU()
Timeout: envDuration("RUNNER_TIMEOUT", 90*time.Second),
MetricsAddr: envOr("RUNNER_METRICS_ADDR", ":9091"),
CatalogueRefreshInterval: envDuration("RUNNER_CATALOGUE_REFRESH_INTERVAL", 0),
SkipInitialCatalogueRefresh: envBool("RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH", false),
},
Meilisearch: Meilisearch{

4
v3/backend/internal/domain/domain.go
View File

@@ -20,6 +20,10 @@ type BookMeta struct {
SourceURL string `json:"source_url"`
Ranking int `json:"ranking,omitempty"`
Rating float64 `json:"rating,omitempty"`
// MetaUpdated is the Unix timestamp (seconds) when the book record was last
// updated in PocketBase. Populated on read; not sent on write (PocketBase
// manages its own updated field).
MetaUpdated int64 `json:"meta_updated,omitempty"`
}
// CatalogueEntry is a lightweight book reference returned by catalogue pages.

81
v3/backend/internal/meili/client.go
View File

@@ -6,7 +6,7 @@
// - Primary key: "slug"
// - Searchable attributes: title, author, genres, summary
// - Filterable attributes: status, genres
// - Sortable attributes: rank, rating, total_chapters
// - Sortable attributes: rank, rating, total_chapters, meta_updated
//
// The client is intentionally simple: UpsertBook and Search only. All
// Meilisearch-specific details (index management, attribute configuration)
@@ -32,8 +32,9 @@ type Client interface {
// Search returns up to limit books matching query.
Search(ctx context.Context, query string, limit int) ([]domain.BookMeta, error)
// Catalogue queries books with optional filters, sort, and pagination.
// Returns books and the total hit count for pagination.
Catalogue(ctx context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, error)
// Returns books, the total hit count for pagination, and a FacetResult
// with available genre and status values from the index.
Catalogue(ctx context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, FacetResult, error)
}
// CatalogueQuery holds parameters for the /api/catalogue endpoint.
@@ -41,11 +42,18 @@ type CatalogueQuery struct {
Q string // full-text query (may be empty for browse)
Genre string // genre filter, e.g. "fantasy" or "all"
Status string // status filter, e.g. "ongoing", "completed", or "all"
Sort string // sort field: "popular", "new", "top-rated", "rank", ""
Sort string // sort field: "popular", "new", "update", "top-rated", "rank", ""
Page int // 1-indexed
Limit int // items per page, default 20
}
// FacetResult holds the available filter values discovered from the index.
// Values are sorted alphabetically and include only those present in the index.
type FacetResult struct {
Genres []string // distinct genre values
Statuses []string // distinct status values
}
// MeiliClient wraps the meilisearch-go SDK.
type MeiliClient struct {
idx meilisearch.IndexManager
@@ -93,7 +101,7 @@ func Configure(host, apiKey string) error {
return fmt.Errorf("meili: update filterable attributes: %w", err)
}
sortable := []string{"rank", "rating", "total_chapters"}
sortable := []string{"rank", "rating", "total_chapters", "meta_updated"}
if _, err := idx.UpdateSortableAttributes(&sortable); err != nil {
return fmt.Errorf("meili: update sortable attributes: %w", err)
}
@@ -114,6 +122,9 @@ type bookDoc struct {
SourceURL string `json:"source_url"`
Rank int `json:"rank"`
Rating float64 `json:"rating"`
// MetaUpdated is the Unix timestamp (seconds) of the last PocketBase update.
// Used for sort=update ("recently updated" ordering).
MetaUpdated int64 `json:"meta_updated"`
}
func toDoc(b domain.BookMeta) bookDoc {
@@ -129,6 +140,7 @@ func toDoc(b domain.BookMeta) bookDoc {
SourceURL: b.SourceURL,
Rank: b.Ranking,
Rating: b.Rating,
MetaUpdated: b.MetaUpdated,
}
}
@@ -145,6 +157,7 @@ func fromDoc(d bookDoc) domain.BookMeta {
SourceURL: d.SourceURL,
Ranking: d.Rank,
Rating: d.Rating,
MetaUpdated: d.MetaUpdated,
}
}
@@ -188,8 +201,9 @@ func (c *MeiliClient) Search(_ context.Context, query string, limit int) ([]doma
}
// Catalogue queries books with optional full-text search, genre/status filters,
// sort order, and pagination. Returns matching books and the total estimate.
func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, error) {
// sort order, and pagination. Returns matching books, the total estimate, and
// a FacetResult containing available genre and status values from the index.
func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, FacetResult, error) {
if q.Limit <= 0 {
q.Limit = 20
}
@@ -200,6 +214,9 @@ func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.B
req := &meilisearch.SearchRequest{
Limit: int64(q.Limit),
Offset: int64((q.Page - 1) * q.Limit),
// Request facet distribution so the UI can build filter options
// dynamically without hardcoding genre/status lists.
Facets: []string{"genres", "status"},
}
// Build filter
@@ -214,7 +231,7 @@ func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.B
req.Filter = strings.Join(filters, " AND ")
}
// Map UI sort tokens to Meilisearch sort expressions
// Map UI sort tokens to Meilisearch sort expressions.
switch q.Sort {
case "rank":
req.Sort = []string{"rank:asc"}
@@ -222,12 +239,14 @@ func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.B
req.Sort = []string{"rating:desc"}
case "new":
req.Sort = []string{"total_chapters:desc"}
case "update":
req.Sort = []string{"meta_updated:desc"}
// "popular" and "" → relevance (no explicit sort)
}
res, err := c.idx.Search(q.Q, req)
if err != nil {
return nil, 0, fmt.Errorf("meili: catalogue query: %w", err)
return nil, 0, FacetResult{}, fmt.Errorf("meili: catalogue query: %w", err)
}
books := make([]domain.BookMeta, 0, len(res.Hits))
@@ -242,7 +261,45 @@ func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.B
}
books = append(books, fromDoc(doc))
}
return books, res.EstimatedTotalHits, nil
facets := parseFacets(res.FacetDistribution)
return books, res.EstimatedTotalHits, facets, nil
}
// parseFacets extracts sorted genre and status slices from a Meilisearch
// facetDistribution raw JSON value.
// The JSON shape is: {"genres":{"fantasy":12,"action":5},"status":{"ongoing":7}}
func parseFacets(raw json.RawMessage) FacetResult {
var result FacetResult
if len(raw) == 0 {
return result
}
var dist map[string]map[string]int64
if err := json.Unmarshal(raw, &dist); err != nil {
return result
}
if m, ok := dist["genres"]; ok {
for k := range m {
result.Genres = append(result.Genres, k)
}
sortStrings(result.Genres)
}
if m, ok := dist["status"]; ok {
for k := range m {
result.Statuses = append(result.Statuses, k)
}
sortStrings(result.Statuses)
}
return result
}
// sortStrings sorts a slice of strings in place.
func sortStrings(s []string) {
for i := 1; i < len(s); i++ {
for j := i; j > 0 && s[j] < s[j-1]; j-- {
s[j], s[j-1] = s[j-1], s[j]
}
}
}
// NoopClient is a no-op Client used when Meilisearch is not configured.
@@ -252,6 +309,6 @@ func (NoopClient) UpsertBook(_ context.Context, _ domain.BookMeta) error { retur
func (NoopClient) Search(_ context.Context, _ string, _ int) ([]domain.BookMeta, error) {
return nil, nil
}
func (NoopClient) Catalogue(_ context.Context, _ CatalogueQuery) ([]domain.BookMeta, int64, error) {
return nil, 0, nil
func (NoopClient) Catalogue(_ context.Context, _ CatalogueQuery) ([]domain.BookMeta, int64, FacetResult, error) {
return nil, 0, FacetResult{}, nil
}

17
v3/backend/internal/novelfire/scraper.go
View File

@@ -194,8 +194,12 @@ func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.Bo
// ── ChapterListProvider ───────────────────────────────────────────────────────
// ScrapeChapterList returns all chapter references for a book, ordered ascending.
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]domain.ChapterRef, error) {
// ScrapeChapterList returns chapter references for a book, ordered ascending.
// upTo > 0 stops pagination as soon as at least upTo chapter numbers have been
// collected — use this for range scrapes so we don't paginate 100 pages just
// to discover refs we'll never scrape. upTo == 0 fetches all pages.
// Each page fetch uses retryGet with 429-aware exponential backoff.
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string, upTo int) ([]domain.ChapterRef, error) {
var refs []domain.ChapterRef
baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters"
page := 1
@@ -210,7 +214,7 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]doma
pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page)
s.log.Info("scraping chapter list", "page", page, "url", pageURL)
raw, err := s.client.GetContent(ctx, pageURL)
raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 6*time.Second)
if err != nil {
return refs, fmt.Errorf("chapter list page %d: %w", page, err)
}
@@ -255,6 +259,13 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]doma
})
}
// Early-stop: if we have seen at least upTo chapter numbers, we have
// enough refs to cover the requested range — no need to paginate further.
if upTo > 0 && len(refs) > 0 && refs[len(refs)-1].Number >= upTo {
s.log.Debug("chapter list early-stop reached", "upTo", upTo, "collected", len(refs))
break
}
page++
}

2
v3/backend/internal/orchestrator/orchestrator.go
View File

@@ -106,7 +106,7 @@ func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) doma
o.log.Info("metadata saved", "slug", meta.Slug, "title", meta.Title)
// ── Step 2: Chapter list ──────────────────────────────────────────────────
refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL)
refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL, task.ToChapter)
if err != nil {
o.log.Error("chapter list scrape failed", "slug", meta.Slug, "err", err)
result.ErrorMessage = fmt.Sprintf("chapter list: %v", err)

2
v3/backend/internal/orchestrator/orchestrator_test.go
View File

@@ -34,7 +34,7 @@ func (s *stubScraper) ScrapeMetadata(_ context.Context, _ string) (domain.BookMe
return s.meta, s.metaErr
}
func (s *stubScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) {
func (s *stubScraper) ScrapeChapterList(_ context.Context, _ string, _ int) ([]domain.ChapterRef, error) {
return s.refs, s.refsErr
}

23
v3/backend/internal/runner/runner.go
View File

@@ -17,6 +17,7 @@ import (
"context"
"fmt"
"log/slog"
"os"
"sync"
"sync/atomic"
"time"
@@ -55,6 +56,11 @@ type Config struct {
// scrapes per-book metadata, downloads covers, and re-indexes everything in
// Meilisearch. Defaults to 24h (expensive — full catalogue walk).
CatalogueRefreshInterval time.Duration
// SkipInitialCatalogueRefresh suppresses the immediate catalogue walk that
// otherwise fires at startup. The periodic ticker (CatalogueRefreshInterval)
// still fires normally. Set RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true for
// quick restarts where the catalogue is already up to date.
SkipInitialCatalogueRefresh bool
// MetricsAddr is the HTTP listen address for the /metrics endpoint.
// Defaults to ":9091". Set to "" to disable.
MetricsAddr string
@@ -175,8 +181,12 @@ func (r *Runner) Run(ctx context.Context) error {
// Run one browse refresh immediately on startup.
go r.runBrowseRefresh(ctx)
// Run one catalogue refresh immediately on startup.
go r.runCatalogueRefresh(ctx)
// Run one catalogue refresh immediately on startup (unless skipped by flag).
if !r.cfg.SkipInitialCatalogueRefresh {
go r.runCatalogueRefresh(ctx)
} else {
r.deps.Log.Info("runner: skipping initial catalogue refresh (RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true)")
}
// Run one poll immediately on startup, then on each tick.
for {
@@ -208,6 +218,15 @@ func (r *Runner) Run(ctx context.Context) error {
// poll claims all available pending tasks and dispatches them to goroutines.
func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg *sync.WaitGroup) {
// ── Heartbeat file ────────────────────────────────────────────────────
// Touch /tmp/runner.alive so the Docker health check can confirm the
// runner is actively polling. Failure is non-fatal — just log it.
if f, err := os.Create("/tmp/runner.alive"); err != nil {
r.deps.Log.Warn("runner: could not write heartbeat file", "err", err)
} else {
f.Close()
}
// ── Reap orphaned tasks ───────────────────────────────────────────────
if n, err := r.deps.Consumer.ReapStaleTasks(ctx, r.cfg.StaleTaskThreshold); err != nil {
r.deps.Log.Warn("runner: reap stale tasks failed", "err", err)

2
v3/backend/internal/runner/runner_test.go
View File

@@ -146,7 +146,7 @@ func (s *stubNovelScraper) ScrapeMetadata(_ context.Context, _ string) (domain.B
return domain.BookMeta{Slug: "test-book", Title: "Test Book", SourceURL: "https://example.com/book/test-book"}, nil
}
func (s *stubNovelScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) {
func (s *stubNovelScraper) ScrapeChapterList(_ context.Context, _ string, _ int) ([]domain.ChapterRef, error) {
return s.chapters, nil
}

4
v3/backend/internal/scraper/scraper.go
View File

@@ -20,8 +20,10 @@ type MetadataProvider interface {
}
// ChapterListProvider can enumerate all chapters of a book.
// upTo > 0 stops pagination once at least upTo chapter numbers have been
// collected (early-exit optimisation for range scrapes). upTo == 0 fetches all pages.
type ChapterListProvider interface {
ScrapeChapterList(ctx context.Context, bookURL string) ([]domain.ChapterRef, error)
ScrapeChapterList(ctx context.Context, bookURL string, upTo int) ([]domain.ChapterRef, error)
}
// ChapterTextProvider can extract the readable text from a single chapter page.

16
v3/backend/internal/storage/store.go
View File

@@ -145,6 +145,10 @@ type pbBook struct {
}
func (b pbBook) toDomain() domain.BookMeta {
var metaUpdated int64
if t, err := time.Parse(time.RFC3339, b.Updated); err == nil {
metaUpdated = t.Unix()
}
return domain.BookMeta{
Slug: b.Slug,
Title: b.Title,
@@ -157,6 +161,7 @@ func (b pbBook) toDomain() domain.BookMeta {
SourceURL: b.SourceURL,
Ranking: b.Ranking,
Rating: b.Rating,
MetaUpdated: metaUpdated,
}
}
@@ -405,6 +410,17 @@ func (s *Store) PresignAvatarURL(ctx context.Context, userID string) (string, bo
return "", false, nil
}
func (s *Store) PutAvatar(ctx context.Context, userID, ext, contentType string, data []byte) (string, error) {
// Delete existing avatar objects for this user before writing the new one
// so old extensions don't linger (e.g. old .png after uploading a .jpg).
_ = s.mc.deleteObjects(ctx, s.mc.bucketAvatars, userID+"/")
key := AvatarObjectKey(userID, ext)
if err := s.mc.putObject(ctx, s.mc.bucketAvatars, key, contentType, data); err != nil {
return "", fmt.Errorf("put avatar: %w", err)
}
return key, nil
}
func (s *Store) DeleteAvatar(ctx context.Context, userID string) error {
return s.mc.deleteObjects(ctx, s.mc.bucketAvatars, userID+"/")
}