diff --git a/.gitea/workflows/ci-v3.yaml b/.gitea/workflows/ci-v3.yaml new file mode 100644 index 0000000..9ea014f --- /dev/null +++ b/.gitea/workflows/ci-v3.yaml @@ -0,0 +1,146 @@ +name: CI / v3 + +on: + push: + branches: ["main", "master"] + paths: + - "v3/**" + - ".gitea/workflows/ci-v3.yaml" + pull_request: + branches: ["main", "master"] + paths: + - "v3/**" + - ".gitea/workflows/ci-v3.yaml" + +concurrency: + group: ${{ gitea.workflow }}-${{ gitea.ref }} + cancel-in-progress: true + +jobs: + # ── backend: lint & test ───────────────────────────────────────────────────── + test-backend: + name: Test backend + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: v3/backend/go.mod + cache-dependency-path: v3/backend/go.sum + + - name: go vet + working-directory: v3/backend + run: go vet ./... + + - name: Run tests + working-directory: v3/backend + run: go test -short -race -count=1 -timeout=60s ./... + + # ── ui: type-check ─────────────────────────────────────────────────────────── + check-ui: + name: Check ui + runs-on: ubuntu-latest + defaults: + run: + working-directory: v3/ui + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: v3/ui/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Type check + run: npm run check + + - name: Build + run: npm run build + + # ── docker: backend (push to Docker Hub on branch push only) ───────────────── + docker-backend: + name: Docker / backend + runs-on: ubuntu-latest + needs: [test-backend] + if: gitea.event_name == 'push' + steps: + - uses: actions/checkout@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: v3/backend + target: backend + push: true + tags: | + ${{ secrets.DOCKER_USER }}/libnovel-v3-backend:latest + ${{ secrets.DOCKER_USER }}/libnovel-v3-backend:${{ gitea.sha }} + build-args: | + VERSION=${{ gitea.sha }} + COMMIT=${{ gitea.sha }} + + # ── docker: runner ──────────────────────────────────────────────────────────── + docker-runner: + name: Docker / runner + runs-on: ubuntu-latest + needs: [test-backend] + if: gitea.event_name == 'push' + steps: + - uses: actions/checkout@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: v3/backend + target: runner + push: true + tags: | + ${{ secrets.DOCKER_USER }}/libnovel-v3-runner:latest + ${{ secrets.DOCKER_USER }}/libnovel-v3-runner:${{ gitea.sha }} + build-args: | + VERSION=${{ gitea.sha }} + COMMIT=${{ gitea.sha }} + + # ── docker: ui ──────────────────────────────────────────────────────────────── + docker-ui: + name: Docker / ui + runs-on: ubuntu-latest + needs: [check-ui] + if: gitea.event_name == 'push' + steps: + - uses: actions/checkout@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: v3/ui + push: true + tags: | + ${{ secrets.DOCKER_USER }}/libnovel-v3-ui:latest + ${{ secrets.DOCKER_USER }}/libnovel-v3-ui:${{ gitea.sha }} + build-args: | + BUILD_VERSION=${{ gitea.sha }} + BUILD_COMMIT=${{ gitea.sha }} diff --git a/.gitea/workflows/release-v3.yaml b/.gitea/workflows/release-v3.yaml new file mode 100644 index 0000000..5e733b0 --- /dev/null +++ b/.gitea/workflows/release-v3.yaml @@ -0,0 +1,177 @@ +name: Release / v3 + +on: + push: + tags: + - "v3/*" # e.g. v3/1.0.0, v3/1.2.3 + - "v3-*" # e.g. v3-1.0.0 (alternative convention) + +concurrency: + group: ${{ gitea.workflow }}-${{ gitea.ref }} + cancel-in-progress: true + +jobs: + # ── backend: lint & test ───────────────────────────────────────────────────── + test-backend: + name: Test backend + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: v3/backend/go.mod + cache-dependency-path: v3/backend/go.sum + + - name: go vet + working-directory: v3/backend + run: go vet ./... + + - name: Run tests + working-directory: v3/backend + run: go test -short -race -count=1 -timeout=60s ./... + + # ── ui: type-check & build ─────────────────────────────────────────────────── + check-ui: + name: Check ui + runs-on: ubuntu-latest + defaults: + run: + working-directory: v3/ui + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: v3/ui/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Type check + run: npm run check + + - name: Build + run: npm run build + + # ── docker: backend ─────────────────────────────────────────────────────────── + docker-backend: + name: Docker / backend + runs-on: ubuntu-latest + needs: [test-backend] + steps: + - uses: actions/checkout@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ secrets.DOCKER_USER }}/libnovel-v3-backend + tags: | + type=match,pattern=v3/(.*),group=1 + type=raw,value=latest + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: v3/backend + target: backend + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ steps.meta.outputs.version }} + COMMIT=${{ gitea.sha }} + + # ── docker: runner ──────────────────────────────────────────────────────────── + docker-runner: + name: Docker / runner + runs-on: ubuntu-latest + needs: [test-backend] + steps: + - uses: actions/checkout@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ secrets.DOCKER_USER }}/libnovel-v3-runner + tags: | + type=match,pattern=v3/(.*),group=1 + type=raw,value=latest + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: v3/backend + target: runner + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + VERSION=${{ steps.meta.outputs.version }} + COMMIT=${{ gitea.sha }} + + # ── docker: ui ──────────────────────────────────────────────────────────────── + docker-ui: + name: Docker / ui + runs-on: ubuntu-latest + needs: [check-ui] + steps: + - uses: actions/checkout@v4 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ secrets.DOCKER_USER }}/libnovel-v3-ui + tags: | + type=match,pattern=v3/(.*),group=1 + type=raw,value=latest + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: v3/ui + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + BUILD_VERSION=${{ steps.meta.outputs.version }} + BUILD_COMMIT=${{ gitea.sha }} + + # ── Gitea release ───────────────────────────────────────────────────────────── + release: + name: Gitea Release + runs-on: ubuntu-latest + needs: [docker-backend, docker-runner, docker-ui] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Create release + uses: actions/gitea-release-action@v1 + with: + token: ${{ secrets.GITEA_TOKEN }} + generate_release_notes: true diff --git a/backend/bin/runner b/backend/bin/runner new file mode 100755 index 0000000..6de49f4 Binary files /dev/null and b/backend/bin/runner differ diff --git a/v3/Caddyfile b/v3/Caddyfile index 2a72753..b538844 100644 --- a/v3/Caddyfile +++ b/v3/Caddyfile @@ -1,6 +1,7 @@ # v3/Caddyfile # # Caddy reverse proxy for LibNovel v3. +# Custom build includes github.com/mholt/caddy-ratelimit. # # Environment variables consumed (set in docker-compose.yml): # DOMAIN — public hostname, e.g. libnovel.example.com @@ -9,51 +10,106 @@ # # Routing rules: # /health → backend:8080 (liveness probe) -# /scrape* → backend:8080 (scrape task creation) +# /scrape* → backend:8080 (Go admin scrape endpoints) # /api/browse → backend:8080 (MinIO-cached browse pages) # /api/book-preview/* → backend:8080 (live scrape, no store write) -# /api/chapter-text-preview/*/* → backend:8080 (live chapter, no store write) -# /api/chapter-text/*/* → backend:8080 (chapter markdown from MinIO) +# /api/chapter-text/* → backend:8080 (chapter markdown from MinIO) # /api/reindex/* → backend:8080 (rebuild chapter index) # /api/cover/* → backend:8080 (proxy cover image) -# /api/audio-proxy/*/* → backend:8080 (proxy generated audio) -# /api/scrape/* → backend:8080 (scrape job status/tasks) +# /api/audio-proxy/* → backend:8080 (proxy generated audio) +# /avatars/* → minio:9000 (presigned avatar GETs) # /* (everything else) → ui:3000 (SvelteKit — handles all -# remaining /api/* routes too) +# remaining /api/* routes) # -# The SvelteKit UI itself proxies to the backend for: ranking, voices, search, -# browse-page, presign, audio, progress, and the Go /api/progress endpoint. -# MinIO and PocketBase are NOT exposed publicly. +# Routes intentionally removed from direct-to-backend: +# /api/scrape/* — SvelteKit has /api/scrape/ counterparts +# that enforce auth; routing directly would +# bypass SK middleware. +# /api/chapter-text-preview/* — Same: SvelteKit owns +# /api/chapter-text-preview/[slug]/[n]. { # Email for Let's Encrypt ACME account registration. - # Optional — omit to use an anonymous ACME account. + # When CADDY_ACME_EMAIL is set this expands to e.g. "email you@example.com". + # When unset it expands to an empty string and is silently ignored. {$CADDY_ACME_EMAIL:} } +(security_headers) { + header { + # Prevent clickjacking + X-Frame-Options "SAMEORIGIN" + # Prevent MIME-type sniffing + X-Content-Type-Options "nosniff" + # Minimal referrer info for cross-origin requests + Referrer-Policy "strict-origin-when-cross-origin" + # Restrict powerful browser features + Permissions-Policy "camera=(), microphone=(), geolocation=(), payment=()" + # Enforce HTTPS for 1 year (includeSubDomains) + Strict-Transport-Security "max-age=31536000; includeSubDomains" + # Enable XSS filter in older browsers + X-XSS-Protection "1; mode=block" + # Remove server identity header + -Server + } +} + {$DOMAIN:localhost} { - # ── Liveness probe ─────────────────────────────────────────────────────── + import security_headers + + # ── Rate limiting ───────────────────────────────────────────────────────── + # Auth endpoints: strict — 10 req/min per IP + rate_limit { + zone auth_zone { + match { + path /api/auth/login /api/auth/register /api/auth/change-password + } + key {remote_host} + window 1m + events 10 + } + } + + # Admin scrape endpoints: moderate — 20 req/min per IP + rate_limit { + zone scrape_zone { + match { + path /scrape* + } + key {remote_host} + window 1m + events 20 + } + } + + # Global: 300 req/min per IP (covers everything) + rate_limit { + zone global_zone { + key {remote_host} + window 1m + events 300 + } + } + + # ── Liveness probe ──────────────────────────────────────────────────────── handle /health { reverse_proxy backend:8080 } - # ── Scrape task creation (Go backend only) ─────────────────────────────── + # ── Scrape task creation (Go backend only) ──────────────────────────────── handle /scrape* { reverse_proxy backend:8080 } # ── Backend-only API paths ──────────────────────────────────────────────── - # These paths are served exclusively by the Go scraper and are not - # implemented in the SvelteKit UI. + # These paths are served exclusively by the Go backend and have no + # SvelteKit counterpart. Routing them here skips SK intentionally. handle /api/browse { reverse_proxy backend:8080 } handle /api/book-preview/* { reverse_proxy backend:8080 } - handle /api/chapter-text-preview/* { - reverse_proxy backend:8080 - } handle /api/chapter-text/* { reverse_proxy backend:8080 } @@ -66,8 +122,10 @@ handle /api/audio-proxy/* { reverse_proxy backend:8080 } - handle /api/scrape/* { - reverse_proxy backend:8080 + + # ── MinIO avatars bucket (presigned GET only) ───────────────────────────── + handle /avatars/* { + reverse_proxy minio:9000 } # ── SvelteKit UI (catch-all — includes all remaining /api/* routes) ─────── @@ -75,7 +133,26 @@ reverse_proxy ui:3000 } - # ── Logging ────────────────────────────────────────────────────────────── + # ── Caddy-level error pages ─────────────────────────────────────────────── + # These fire when the upstream (backend or ui) is completely unreachable. + # SvelteKit's own +error.svelte handles application-level errors (404, 500). + handle_errors 502 { + root * /srv/errors + rewrite * /502.html + file_server + } + handle_errors 503 { + root * /srv/errors + rewrite * /503.html + file_server + } + handle_errors 504 { + root * /srv/errors + rewrite * /504.html + file_server + } + + # ── Logging ─────────────────────────────────────────────────────────────── log { output stdout format json diff --git a/v3/backend/bin/runner b/v3/backend/bin/runner new file mode 100755 index 0000000..6de49f4 Binary files /dev/null and b/v3/backend/bin/runner differ diff --git a/v3/backend/cmd/runner/main.go b/v3/backend/cmd/runner/main.go index 01a230c..f534d68 100644 --- a/v3/backend/cmd/runner/main.go +++ b/v3/backend/cmd/runner/main.go @@ -111,7 +111,8 @@ func run() error { MaxConcurrentAudio: cfg.Runner.MaxConcurrentAudio, OrchestratorWorkers: workers, MetricsAddr: cfg.Runner.MetricsAddr, - CatalogueRefreshInterval: cfg.Runner.CatalogueRefreshInterval, + CatalogueRefreshInterval: cfg.Runner.CatalogueRefreshInterval, + SkipInitialCatalogueRefresh: cfg.Runner.SkipInitialCatalogueRefresh, } deps := runner.Dependencies{ Consumer: store, diff --git a/v3/backend/healthcheck b/v3/backend/healthcheck new file mode 100755 index 0000000..9d0e8ec Binary files /dev/null and b/v3/backend/healthcheck differ diff --git a/v3/backend/internal/backend/handlers.go b/v3/backend/internal/backend/handlers.go index 8d5a835..eda8eec 100644 --- a/v3/backend/internal/backend/handlers.go +++ b/v3/backend/internal/backend/handlers.go @@ -7,8 +7,7 @@ package backend // handleScrapeStatus, handleScrapeTasks // handleBrowse, handleSearch // handleGetRanking, handleGetCover -// handleBookPreview, handleChapterText, handleReindex -// handleChapterText, handleReindex +// handleBookPreview, handleChapterText, handleChapterTextPreview, handleChapterMarkdown, handleReindex // handleAudioGenerate, handleAudioStatus, handleAudioProxy // handleVoices // handlePresignChapter, handlePresignAudio, handlePresignVoiceSample @@ -29,6 +28,8 @@ package backend // by the runner after each catalogue scrape). // - GET /api/book-preview returns stored data when in library, or enqueues a // scrape task and returns 202 when not. The backend never scrapes directly. +// - GET /api/chapter-text-preview scrapes a chapter live from novelfire.net +// directly (no runner task, no store writes). Used for unscraped books. import ( "context" @@ -45,6 +46,8 @@ import ( "github.com/libnovel/backend/internal/domain" "github.com/libnovel/backend/internal/kokoro" "github.com/libnovel/backend/internal/meili" + "github.com/libnovel/backend/internal/novelfire/htmlutil" + "github.com/libnovel/backend/internal/scraper" ) const ( @@ -502,6 +505,117 @@ func (s *Server) handleChapterMarkdown(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, raw) } +// handleChapterTextPreview handles GET /api/chapter-text-preview/{slug}/{n}. +// +// Fetches a chapter live from novelfire.net and returns its plain text without +// writing anything to PocketBase or MinIO. This is the preview path used when +// a chapter has not yet been scraped into the library. +// +// Optional query params: +// +// chapter_url — the canonical chapter URL (preferred over constructing one) +// title — hint for the chapter title (used when the page title is empty) +// +// Response: {"slug":string,"number":int,"title":string,"text":string,"url":string} +func (s *Server) handleChapterTextPreview(w http.ResponseWriter, r *http.Request) { + slug := r.PathValue("slug") + n, err := strconv.Atoi(r.PathValue("n")) + if err != nil || n < 1 || slug == "" { + jsonError(w, http.StatusBadRequest, "invalid slug or chapter number") + return + } + + // Determine the chapter URL to fetch. + chapterURL := r.URL.Query().Get("chapter_url") + if chapterURL == "" { + // Best-effort: novelfire chapter URLs follow /book/{slug}/chapter-{n} + chapterURL = fmt.Sprintf("%s/book/%s/chapter-%d", novelFireBase, slug, n) + } + + titleHint := r.URL.Query().Get("title") + + ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) + defer cancel() + + // Fetch the chapter page. + req, err := http.NewRequestWithContext(ctx, http.MethodGet, chapterURL, nil) + if err != nil { + s.deps.Log.Error("chapter-text-preview: build request failed", "url", chapterURL, "err", err) + jsonError(w, http.StatusInternalServerError, "failed to build request") + return + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-backend/2)") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + s.deps.Log.Warn("chapter-text-preview: fetch failed", "url", chapterURL, "err", err) + jsonError(w, http.StatusBadGateway, "failed to fetch chapter") + return + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + jsonError(w, http.StatusNotFound, "chapter not found") + return + } + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + s.deps.Log.Warn("chapter-text-preview: upstream error", + "url", chapterURL, "status", resp.StatusCode, "body_snippet", string(body)) + jsonError(w, http.StatusBadGateway, fmt.Sprintf("upstream returned %d", resp.StatusCode)) + return + } + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + s.deps.Log.Error("chapter-text-preview: read body failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to read response") + return + } + + // Parse HTML and extract the #content node. + root, err := htmlutil.ParseHTML(string(bodyBytes)) + if err != nil { + s.deps.Log.Error("chapter-text-preview: html parse failed", "err", err) + jsonError(w, http.StatusInternalServerError, "failed to parse chapter HTML") + return + } + + container := htmlutil.FindFirst(root, scraper.Selector{ID: "content"}) + if container == nil { + s.deps.Log.Warn("chapter-text-preview: #content not found", "url", chapterURL) + jsonError(w, http.StatusNotFound, "chapter content not found on page") + return + } + + markdownText := htmlutil.NodeToMarkdown(container) + plainText := stripMarkdown(markdownText) + + // Extract the chapter title from the page or <h1> if not hinted. + chapterTitle := titleHint + if chapterTitle == "" { + // Try <h1 class="chapter-title"> first, then <h2 class="chapter-title"> + for _, tag := range []string{"h1", "h2", "h3"} { + if node := htmlutil.FindFirst(root, scraper.Selector{Tag: tag, Class: "chapter-title"}); node != nil { + chapterTitle = strings.TrimSpace(htmlutil.TextContent(node)) + break + } + } + } + if chapterTitle == "" { + chapterTitle = fmt.Sprintf("Chapter %d", n) + } + + writeJSON(w, 0, map[string]any{ + "slug": slug, + "number": n, + "title": chapterTitle, + "text": plainText, + "url": chapterURL, + }) +} + // handleReindex handles POST /api/reindex/{slug}. // Rebuilds the chapters_idx PocketBase collection for a book from MinIO objects. func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) { @@ -741,6 +855,59 @@ func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request writeJSON(w, 0, map[string]string{"url": u}) } +// handleAvatarUpload handles PUT /api/avatar-upload/{userId}. +// The request body must be the raw image bytes; Content-Type must be +// image/jpeg, image/png, or image/webp. +// +// This endpoint is called by the SvelteKit server (not the browser directly), +// so MinIO credentials and internal networking are not a concern. +// +// Returns: { "key": "<objectKey>" } +func (s *Server) handleAvatarUpload(w http.ResponseWriter, r *http.Request) { + userID := r.PathValue("userId") + if userID == "" { + jsonError(w, http.StatusBadRequest, "missing userId") + return + } + + ct := r.Header.Get("Content-Type") + var ext string + switch { + case strings.HasPrefix(ct, "image/jpeg"): + ext = "jpg" + case strings.HasPrefix(ct, "image/png"): + ext = "png" + case strings.HasPrefix(ct, "image/webp"): + ext = "webp" + default: + jsonError(w, http.StatusBadRequest, "unsupported content-type; use image/jpeg, image/png, or image/webp") + return + } + + const maxSize = 5 << 20 // 5 MiB + data, err := io.ReadAll(io.LimitReader(r.Body, maxSize+1)) + if err != nil { + jsonError(w, http.StatusBadRequest, "failed to read body") + return + } + if len(data) > maxSize { + jsonError(w, http.StatusRequestEntityTooLarge, "image too large (max 5 MiB)") + return + } + if len(data) == 0 { + jsonError(w, http.StatusBadRequest, "empty body") + return + } + + key, err := s.deps.PresignStore.PutAvatar(r.Context(), userID, ext, ct, data) + if err != nil { + s.deps.Log.Error("avatar upload failed", "userId", userID, "err", err) + jsonError(w, http.StatusInternalServerError, "upload failed") + return + } + writeJSON(w, 0, map[string]string{"key": key}) +} + // handlePresignAvatarUpload handles GET /api/presign/avatar-upload/{userId}. // Query params: ext (jpg|png|webp, defaults to jpg) func (s *Server) handlePresignAvatarUpload(w http.ResponseWriter, r *http.Request) { @@ -912,7 +1079,7 @@ func (s *Server) handleCatalogue(w http.ResponseWriter, r *http.Request) { Limit: limit, } - books, total, err := s.deps.SearchIndex.Catalogue(r.Context(), cq) + books, total, facets, err := s.deps.SearchIndex.Catalogue(r.Context(), cq) if err != nil { s.deps.Log.Error("handleCatalogue: Catalogue query failed", "err", err) jsonError(w, http.StatusInternalServerError, "search failed") @@ -928,6 +1095,10 @@ func (s *Server) handleCatalogue(w http.ResponseWriter, r *http.Request) { "limit": limit, "total": total, "has_next": hasNext, + "facets": map[string]any{ + "genres": facets.Genres, + "statuses": facets.Statuses, + }, }) } diff --git a/v3/backend/internal/backend/server.go b/v3/backend/internal/backend/server.go index 93486ff..148213a 100644 --- a/v3/backend/internal/backend/server.go +++ b/v3/backend/internal/backend/server.go @@ -144,6 +144,10 @@ func (s *Server) ListenAndServe(ctx context.Context) error { // Use this instead of presign+fetch to avoid SvelteKit→MinIO network path. mux.HandleFunc("GET /api/chapter-markdown/{slug}/{n}", s.handleChapterMarkdown) + // Chapter text preview — live scrape from novelfire.net, no store writes. + // Used when the chapter is not yet in the library (preview mode). + mux.HandleFunc("GET /api/chapter-text-preview/{slug}/{n}", s.handleChapterTextPreview) + // Reindex chapters_idx from MinIO mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex) @@ -161,6 +165,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error { mux.HandleFunc("GET /api/presign/voice-sample/{voice}", s.handlePresignVoiceSample) mux.HandleFunc("GET /api/presign/avatar-upload/{userId}", s.handlePresignAvatarUpload) mux.HandleFunc("GET /api/presign/avatar/{userId}", s.handlePresignAvatar) + mux.HandleFunc("PUT /api/avatar-upload/{userId}", s.handleAvatarUpload) // Reading progress mux.HandleFunc("GET /api/progress", s.handleGetProgress) diff --git a/v3/backend/internal/bookstore/bookstore.go b/v3/backend/internal/bookstore/bookstore.go index 509f01d..65ab4f9 100644 --- a/v3/backend/internal/bookstore/bookstore.go +++ b/v3/backend/internal/bookstore/bookstore.go @@ -105,6 +105,10 @@ type PresignStore interface { // Returns ("", false, nil) when no avatar exists. PresignAvatarURL(ctx context.Context, userID string) (string, bool, error) + // PutAvatar stores raw image bytes for a user avatar directly in MinIO. + // ext should be "jpg", "png", or "webp". Returns the object key. + PutAvatar(ctx context.Context, userID, ext, contentType string, data []byte) (key string, err error) + // DeleteAvatar removes all avatar objects for a user. DeleteAvatar(ctx context.Context, userID string) error } diff --git a/v3/backend/internal/bookstore/bookstore_test.go b/v3/backend/internal/bookstore/bookstore_test.go index 2bbc5d0..aec804c 100644 --- a/v3/backend/internal/bookstore/bookstore_test.go +++ b/v3/backend/internal/bookstore/bookstore_test.go @@ -68,6 +68,9 @@ func (m *mockStore) PresignAvatarUpload(_ context.Context, _, _ string) (string, func (m *mockStore) PresignAvatarURL(_ context.Context, _ string) (string, bool, error) { return "", false, nil } +func (m *mockStore) PutAvatar(_ context.Context, _, _, _ string, _ []byte) (string, error) { + return "", nil +} func (m *mockStore) DeleteAvatar(_ context.Context, _ string) error { return nil } // ProgressStore diff --git a/v3/backend/internal/config/config.go b/v3/backend/internal/config/config.go index 91036ba..f7fb68b 100644 --- a/v3/backend/internal/config/config.go +++ b/v3/backend/internal/config/config.go @@ -101,6 +101,11 @@ type Runner struct { // scrapes per-book metadata, downloads covers, and re-indexes in Meilisearch. // Defaults to 24h. Set to 0 to use the default. CatalogueRefreshInterval time.Duration + // SkipInitialCatalogueRefresh prevents the runner from running a full + // catalogue walk on startup. Useful for quick restarts where the catalogue + // is already indexed and a 24h walk would be wasteful. + // Controlled by RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true. + SkipInitialCatalogueRefresh bool } // Config is the top-level configuration struct consumed by both binaries. @@ -142,7 +147,7 @@ func Load() Config { PublicUseSSL: envBool("MINIO_PUBLIC_USE_SSL", true), BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"), BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"), - BucketAvatars: envOr("MINIO_BUCKET_AVATARS", "libnovel-avatars"), + BucketAvatars: envOr("MINIO_BUCKET_AVATARS", "avatars"), BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"), }, @@ -156,14 +161,15 @@ func Load() Config { }, Runner: Runner{ - PollInterval: envDuration("RUNNER_POLL_INTERVAL", 30*time.Second), - MaxConcurrentScrape: envInt("RUNNER_MAX_CONCURRENT_SCRAPE", 1), - MaxConcurrentAudio: envInt("RUNNER_MAX_CONCURRENT_AUDIO", 1), - WorkerID: envOr("RUNNER_WORKER_ID", workerID), - Workers: envInt("RUNNER_WORKERS", 0), // 0 → runtime.NumCPU() - Timeout: envDuration("RUNNER_TIMEOUT", 90*time.Second), - MetricsAddr: envOr("RUNNER_METRICS_ADDR", ":9091"), - CatalogueRefreshInterval: envDuration("RUNNER_CATALOGUE_REFRESH_INTERVAL", 0), + PollInterval: envDuration("RUNNER_POLL_INTERVAL", 30*time.Second), + MaxConcurrentScrape: envInt("RUNNER_MAX_CONCURRENT_SCRAPE", 1), + MaxConcurrentAudio: envInt("RUNNER_MAX_CONCURRENT_AUDIO", 1), + WorkerID: envOr("RUNNER_WORKER_ID", workerID), + Workers: envInt("RUNNER_WORKERS", 0), // 0 → runtime.NumCPU() + Timeout: envDuration("RUNNER_TIMEOUT", 90*time.Second), + MetricsAddr: envOr("RUNNER_METRICS_ADDR", ":9091"), + CatalogueRefreshInterval: envDuration("RUNNER_CATALOGUE_REFRESH_INTERVAL", 0), + SkipInitialCatalogueRefresh: envBool("RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH", false), }, Meilisearch: Meilisearch{ diff --git a/v3/backend/internal/domain/domain.go b/v3/backend/internal/domain/domain.go index 80ea3da..9597f34 100644 --- a/v3/backend/internal/domain/domain.go +++ b/v3/backend/internal/domain/domain.go @@ -20,6 +20,10 @@ type BookMeta struct { SourceURL string `json:"source_url"` Ranking int `json:"ranking,omitempty"` Rating float64 `json:"rating,omitempty"` + // MetaUpdated is the Unix timestamp (seconds) when the book record was last + // updated in PocketBase. Populated on read; not sent on write (PocketBase + // manages its own updated field). + MetaUpdated int64 `json:"meta_updated,omitempty"` } // CatalogueEntry is a lightweight book reference returned by catalogue pages. diff --git a/v3/backend/internal/meili/client.go b/v3/backend/internal/meili/client.go index 8525c37..6d9904e 100644 --- a/v3/backend/internal/meili/client.go +++ b/v3/backend/internal/meili/client.go @@ -6,7 +6,7 @@ // - Primary key: "slug" // - Searchable attributes: title, author, genres, summary // - Filterable attributes: status, genres -// - Sortable attributes: rank, rating, total_chapters +// - Sortable attributes: rank, rating, total_chapters, meta_updated // // The client is intentionally simple: UpsertBook and Search only. All // Meilisearch-specific details (index management, attribute configuration) @@ -32,8 +32,9 @@ type Client interface { // Search returns up to limit books matching query. Search(ctx context.Context, query string, limit int) ([]domain.BookMeta, error) // Catalogue queries books with optional filters, sort, and pagination. - // Returns books and the total hit count for pagination. - Catalogue(ctx context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, error) + // Returns books, the total hit count for pagination, and a FacetResult + // with available genre and status values from the index. + Catalogue(ctx context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, FacetResult, error) } // CatalogueQuery holds parameters for the /api/catalogue endpoint. @@ -41,11 +42,18 @@ type CatalogueQuery struct { Q string // full-text query (may be empty for browse) Genre string // genre filter, e.g. "fantasy" or "all" Status string // status filter, e.g. "ongoing", "completed", or "all" - Sort string // sort field: "popular", "new", "top-rated", "rank", "" + Sort string // sort field: "popular", "new", "update", "top-rated", "rank", "" Page int // 1-indexed Limit int // items per page, default 20 } +// FacetResult holds the available filter values discovered from the index. +// Values are sorted alphabetically and include only those present in the index. +type FacetResult struct { + Genres []string // distinct genre values + Statuses []string // distinct status values +} + // MeiliClient wraps the meilisearch-go SDK. type MeiliClient struct { idx meilisearch.IndexManager @@ -93,7 +101,7 @@ func Configure(host, apiKey string) error { return fmt.Errorf("meili: update filterable attributes: %w", err) } - sortable := []string{"rank", "rating", "total_chapters"} + sortable := []string{"rank", "rating", "total_chapters", "meta_updated"} if _, err := idx.UpdateSortableAttributes(&sortable); err != nil { return fmt.Errorf("meili: update sortable attributes: %w", err) } @@ -114,6 +122,9 @@ type bookDoc struct { SourceURL string `json:"source_url"` Rank int `json:"rank"` Rating float64 `json:"rating"` + // MetaUpdated is the Unix timestamp (seconds) of the last PocketBase update. + // Used for sort=update ("recently updated" ordering). + MetaUpdated int64 `json:"meta_updated"` } func toDoc(b domain.BookMeta) bookDoc { @@ -129,6 +140,7 @@ func toDoc(b domain.BookMeta) bookDoc { SourceURL: b.SourceURL, Rank: b.Ranking, Rating: b.Rating, + MetaUpdated: b.MetaUpdated, } } @@ -145,6 +157,7 @@ func fromDoc(d bookDoc) domain.BookMeta { SourceURL: d.SourceURL, Ranking: d.Rank, Rating: d.Rating, + MetaUpdated: d.MetaUpdated, } } @@ -188,8 +201,9 @@ func (c *MeiliClient) Search(_ context.Context, query string, limit int) ([]doma } // Catalogue queries books with optional full-text search, genre/status filters, -// sort order, and pagination. Returns matching books and the total estimate. -func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, error) { +// sort order, and pagination. Returns matching books, the total estimate, and +// a FacetResult containing available genre and status values from the index. +func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, FacetResult, error) { if q.Limit <= 0 { q.Limit = 20 } @@ -200,6 +214,9 @@ func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.B req := &meilisearch.SearchRequest{ Limit: int64(q.Limit), Offset: int64((q.Page - 1) * q.Limit), + // Request facet distribution so the UI can build filter options + // dynamically without hardcoding genre/status lists. + Facets: []string{"genres", "status"}, } // Build filter @@ -214,7 +231,7 @@ func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.B req.Filter = strings.Join(filters, " AND ") } - // Map UI sort tokens to Meilisearch sort expressions + // Map UI sort tokens to Meilisearch sort expressions. switch q.Sort { case "rank": req.Sort = []string{"rank:asc"} @@ -222,12 +239,14 @@ func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.B req.Sort = []string{"rating:desc"} case "new": req.Sort = []string{"total_chapters:desc"} + case "update": + req.Sort = []string{"meta_updated:desc"} // "popular" and "" → relevance (no explicit sort) } res, err := c.idx.Search(q.Q, req) if err != nil { - return nil, 0, fmt.Errorf("meili: catalogue query: %w", err) + return nil, 0, FacetResult{}, fmt.Errorf("meili: catalogue query: %w", err) } books := make([]domain.BookMeta, 0, len(res.Hits)) @@ -242,7 +261,45 @@ func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.B } books = append(books, fromDoc(doc)) } - return books, res.EstimatedTotalHits, nil + + facets := parseFacets(res.FacetDistribution) + return books, res.EstimatedTotalHits, facets, nil +} + +// parseFacets extracts sorted genre and status slices from a Meilisearch +// facetDistribution raw JSON value. +// The JSON shape is: {"genres":{"fantasy":12,"action":5},"status":{"ongoing":7}} +func parseFacets(raw json.RawMessage) FacetResult { + var result FacetResult + if len(raw) == 0 { + return result + } + var dist map[string]map[string]int64 + if err := json.Unmarshal(raw, &dist); err != nil { + return result + } + if m, ok := dist["genres"]; ok { + for k := range m { + result.Genres = append(result.Genres, k) + } + sortStrings(result.Genres) + } + if m, ok := dist["status"]; ok { + for k := range m { + result.Statuses = append(result.Statuses, k) + } + sortStrings(result.Statuses) + } + return result +} + +// sortStrings sorts a slice of strings in place. +func sortStrings(s []string) { + for i := 1; i < len(s); i++ { + for j := i; j > 0 && s[j] < s[j-1]; j-- { + s[j], s[j-1] = s[j-1], s[j] + } + } } // NoopClient is a no-op Client used when Meilisearch is not configured. @@ -252,6 +309,6 @@ func (NoopClient) UpsertBook(_ context.Context, _ domain.BookMeta) error { retur func (NoopClient) Search(_ context.Context, _ string, _ int) ([]domain.BookMeta, error) { return nil, nil } -func (NoopClient) Catalogue(_ context.Context, _ CatalogueQuery) ([]domain.BookMeta, int64, error) { - return nil, 0, nil +func (NoopClient) Catalogue(_ context.Context, _ CatalogueQuery) ([]domain.BookMeta, int64, FacetResult, error) { + return nil, 0, FacetResult{}, nil } diff --git a/v3/backend/internal/novelfire/scraper.go b/v3/backend/internal/novelfire/scraper.go index 7122b9a..dad5b1b 100644 --- a/v3/backend/internal/novelfire/scraper.go +++ b/v3/backend/internal/novelfire/scraper.go @@ -194,8 +194,12 @@ func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.Bo // ── ChapterListProvider ─────────────────────────────────────────────────────── -// ScrapeChapterList returns all chapter references for a book, ordered ascending. -func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]domain.ChapterRef, error) { +// ScrapeChapterList returns chapter references for a book, ordered ascending. +// upTo > 0 stops pagination as soon as at least upTo chapter numbers have been +// collected — use this for range scrapes so we don't paginate 100 pages just +// to discover refs we'll never scrape. upTo == 0 fetches all pages. +// Each page fetch uses retryGet with 429-aware exponential backoff. +func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string, upTo int) ([]domain.ChapterRef, error) { var refs []domain.ChapterRef baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters" page := 1 @@ -210,7 +214,7 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]doma pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page) s.log.Info("scraping chapter list", "page", page, "url", pageURL) - raw, err := s.client.GetContent(ctx, pageURL) + raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 6*time.Second) if err != nil { return refs, fmt.Errorf("chapter list page %d: %w", page, err) } @@ -255,6 +259,13 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]doma }) } + // Early-stop: if we have seen at least upTo chapter numbers, we have + // enough refs to cover the requested range — no need to paginate further. + if upTo > 0 && len(refs) > 0 && refs[len(refs)-1].Number >= upTo { + s.log.Debug("chapter list early-stop reached", "upTo", upTo, "collected", len(refs)) + break + } + page++ } diff --git a/v3/backend/internal/orchestrator/orchestrator.go b/v3/backend/internal/orchestrator/orchestrator.go index e5e3458..dd7214d 100644 --- a/v3/backend/internal/orchestrator/orchestrator.go +++ b/v3/backend/internal/orchestrator/orchestrator.go @@ -106,7 +106,7 @@ func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) doma o.log.Info("metadata saved", "slug", meta.Slug, "title", meta.Title) // ── Step 2: Chapter list ────────────────────────────────────────────────── - refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL) + refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL, task.ToChapter) if err != nil { o.log.Error("chapter list scrape failed", "slug", meta.Slug, "err", err) result.ErrorMessage = fmt.Sprintf("chapter list: %v", err) diff --git a/v3/backend/internal/orchestrator/orchestrator_test.go b/v3/backend/internal/orchestrator/orchestrator_test.go index b1edaf9..90aa9d1 100644 --- a/v3/backend/internal/orchestrator/orchestrator_test.go +++ b/v3/backend/internal/orchestrator/orchestrator_test.go @@ -34,7 +34,7 @@ func (s *stubScraper) ScrapeMetadata(_ context.Context, _ string) (domain.BookMe return s.meta, s.metaErr } -func (s *stubScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) { +func (s *stubScraper) ScrapeChapterList(_ context.Context, _ string, _ int) ([]domain.ChapterRef, error) { return s.refs, s.refsErr } diff --git a/v3/backend/internal/runner/runner.go b/v3/backend/internal/runner/runner.go index f3490e6..3c7f3a6 100644 --- a/v3/backend/internal/runner/runner.go +++ b/v3/backend/internal/runner/runner.go @@ -17,6 +17,7 @@ import ( "context" "fmt" "log/slog" + "os" "sync" "sync/atomic" "time" @@ -55,6 +56,11 @@ type Config struct { // scrapes per-book metadata, downloads covers, and re-indexes everything in // Meilisearch. Defaults to 24h (expensive — full catalogue walk). CatalogueRefreshInterval time.Duration + // SkipInitialCatalogueRefresh suppresses the immediate catalogue walk that + // otherwise fires at startup. The periodic ticker (CatalogueRefreshInterval) + // still fires normally. Set RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true for + // quick restarts where the catalogue is already up to date. + SkipInitialCatalogueRefresh bool // MetricsAddr is the HTTP listen address for the /metrics endpoint. // Defaults to ":9091". Set to "" to disable. MetricsAddr string @@ -175,8 +181,12 @@ func (r *Runner) Run(ctx context.Context) error { // Run one browse refresh immediately on startup. go r.runBrowseRefresh(ctx) - // Run one catalogue refresh immediately on startup. - go r.runCatalogueRefresh(ctx) + // Run one catalogue refresh immediately on startup (unless skipped by flag). + if !r.cfg.SkipInitialCatalogueRefresh { + go r.runCatalogueRefresh(ctx) + } else { + r.deps.Log.Info("runner: skipping initial catalogue refresh (RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true)") + } // Run one poll immediately on startup, then on each tick. for { @@ -208,6 +218,15 @@ func (r *Runner) Run(ctx context.Context) error { // poll claims all available pending tasks and dispatches them to goroutines. func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg *sync.WaitGroup) { + // ── Heartbeat file ──────────────────────────────────────────────────── + // Touch /tmp/runner.alive so the Docker health check can confirm the + // runner is actively polling. Failure is non-fatal — just log it. + if f, err := os.Create("/tmp/runner.alive"); err != nil { + r.deps.Log.Warn("runner: could not write heartbeat file", "err", err) + } else { + f.Close() + } + // ── Reap orphaned tasks ─────────────────────────────────────────────── if n, err := r.deps.Consumer.ReapStaleTasks(ctx, r.cfg.StaleTaskThreshold); err != nil { r.deps.Log.Warn("runner: reap stale tasks failed", "err", err) diff --git a/v3/backend/internal/runner/runner_test.go b/v3/backend/internal/runner/runner_test.go index 2fa8888..9770089 100644 --- a/v3/backend/internal/runner/runner_test.go +++ b/v3/backend/internal/runner/runner_test.go @@ -146,7 +146,7 @@ func (s *stubNovelScraper) ScrapeMetadata(_ context.Context, _ string) (domain.B return domain.BookMeta{Slug: "test-book", Title: "Test Book", SourceURL: "https://example.com/book/test-book"}, nil } -func (s *stubNovelScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) { +func (s *stubNovelScraper) ScrapeChapterList(_ context.Context, _ string, _ int) ([]domain.ChapterRef, error) { return s.chapters, nil } diff --git a/v3/backend/internal/scraper/scraper.go b/v3/backend/internal/scraper/scraper.go index bba8f13..a8081f7 100644 --- a/v3/backend/internal/scraper/scraper.go +++ b/v3/backend/internal/scraper/scraper.go @@ -20,8 +20,10 @@ type MetadataProvider interface { } // ChapterListProvider can enumerate all chapters of a book. +// upTo > 0 stops pagination once at least upTo chapter numbers have been +// collected (early-exit optimisation for range scrapes). upTo == 0 fetches all pages. type ChapterListProvider interface { - ScrapeChapterList(ctx context.Context, bookURL string) ([]domain.ChapterRef, error) + ScrapeChapterList(ctx context.Context, bookURL string, upTo int) ([]domain.ChapterRef, error) } // ChapterTextProvider can extract the readable text from a single chapter page. diff --git a/v3/backend/internal/storage/store.go b/v3/backend/internal/storage/store.go index 3e10125..a0159b2 100644 --- a/v3/backend/internal/storage/store.go +++ b/v3/backend/internal/storage/store.go @@ -145,6 +145,10 @@ type pbBook struct { } func (b pbBook) toDomain() domain.BookMeta { + var metaUpdated int64 + if t, err := time.Parse(time.RFC3339, b.Updated); err == nil { + metaUpdated = t.Unix() + } return domain.BookMeta{ Slug: b.Slug, Title: b.Title, @@ -157,6 +161,7 @@ func (b pbBook) toDomain() domain.BookMeta { SourceURL: b.SourceURL, Ranking: b.Ranking, Rating: b.Rating, + MetaUpdated: metaUpdated, } } @@ -405,6 +410,17 @@ func (s *Store) PresignAvatarURL(ctx context.Context, userID string) (string, bo return "", false, nil } +func (s *Store) PutAvatar(ctx context.Context, userID, ext, contentType string, data []byte) (string, error) { + // Delete existing avatar objects for this user before writing the new one + // so old extensions don't linger (e.g. old .png after uploading a .jpg). + _ = s.mc.deleteObjects(ctx, s.mc.bucketAvatars, userID+"/") + key := AvatarObjectKey(userID, ext) + if err := s.mc.putObject(ctx, s.mc.bucketAvatars, key, contentType, data); err != nil { + return "", fmt.Errorf("put avatar: %w", err) + } + return key, nil +} + func (s *Store) DeleteAvatar(ctx context.Context, userID string) error { return s.mc.deleteObjects(ctx, s.mc.bucketAvatars, userID+"/") } diff --git a/v3/caddy/Dockerfile b/v3/caddy/Dockerfile new file mode 100644 index 0000000..9f27a5b --- /dev/null +++ b/v3/caddy/Dockerfile @@ -0,0 +1,7 @@ +FROM caddy:2-builder AS builder + +RUN xcaddy build \ + --with github.com/mholt/caddy-ratelimit + +FROM caddy:2-alpine +COPY --from=builder /usr/bin/caddy /usr/bin/caddy diff --git a/v3/caddy/errors/502.html b/v3/caddy/errors/502.html new file mode 100644 index 0000000..74148be --- /dev/null +++ b/v3/caddy/errors/502.html @@ -0,0 +1,51 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0" /> + <title>502 — Service Unavailable + + + +
502
+

Service Unavailable

+

The server is temporarily unreachable. Please try again in a moment.

+ Go home + + diff --git a/v3/caddy/errors/503.html b/v3/caddy/errors/503.html new file mode 100644 index 0000000..1080cf9 --- /dev/null +++ b/v3/caddy/errors/503.html @@ -0,0 +1,51 @@ + + + + + + 503 — Maintenance + + + +
503
+

Under Maintenance

+

LibNovel is briefly offline for maintenance. We’ll be back shortly.

+ Try again + + diff --git a/v3/caddy/errors/504.html b/v3/caddy/errors/504.html new file mode 100644 index 0000000..53faf9c --- /dev/null +++ b/v3/caddy/errors/504.html @@ -0,0 +1,51 @@ + + + + + + 504 — Gateway Timeout + + + +
504
+

Gateway Timeout

+

The request took too long to complete. Please refresh and try again.

+ Go home + + diff --git a/v3/docker-compose.yml b/v3/docker-compose.yml index de6d60e..c08f14d 100644 --- a/v3/docker-compose.yml +++ b/v3/docker-compose.yml @@ -8,8 +8,8 @@ x-infra-env: &infra-env MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}" MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}" MINIO_USE_SSL: "false" - MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-}" - MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-false}" + MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-localhost}" + MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-true}" # PocketBase POCKETBASE_URL: "http://pocketbase:8090" POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}" @@ -53,7 +53,7 @@ services: mc alias set local http://minio:9000 $${MINIO_ROOT_USER:-admin} $${MINIO_ROOT_PASSWORD:-changeme123}; mc mb --ignore-existing local/libnovel-chapters; mc mb --ignore-existing local/libnovel-audio; - mc mb --ignore-existing local/libnovel-avatars; + mc mb --ignore-existing local/avatars; mc mb --ignore-existing local/libnovel-browse; echo 'buckets ready'; " @@ -135,6 +135,8 @@ services: args: VERSION: "${GIT_TAG:-dev}" COMMIT: "${GIT_COMMIT:-unknown}" + labels: + com.centurylinklabs.watchtower.enable: "true" restart: unless-stopped stop_grace_period: 35s depends_on: @@ -170,6 +172,8 @@ services: args: VERSION: "${GIT_TAG:-dev}" COMMIT: "${GIT_COMMIT:-unknown}" + labels: + com.centurylinklabs.watchtower.enable: "true" restart: unless-stopped stop_grace_period: 135s depends_on: @@ -215,6 +219,8 @@ services: args: BUILD_VERSION: "${GIT_TAG:-dev}" BUILD_COMMIT: "${GIT_COMMIT:-unknown}" + labels: + com.centurylinklabs.watchtower.enable: "true" restart: unless-stopped stop_grace_period: 35s depends_on: @@ -238,7 +244,7 @@ services: POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}" POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}" AUTH_SECRET: "${AUTH_SECRET:-dev_secret_change_in_production}" - PUBLIC_MINIO_PUBLIC_URL: "${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000}" + PUBLIC_MINIO_PUBLIC_URL: "${MINIO_PUBLIC_ENDPOINT:-https://localhost}" # Valkey VALKEY_ADDR: "valkey:6379" healthcheck: @@ -248,8 +254,11 @@ services: retries: 3 # ─── Caddy (reverse proxy + automatic HTTPS) ────────────────────────────────── + # Custom build includes github.com/mholt/caddy-ratelimit. caddy: - image: caddy:2-alpine + build: + context: ./caddy + dockerfile: Dockerfile restart: unless-stopped depends_on: backend: @@ -265,9 +274,24 @@ services: CADDY_ACME_EMAIL: "${CADDY_ACME_EMAIL:-}" volumes: - ./Caddyfile:/etc/caddy/Caddyfile:ro + - ./caddy/errors:/srv/errors:ro - caddy_data:/data - caddy_config:/config + # ─── Watchtower (auto-redeploy custom services on new images) ──────────────── + # Only watches services labelled com.centurylinklabs.watchtower.enable=true. + # Third-party infra images (minio, pocketbase, meilisearch, etc.) are excluded. + watchtower: + image: containrrr/watchtower:latest + restart: unless-stopped + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: --label-enable --interval 300 --cleanup + environment: + WATCHTOWER_NOTIFICATIONS: "${WATCHTOWER_NOTIFICATIONS:-}" + WATCHTOWER_NOTIFICATION_URL: "${WATCHTOWER_NOTIFICATION_URL:-}" + DOCKER_API_VERSION: "1.44" + volumes: minio_data: pb_data: diff --git a/v3/docs/api-endpoints.md b/v3/docs/api-endpoints.md index dbba6b4..01019bb 100644 --- a/v3/docs/api-endpoints.md +++ b/v3/docs/api-endpoints.md @@ -1,8 +1,8 @@ # API Endpoint Reference -All endpoints served by the Go **backend** binary on `:8080`. In production all -traffic is routed through Caddy — `/api/*` and `/health` are proxied to the -backend; everything else goes to the SvelteKit UI. +> **Routing ownership map**: see [`docs/d2/api-routing.svg`](d2/api-routing.svg) (source: [`docs/d2/api-routing.d2`](d2/api-routing.d2)) for a visual overview of which paths Caddy sends to the backend directly vs. through SvelteKit, with auth levels colour-coded. + +All traffic enters through **Caddy :443**. Caddy routes a subset of paths directly to the Go backend (bypassing SvelteKit); everything else goes to SvelteKit, which enforces auth before proxying onward. ## Health / Version diff --git a/v3/docs/architecture.svg b/v3/docs/architecture.svg deleted file mode 100644 index 55082b5..0000000 --- a/v3/docs/architecture.svg +++ /dev/null @@ -1,125 +0,0 @@ -novelfire.netKokoro-FastAPI TTSLet's EncryptBrowser / iOS AppInit containersStorageApplicationminio-init(mc: create buckets)pb-init(bootstrap collections)MinIO :9000 buckets: libnovel-chapters libnovel-audio libnovel-avatars libnovel-browsePocketBase :8090 collections: books chapters_idx audio_cache progress scrape_jobs app_users rankingValkey :6379 (presign URL cacheTTL-based, shared)Meilisearch :7700 indices: booksCaddy :443 / :80(reverse proxyauto-HTTPS via Let's Encrypt)Backend API :8080(Go — HTTP API server)Runner :9091(Go — background workerscraping + TTS jobs/metrics endpoint)SvelteKit UI :3000(adapter-node) create bucketsbootstrap schema blobs (chapters, audio,avatars, browse)structured records(books, progress, jobs…)cache presigned URLs(SET/GET with TTL)write chapter markdown& audio MP3sread/update scrape jobswrite book recordsindex books onscrape completionread presigned URL cache(replaces in-process Map)REST API calls(server-side)/* (proxy)/api/*, /health (proxy)/s3/* (proxy, internal only)scrape(HTTP GET)TTS generation(HTTP POST)ACME certificate(TLS-ALPN-01)HTTPS :443(single entry point) - - - - - - - - - - - - - - - - - - - diff --git a/v3/docs/d2/api-routing.d2 b/v3/docs/d2/api-routing.d2 new file mode 100644 index 0000000..10966d9 --- /dev/null +++ b/v3/docs/d2/api-routing.d2 @@ -0,0 +1,201 @@ +direction: right + +# ─── Legend ─────────────────────────────────────────────────────────────────── + +legend: Legend { + style.fill: "#fafafa" + style.stroke: "#d4d4d8" + + pub: public { + style.fill: "#f0fdf4" + style.font-color: "#15803d" + style.stroke: "#86efac" + } + user: user auth { + style.fill: "#eff6ff" + style.font-color: "#1d4ed8" + style.stroke: "#93c5fd" + } + adm: admin only { + style.fill: "#fff7ed" + style.font-color: "#c2410c" + style.stroke: "#fdba74" + } +} + +# ─── Client ─────────────────────────────────────────────────────────────────── + +client: Browser / iOS App { + shape: person + style.fill: "#fff9e6" +} + +# ─── Caddy ──────────────────────────────────────────────────────────────────── + +caddy: Caddy :443 { + shape: rectangle + style.fill: "#f1f5f9" + label: "Caddy :443\ncustom build · caddy-ratelimit\nsecurity headers · rate limiting\nstatic error pages" +} + +# ─── SvelteKit UI ───────────────────────────────────────────────────────────── +# Handles: auth enforcement, session, all /api/* routes that have SK counterparts + +sk: SvelteKit UI :3000 { + style.fill: "#fef3c7" + + auth: Auth { + style.fill: "#fde68a" + style.stroke: "#f59e0b" + label: "POST /api/auth/login\nPOST /api/auth/register\nPOST /api/auth/change-password\nGET /api/auth/session" + } + + catalogue_sk: Catalogue { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/catalogue-page\nGET /api/search" + } + + book_sk: Book { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/book/{slug}\nGET /api/chapter/{slug}/{n}\nGET /api/chapter-text-preview/{slug}/{n}" + } + + scrape_sk: Scrape (admin) { + style.fill: "#fff7ed" + style.stroke: "#fdba74" + label: "GET /api/scrape/status\nGET /api/scrape/tasks\nPOST /api/scrape\nPOST /api/scrape/range\nPOST /api/scrape/cancel/{id}" + } + + audio_sk: Audio { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "POST /api/audio/{slug}/{n}\nGET /api/audio/status/{slug}/{n}\nGET /api/voices" + } + + presign_sk: Presigned URLs { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/presign/chapter/{slug}/{n}\nGET /api/presign/audio/{slug}/{n}\nGET /api/presign/voice-sample/{voice}" + } + + presign_user: Presigned URLs (user) { + style.fill: "#eff6ff" + style.stroke: "#93c5fd" + label: "GET /api/presign/avatar-upload/{userId}\nGET /api/presign/avatar/{userId}" + } + + progress_sk: Progress { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/progress\nPOST /api/progress/{slug}\nDELETE /api/progress/{slug}" + } + + library_sk: Library { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/library\nPOST /api/library/{slug}\nDELETE /api/library/{slug}" + } + + comments_sk: Comments { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/comments/{slug}\nPOST /api/comments/{slug}" + } +} + +# ─── Go Backend ─────────────────────────────────────────────────────────────── +# Caddy proxies these paths directly — no SvelteKit auth layer + +be: Backend API :8080 { + style.fill: "#eef3ff" + + health_be: Health { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /health\nGET /api/version" + } + + scrape_be: Scrape admin (direct) { + style.fill: "#fff7ed" + style.stroke: "#fdba74" + label: "POST /scrape\nPOST /scrape/book\nPOST /scrape/book/range" + } + + catalogue_be: Catalogue { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/browse\nGET /api/catalogue\nGET /api/ranking\nGET /api/cover/{domain}/{slug}" + } + + book_be: Book / Chapter { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/book-preview/{slug}\nGET /api/chapter-text/{slug}/{n}\nGET /api/chapter-markdown/{slug}/{n}\nPOST /api/reindex/{slug} ⚠ admin" + } + + audio_be: Audio { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/audio-proxy/{slug}/{n}\nGET /api/voices" + } +} + +# ─── Storage ────────────────────────────────────────────────────────────────── + +storage: Storage { + style.fill: "#eaf7ea" + + pb: PocketBase :8090 { + shape: cylinder + label: "auth · books · progress\ncomments · library\nscrape_jobs · audio_cache" + } + mn: MinIO :9000 { + shape: cylinder + label: "chapters · audio\navatars · browse" + } + ms: Meilisearch :7700 { + shape: cylinder + label: "index: books" + } + vk: Valkey :6379 { + shape: cylinder + label: "presign URL cache" + } +} + +# ─── Caddy routing ──────────────────────────────────────────────────────────── + +client -> caddy: HTTPS :443 + +caddy -> sk: "/* (catch-all)\n→ SvelteKit handles auth" +caddy -> be: "/health /scrape*\n/api/browse /api/book-preview/*\n/api/chapter-text/* /api/chapter-markdown/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/* /api/catalogue /api/ranking" +caddy -> storage.mn: "/avatars/*\n(presigned GETs)" + +# ─── SvelteKit → Backend (server-side proxy) ────────────────────────────────── + +sk.catalogue_sk -> be.catalogue_be: internal proxy +sk.book_sk -> be.book_be: internal proxy +sk.audio_sk -> be.audio_be: internal proxy +sk.presign_sk -> storage.vk: check cache +sk.presign_sk -> storage.mn: generate presign +sk.presign_user -> storage.mn: generate presign + +# ─── SvelteKit → Storage (direct) ──────────────────────────────────────────── + +sk.auth -> storage.pb: sessions / users +sk.scrape_sk -> storage.pb: scrape job records +sk.progress_sk -> storage.pb +sk.library_sk -> storage.pb +sk.comments_sk -> storage.pb + +# ─── Backend → Storage ──────────────────────────────────────────────────────── + +be.catalogue_be -> storage.ms: full-text search +be.catalogue_be -> storage.pb: ranking records +be.catalogue_be -> storage.mn: cover presign +be.book_be -> storage.mn: chapter objects +be.book_be -> storage.pb: book metadata +be.audio_be -> storage.mn: audio presign +be.audio_be -> storage.vk: presign cache diff --git a/v3/docs/d2/api-routing.svg b/v3/docs/d2/api-routing.svg new file mode 100644 index 0000000..ae42f95 --- /dev/null +++ b/v3/docs/d2/api-routing.svg @@ -0,0 +1,127 @@ +LegendBrowser / iOS AppCaddy :443custom build · caddy-ratelimitsecurity headers · rate limitingstatic error pagesSvelteKit UI :3000Backend API :8080Storagepublicuser authadmin onlyPOST /api/auth/loginPOST /api/auth/registerPOST /api/auth/change-passwordGET /api/auth/sessionGET /api/catalogue-pageGET /api/searchGET /api/book/{slug}GET /api/chapter/{slug}/{n}GET /api/chapter-text-preview/{slug}/{n}GET /api/scrape/statusGET /api/scrape/tasksPOST /api/scrapePOST /api/scrape/rangePOST /api/scrape/cancel/{id}POST /api/audio/{slug}/{n}GET /api/audio/status/{slug}/{n}GET /api/voicesGET /api/presign/chapter/{slug}/{n}GET /api/presign/audio/{slug}/{n}GET /api/presign/voice-sample/{voice}GET /api/presign/avatar-upload/{userId}GET /api/presign/avatar/{userId}GET /api/progressPOST /api/progress/{slug}DELETE /api/progress/{slug}GET /api/libraryPOST /api/library/{slug}DELETE /api/library/{slug}GET /api/comments/{slug}POST /api/comments/{slug}GET /healthGET /api/versionPOST /scrapePOST /scrape/bookPOST /scrape/book/rangeGET /api/browseGET /api/catalogueGET /api/rankingGET /api/cover/{domain}/{slug}GET /api/book-preview/{slug}GET /api/chapter-text/{slug}/{n}GET /api/chapter-markdown/{slug}/{n}POST /api/reindex/{slug} ⚠ adminGET /api/audio-proxy/{slug}/{n}GET /api/voicesauth · books · progresscomments · libraryscrape_jobs · audio_cachechapters · audioavatars · browseindex: bookspresign URL cache HTTPS :443/* (catch-all)→ SvelteKit handles auth/health /scrape*/api/browse /api/book-preview/*/api/chapter-text/* /api/chapter-markdown/*/api/reindex/* /api/cover/*/api/audio-proxy/* /api/catalogue /api/ranking/avatars/*(presigned GETs)internal proxyinternal proxyinternal proxycheck cachegenerate presigngenerate presignsessions / usersscrape job recordsfull-text searchranking recordscover presignchapter objectsbook metadataaudio presignpresign cache + + + + + + + + + + + + + + + + + + + + + diff --git a/v3/docs/architecture.d2 b/v3/docs/d2/architecture.d2 similarity index 69% rename from v3/docs/architecture.d2 rename to v3/docs/d2/architecture.d2 index 1e10009..06b731b 100644 --- a/v3/docs/architecture.d2 +++ b/v3/docs/d2/architecture.d2 @@ -46,7 +46,7 @@ storage: Storage { minio: MinIO { shape: cylinder - label: "MinIO :9000\n\nbuckets:\n libnovel-chapters\n libnovel-audio\n libnovel-avatars\n libnovel-browse" + label: "MinIO :9000\n\nbuckets:\n libnovel-chapters\n libnovel-audio\n avatars\n libnovel-browse" } pocketbase: PocketBase { @@ -72,7 +72,7 @@ app: Application { caddy: caddy { shape: rectangle - label: "Caddy :443 / :80\n(reverse proxy\nauto-HTTPS via Let's Encrypt)" + label: "Caddy :443 / :80\ncustom build + caddy-ratelimit\n\nfeatures:\n auto-HTTPS (Let's Encrypt)\n security headers\n rate limiting (per-IP)\n static error pages (502/503/504)" } backend: backend { @@ -91,6 +91,17 @@ app: Application { } } +# ─── Ops ────────────────────────────────────────────────────────────────────── + +ops: Ops { + style.fill: "#fef9ec" + + watchtower: Watchtower { + shape: rectangle + label: "Watchtower\n(containrrr/watchtower)\n\npolls every 5 min\nautopulls + redeploys:\n backend · runner · ui" + } +} + # ─── Init → Storage deps ────────────────────────────────────────────────────── init.minio-init -> storage.minio: create buckets {style.stroke-dash: 4} @@ -106,17 +117,25 @@ app.runner -> storage.minio: write chapter markdown\n& audio MP3s app.runner -> storage.pocketbase: read/update scrape jobs\nwrite book records app.runner -> storage.meilisearch: index books on\nscrape completion -app.ui -> storage.valkey: read presigned URL cache\n(replaces in-process Map) +app.ui -> storage.valkey: read presigned URL cache +app.ui -> storage.pocketbase: auth, progress,\ncomments, settings # ─── App internal ───────────────────────────────────────────────────────────── -app.ui -> app.backend: REST API calls\n(server-side) +app.ui -> app.backend: REST API calls (server-side)\n/api/catalogue /api/book-preview\n/api/chapter-text /api/audio etc. # ─── Caddy routing ──────────────────────────────────────────────────────────── +# Routes sent directly to backend (no SvelteKit counterpart): +# /health /scrape* +# /api/browse /api/book-preview/* /api/chapter-text/* +# /api/reindex/* /api/cover/* /api/audio-proxy/* +# Routes sent to MinIO: +# /avatars/* +# Everything else → SvelteKit UI (including /api/scrape/*, /api/chapter-text-preview/*) -app.caddy -> app.ui: /* (proxy) -app.caddy -> app.backend: /api/*, /health (proxy) -app.caddy -> storage.minio: /s3/* (proxy, internal only) +app.caddy -> app.ui: "/* (catch-all)\n/api/scrape/*\n/api/chapter-text-preview/*\n→ SvelteKit (auth enforced)" +app.caddy -> app.backend: "/health /scrape*\n/api/browse /api/book-preview/*\n/api/chapter-text/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/*" +app.caddy -> storage.minio: "/avatars/*\n(presigned avatar GETs)" # ─── External → App ─────────────────────────────────────────────────────────── @@ -124,6 +143,12 @@ app.runner -> novelfire: scrape\n(HTTP GET) app.runner -> kokoro: TTS generation\n(HTTP POST) app.caddy -> letsencrypt: ACME certificate\n(TLS-ALPN-01) +# ─── Ops → Docker socket ────────────────────────────────────────────────────── + +ops.watchtower -> app.backend: watch (label-enabled) +ops.watchtower -> app.runner: watch (label-enabled) +ops.watchtower -> app.ui: watch (label-enabled) + # ─── Browser ────────────────────────────────────────────────────────────────── browser -> app.caddy: HTTPS :443\n(single entry point) diff --git a/v3/docs/d2/architecture.svg b/v3/docs/d2/architecture.svg new file mode 100644 index 0000000..b5aa8d0 --- /dev/null +++ b/v3/docs/d2/architecture.svg @@ -0,0 +1,129 @@ +novelfire.netKokoro-FastAPI TTSLet's EncryptBrowser / iOS AppInit containersStorageApplicationOpsminio-init(mc: create buckets)pb-init(bootstrap collections)MinIO :9000 buckets: libnovel-chapters libnovel-audio avatars libnovel-browsePocketBase :8090 collections: books chapters_idx audio_cache progress scrape_jobs app_users rankingValkey :6379 (presign URL cacheTTL-based, shared)Meilisearch :7700 indices: booksCaddy :443 / :80custom build + caddy-ratelimit features: auto-HTTPS (Let's Encrypt) security headers rate limiting (per-IP) static error pages (502/503/504)Backend API :8080(Go — HTTP API server)Runner :9091(Go — background workerscraping + TTS jobs/metrics endpoint)SvelteKit UI :3000(adapter-node)Watchtower(containrrr/watchtower) polls every 5 minautopulls + redeploys: backend · runner · ui create bucketsbootstrap schema blobs (chapters, audio,avatars, browse)structured records(books, progress, jobs…)cache presigned URLs(SET/GET with TTL)write chapter markdown& audio MP3sread/update scrape jobswrite book recordsindex books onscrape completionread presigned URL cacheauth, progress,comments, settingsREST API calls (server-side)/api/catalogue /api/book-preview/api/chapter-text /api/audio etc./* (catch-all)/api/scrape/*/api/chapter-text-preview/*→ SvelteKit (auth enforced)/health /scrape*/api/browse /api/book-preview/*/api/chapter-text/*/api/reindex/* /api/cover/*/api/audio-proxy/*/avatars/*(presigned avatar GETs)scrape(HTTP GET)TTS generation(HTTP POST)ACME certificate(TLS-ALPN-01)watch (label-enabled)watch (label-enabled)watch (label-enabled)HTTPS :443(single entry point) + + + + + + + + + + + + + + + + + + + + + + + diff --git a/v3/docs/architecture.mermaid.md b/v3/docs/mermaid/architecture.mermaid.md similarity index 72% rename from v3/docs/architecture.mermaid.md rename to v3/docs/mermaid/architecture.mermaid.md index 3e652e4..4752934 100644 --- a/v3/docs/architecture.mermaid.md +++ b/v3/docs/mermaid/architecture.mermaid.md @@ -1,3 +1,5 @@ +# Architecture Overview + ```mermaid graph LR %% ── External ────────────────────────────────────────────────────────── @@ -22,12 +24,17 @@ graph LR %% ── Application ─────────────────────────────────────────────────────── subgraph APP["Application"] - CD[Caddy :443/:80\nreverse proxy\nauto-HTTPS] + CD["Caddy :443/:80\ncustom build + caddy-ratelimit\nauto-HTTPS · security headers\nrate limiting · error pages"] BE[Backend API :8080\nGo HTTP server] RN[Runner :9091\nGo background worker\n/metrics endpoint] UI[SvelteKit UI :3000\nadapter-node] end + %% ── Ops ─────────────────────────────────────────────────────────────── + subgraph OPS["Ops"] + WT[Watchtower\npolls every 5 min\nautopull + redeploy\nbackend · runner · ui] + end + %% ── Init → Storage ──────────────────────────────────────────────────── MI -.->|create buckets| MN PI -.->|bootstrap schema| PB @@ -40,20 +47,26 @@ graph LR RN -->|read/update jobs & books| PB RN -->|index books on scrape| MS UI -->|read presign cache| VK + UI -->|auth · progress · comments| PB %% ── App internal ────────────────────────────────────────────────────── - UI -->|REST API| BE + UI -->|"REST API (server-side)\n/api/catalogue /api/book-preview\n/api/chapter-text /api/audio"| BE %% ── Caddy routing ───────────────────────────────────────────────────── - CD -->|/* proxy| UI - CD -->|/api/* /health proxy| BE - CD -->|/s3/* proxy internal| MN + CD -->|"/* catch-all\n/api/scrape/*\n/api/chapter-text-preview/*\n→ SvelteKit (auth enforced)"| UI + CD -->|"/health /scrape*\n/api/browse /api/book-preview/*\n/api/chapter-text/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/*"| BE + CD -->|/avatars/* presigned GETs| MN %% ── Runner → External ───────────────────────────────────────────────── RN -->|scrape HTTP GET| NF RN -->|TTS HTTP POST| KK CD -->|ACME certificate| LE + %% ── Ops ─────────────────────────────────────────────────────────────── + WT -->|watch label-enabled| BE + WT -->|watch label-enabled| RN + WT -->|watch label-enabled| UI + %% ── Client ──────────────────────────────────────────────────────────── CL -->|HTTPS :443 single entry| CD ``` diff --git a/v3/docs/data-flow.mermaid.md b/v3/docs/mermaid/data-flow.mermaid.md similarity index 86% rename from v3/docs/data-flow.mermaid.md rename to v3/docs/mermaid/data-flow.mermaid.md index 33f1678..9880853 100644 --- a/v3/docs/data-flow.mermaid.md +++ b/v3/docs/mermaid/data-flow.mermaid.md @@ -18,7 +18,7 @@ flowchart TD E --> G{New chapters\nfound?} G -- no --> Z([Done — next book]) G -- yes --> H - F --> H[Scrape chapter list\n→ chapters_idx in PocketBase] + F --> H[Scrape chapter list with upTo limit\n→ chapters_idx in PocketBase\nretries on 429 with Retry-After backoff] H --> I[Worker pool — N goroutines\nRUNNER_MAX_CONCURRENT_SCRAPE] I --> J[For each missing chapter:\nGET chapter HTML from novelfire.net] J --> K[Parse HTML → Markdown\nhtmlutil.NodeToMarkdown] @@ -31,7 +31,7 @@ flowchart TD ## On-Demand Single-Book Scrape Triggered when a user visits `/books/{slug}` and the book is not in PocketBase. -The UI calls `GET /api/book-preview/{slug}` → backend enqueues a task. +The UI calls `GET /api/book-preview/{slug}` → backend enqueues a scrape task. ```mermaid sequenceDiagram @@ -76,23 +76,23 @@ requests poll for completion and then stream from MinIO via presigned URL. ```mermaid flowchart TD - A([POST /api/audio/{slug}/{n}\nbody: voice=af_bella]) --> B{Audio already\nin MinIO?} + A(["POST /api/audio/{slug}/{n}\nbody: voice=af_bella"]) --> B{Audio already\nin MinIO?} B -- yes --> C[200 status: done] B -- no --> D{Job already\nin queue?} - D -- yes pending/generating --> E[202 task_id + status] + D -- "yes pending/generating" --> E[202 task_id + status] D -- no --> F[INSERT audio_task\nstatus=pending\nin PocketBase] F --> E G([Runner polls task queue]) --> H[Claim audio_task\nstatus=generating] - H --> I[GET /api/chapter-text/{slug}/{n}\nfrom backend — plain text] + H --> I["GET /api/chapter-text/{slug}/{n}\nfrom backend — plain text"] I --> J[POST /v1/audio/speech\nto Kokoro-FastAPI\nbody: text + voice] J --> K[Stream MP3 response] K --> L[PUT object to MinIO\nlibnovel-audio/{slug}/{n}/{voice}.mp3] L --> M[UPDATE audio_task\nstatus=done] - N([Client polls\nGET /api/audio/status/{slug}/{n}]) --> O{status?} - O -- pending/generating --> N - O -- done --> P[GET /api/presign/audio/{slug}/{n}] + N(["Client polls\nGET /api/audio/status/{slug}/{n}"]) --> O{status?} + O -- "pending/generating" --> N + O -- done --> P["GET /api/presign/audio/{slug}/{n}"] P --> Q{Valkey cache hit?} Q -- yes --> R[302 → presigned URL] Q -- no --> S[GeneratePresignedURL\nfrom MinIO — TTL 1h] diff --git a/v3/docs/request-flow.mermaid.md b/v3/docs/mermaid/request-flow.mermaid.md similarity index 55% rename from v3/docs/request-flow.mermaid.md rename to v3/docs/mermaid/request-flow.mermaid.md index af60b6b..834182b 100644 --- a/v3/docs/request-flow.mermaid.md +++ b/v3/docs/mermaid/request-flow.mermaid.md @@ -3,7 +3,7 @@ Two representative request paths through the stack: a **page load** (SSR) and a **media playback** (presigned URL → direct MinIO stream). -## SSR Page Load — Browse / Book Detail +## SSR Page Load — Catalogue / Book Detail ```mermaid sequenceDiagram @@ -14,10 +14,9 @@ sequenceDiagram participant MS as Meilisearch :7700 participant PB as PocketBase :8090 participant VK as Valkey :6379 - participant MN as MinIO :9000 - C->>CD: HTTPS GET /browse - CD->>UI: proxy /* + C->>CD: HTTPS GET /catalogue + CD->>UI: proxy /* (SvelteKit catch-all) UI->>BE: GET /api/catalogue?page=1&sort=popular BE->>MS: search(query, filters, sort) MS-->>BE: [{slug, title, …}, …] @@ -25,9 +24,9 @@ sequenceDiagram UI-->>CD: SSR HTML CD-->>C: 200 HTML - Note over C,UI: Infinite scroll — client fetches next page - C->>CD: HTTPS GET /api/browse-page?page=2 - CD->>UI: proxy (SvelteKit API route) + Note over C,UI: Infinite scroll — client fetches next page via SvelteKit API route + C->>CD: HTTPS GET /api/catalogue-page?page=2 + CD->>UI: proxy /* (SvelteKit /api/catalogue-page server route) UI->>BE: GET /api/catalogue?page=2 BE->>MS: search(…) MS-->>BE: next page @@ -47,16 +46,19 @@ sequenceDiagram participant MN as MinIO :9000 C->>CD: GET /api/presign/audio/{slug}/{n}?voice=af_bella - CD->>BE: proxy /api/* + CD->>UI: proxy /* (SvelteKit /api/presign/audio route) + UI->>BE: GET /api/presign/audio/{slug}/{n}?voice=af_bella BE->>VK: GET presign:audio:{slug}:{n}:{voice} alt cache hit VK-->>BE: presigned URL (TTL remaining) - BE-->>C: 302 redirect → presigned URL + BE-->>UI: 302 redirect → presigned URL + UI-->>C: 302 redirect else cache miss BE->>MN: GeneratePresignedURL(audio-bucket, key, 1h) MN-->>BE: presigned URL BE->>VK: SET presign:audio:… EX 3500 - BE-->>C: 302 redirect → presigned URL + BE-->>UI: 302 redirect → presigned URL + UI-->>C: 302 redirect end C->>MN: GET presigned URL (direct, no proxy) MN-->>C: audio/mpeg stream @@ -74,14 +76,36 @@ sequenceDiagram participant MN as MinIO :9000 C->>CD: HTTPS GET /books/{slug}/chapters/{n} - CD->>UI: proxy /* + CD->>UI: proxy /* (SvelteKit catch-all) UI->>PB: getBook(slug) + listChapterIdx(slug) PB-->>UI: book meta + chapter list - UI->>BE: GET /api/chapter-markdown/{slug}/{n} + UI->>BE: GET /api/chapter-text/{slug}/{n} BE->>MN: GetObject(chapters-bucket, {slug}/{n}.md) MN-->>BE: markdown text - BE-->>UI: markdown body + BE-->>UI: plain text (markdown stripped) Note over UI: marked() → HTML UI-->>CD: SSR HTML CD-->>C: 200 HTML ``` + +## Caddy Request Lifecycle + +Shows how security hardening applies before a request reaches any upstream. + +```mermaid +flowchart TD + A([Incoming HTTPS request]) --> B[TLS termination\nLet's Encrypt cert] + B --> C{Rate limit check\ncaddy-ratelimit} + C -- over limit --> D[429 Too Many Requests] + C -- ok --> E[Add security headers\nX-Frame-Options · X-Content-Type-Options\nReferrer-Policy · Permissions-Policy\nHSTS · X-XSS-Protection\nremove Server header] + E --> F{Route match} + F -- "/health /scrape*\n/api/browse /api/book-preview/*\n/api/chapter-text/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/*" --> G[reverse_proxy → backend:8080] + F -- "/avatars/*" --> H[reverse_proxy → minio:9000] + F -- "/* everything else\n(incl. /api/scrape/*\n/api/chapter-text-preview/*)" --> I[reverse_proxy → ui:3000\nSvelteKit auth middleware runs] + G --> J{Upstream healthy?} + H --> J + I --> J + J -- yes --> K([Response to client]) + J -- "502/503/504" --> L[handle_errors\nstatic HTML from /srv/errors/] + L --> K +``` diff --git a/v3/ui/src/lib/components/CommentsSection.svelte b/v3/ui/src/lib/components/CommentsSection.svelte index 34316ff..e686eb1 100644 --- a/v3/ui/src/lib/components/CommentsSection.svelte +++ b/v3/ui/src/lib/components/CommentsSection.svelte @@ -2,20 +2,7 @@ import { Button } from '$lib/components/ui/button'; import { Textarea } from '$lib/components/ui/textarea'; import { cn } from '$lib/utils'; - - interface BookComment { - id: string; - slug: string; - user_id: string; - username: string; - body: string; - upvotes: number; - downvotes: number; - created: string; - parent_id?: string; - replies?: BookComment[]; - } - + import type { BookComment } from '$lib/types'; let { slug, isLoggedIn = false, @@ -312,7 +299,7 @@ {:else}

- Log in + Log in to leave a comment.

{/if} diff --git a/v3/ui/src/lib/server/catalogue.ts b/v3/ui/src/lib/server/catalogue.ts new file mode 100644 index 0000000..7a1b644 --- /dev/null +++ b/v3/ui/src/lib/server/catalogue.ts @@ -0,0 +1,71 @@ +/** + * Shared types and helpers for the /api/catalogue backend response. + * + * Imported by both: + * - src/routes/catalogue/+page.server.ts (SSR page load) + * - src/routes/api/catalogue-page/+server.ts (infinite-scroll proxy) + */ + +/** Shape of a single book as returned by GET /api/catalogue on the Go backend. */ +export interface CatalogueBook { + slug: string; + title: string; + author: string; + cover: string; + status: string; + genres: string[]; + summary: string; + total_chapters: number; + source_url: string; + ranking: number; + rating: number; +} + +/** Facets returned alongside catalogue results for dynamic filter options. */ +export interface CatalogueFacets { + genres: string[]; + statuses: string[]; +} + +/** Full response shape from GET /api/catalogue. */ +export interface CatalogueResponse { + books: CatalogueBook[]; + page: number; + limit: number; + total: number; + has_next: boolean; + facets?: CatalogueFacets; +} + +/** Normalised book shape consumed by the catalogue UI. */ +export interface NovelListing { + slug: string; + title: string; + cover: string; + rank: string; + rating: string; + chapters: string; + url: string; + // enriched fields + author?: string; + status?: string; + genres?: string[]; + source_url?: string; +} + +/** Convert a raw CatalogueBook into the UI NovelListing shape. */ +export function bookToListing(book: CatalogueBook): NovelListing { + return { + slug: book.slug, + title: book.title, + cover: book.cover, + rank: book.ranking > 0 ? `#${book.ranking}` : '', + rating: book.rating > 0 ? String(book.rating) : '', + chapters: book.total_chapters > 0 ? `${book.total_chapters} chapters` : '', + url: book.source_url ?? '', + author: book.author, + status: book.status, + genres: book.genres ?? [], + source_url: book.source_url + }; +} diff --git a/v3/ui/src/lib/server/minio.ts b/v3/ui/src/lib/server/minio.ts index 0dccca9..149573e 100644 --- a/v3/ui/src/lib/server/minio.ts +++ b/v3/ui/src/lib/server/minio.ts @@ -51,7 +51,7 @@ export async function presignAvatarUrl(userId: string): Promise { throw new Error(`presign avatar failed: ${res.status} ${body}`); } const data = (await res.json()) as { url: string }; - return data.url ?? null; + return data.url ? rewriteHost(data.url) : null; } /** diff --git a/v3/ui/src/lib/server/pocketbase.ts b/v3/ui/src/lib/server/pocketbase.ts index 7f1897d..763afe8 100644 --- a/v3/ui/src/lib/server/pocketbase.ts +++ b/v3/ui/src/lib/server/pocketbase.ts @@ -46,7 +46,7 @@ export interface Progress { updated: string; } -export interface UserSettings { +export interface PBUserSettings { id?: string; session_id: string; user_id?: string; @@ -583,8 +583,8 @@ function settingsFilter(sessionId: string, userId?: string): string { export async function getSettings( sessionId: string, userId?: string -): Promise { - return listOne('user_settings', settingsFilter(sessionId, userId)); +): Promise { + return listOne('user_settings', settingsFilter(sessionId, userId)); } export async function saveSettings( @@ -592,12 +592,12 @@ export async function saveSettings( settings: { autoNext: boolean; voice: string; speed: number }, userId?: string ): Promise { - const existing = await listOne( + const existing = await listOne( 'user_settings', settingsFilter(sessionId, userId) ); - const payload: Partial = { + const payload: Partial = { session_id: sessionId, auto_next: settings.autoNext, voice: settings.voice, @@ -684,6 +684,8 @@ export interface ScrapingTask { books_found: number; chapters_scraped: number; chapters_skipped: number; + from_chapter: number; + to_chapter: number; errors: number; started: string; finished: string; @@ -694,6 +696,10 @@ export async function listScrapingTasks(): Promise { return listAll('scraping_tasks', '', '-started'); } +export async function getScrapingTask(id: string): Promise { + return listOne('scraping_tasks', `id="${id}"`); +} + // ─── Audio jobs ─────────────────────────────────────────────────────────────── export interface AudioJob { @@ -850,7 +856,7 @@ export async function updateUserAvatarUrl(userId: string, avatarUrl: string): Pr // ─── Comments ───────────────────────────────────────────────────────────────── -export interface BookComment { +export interface PBBookComment { id: string; slug: string; user_id: string; @@ -881,7 +887,7 @@ export type CommentSort = 'top' | 'new'; export async function listComments( slug: string, sort: CommentSort = 'new' -): Promise { +): Promise { const token = await getToken(); const slugEsc = slug.replace(/"/g, '\\"'); // Only top-level comments (parent_id is empty or missing) @@ -896,7 +902,7 @@ export async function listComments( ); if (!res.ok) return []; const data = await res.json(); - let items = (data.items ?? []) as BookComment[]; + let items = (data.items ?? []) as PBBookComment[]; if (sort === 'top') { items = items.sort((a, b) => { const scoreB = (b.upvotes ?? 0) - (b.downvotes ?? 0); @@ -913,7 +919,7 @@ export async function listComments( * List replies (1-level deep) for a single parent comment. * Always sorted oldest-first so the conversation reads naturally. */ -export async function listReplies(parentId: string): Promise { +export async function listReplies(parentId: string): Promise { const token = await getToken(); const filter = encodeURIComponent(`parent_id="${parentId.replace(/"/g, '\\"')}"`); const res = await fetch( @@ -922,7 +928,7 @@ export async function listReplies(parentId: string): Promise { ); if (!res.ok) return []; const data = await res.json(); - return (data.items ?? []) as BookComment[]; + return (data.items ?? []) as PBBookComment[]; } /** @@ -935,7 +941,7 @@ export async function createComment( userId: string | undefined, username: string, parentId?: string -): Promise { +): Promise { const token = await getToken(); const res = await fetch(`${PB_URL}/api/collections/book_comments/records`, { method: 'POST', @@ -955,7 +961,7 @@ export async function createComment( const text = await res.text().catch(() => ''); throw new Error(`createComment failed: ${res.status} ${text}`); } - return res.json() as Promise; + return res.json() as Promise; } /** @@ -971,7 +977,7 @@ export async function deleteComment(commentId: string, userId: string): Promise< headers: { Authorization: `Bearer ${token}` } }); if (!getRes.ok) throw new Error(`Comment not found: ${commentId}`); - const comment = (await getRes.json()) as BookComment; + const comment = (await getRes.json()) as PBBookComment; if (comment.user_id !== userId) throw new Error('Not authorized to delete this comment'); // Delete any replies first @@ -982,7 +988,7 @@ export async function deleteComment(commentId: string, userId: string): Promise< ); if (repliesRes.ok) { const repliesData = await repliesRes.json(); - const replies = (repliesData.items ?? []) as BookComment[]; + const replies = (repliesData.items ?? []) as PBBookComment[]; await Promise.all( replies.map((r) => fetch(`${PB_URL}/api/collections/book_comments/records/${r.id}`, { @@ -1035,7 +1041,7 @@ export async function voteComment( vote: 'up' | 'down', sessionId: string, userId?: string -): Promise { +): Promise { const token = await getToken(); // Fetch current comment @@ -1043,7 +1049,7 @@ export async function voteComment( headers: { Authorization: `Bearer ${token}` } }); if (!commentRes.ok) throw new Error(`Comment not found: ${commentId}`); - const comment = (await commentRes.json()) as BookComment; + const comment = (await commentRes.json()) as PBBookComment; const existing = await getCommentVote(commentId, sessionId, userId); @@ -1086,7 +1092,7 @@ export async function voteComment( }) }); if (!patchRes.ok) throw new Error(`Failed to update vote counts on comment ${commentId}`); - return patchRes.json() as Promise; + return patchRes.json() as Promise; } /** diff --git a/v3/ui/src/lib/server/scraper.ts b/v3/ui/src/lib/server/scraper.ts index cdecfc6..f5871a5 100644 --- a/v3/ui/src/lib/server/scraper.ts +++ b/v3/ui/src/lib/server/scraper.ts @@ -33,4 +33,28 @@ export async function backendFetch(path: string, init?: RequestInit): Promise + import { page } from '$app/state'; + + const status = $derived(page.status); + const message = $derived(page.error?.message ?? 'Something went wrong.'); + + const title = $derived( + status === 404 + ? 'Page not found' + : status === 403 + ? 'Access denied' + : status === 429 + ? 'Too many requests' + : status >= 500 + ? 'Server error' + : 'Error' + ); + + const description = $derived( + status === 404 + ? "The page you're looking for doesn't exist or has been moved." + : status === 403 + ? "You don't have permission to access this page." + : status === 429 + ? 'You are sending too many requests. Please slow down and try again shortly.' + : status >= 500 + ? 'An unexpected error occurred on our end. Try refreshing, or come back in a moment.' + : message + ); + + const code = $derived(String(status)); + + + + {status} — {title} · libnovel + + + +
+ +

+ {code} +

+ + +
+

{title}

+

{description}

+
+ + +
+ + Go home + + +
+ + +

libnovel

+
diff --git a/v3/ui/src/routes/+layout.svelte b/v3/ui/src/routes/+layout.svelte index c4ce279..fdbd775 100644 --- a/v3/ui/src/routes/+layout.svelte +++ b/v3/ui/src/routes/+layout.svelte @@ -233,8 +233,8 @@ Library @@ -250,15 +250,9 @@ - {/if} (menuOpen = false)} - class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/browse') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}" + class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/catalogue') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}" > Discover @@ -381,7 +375,7 @@