feat(v3): Caddy hardening, Watchtower, scrape fixes, docs diagrams

- Caddy: custom image with caddy-ratelimit plugin, security headers
  (X-Frame-Options, HSTS, CSP-adjacent, etc.), per-IP rate limiting on
  auth/scrape/global zones, static error pages (502/503/504), fix routing
  to remove /api/scrape/* and /api/chapter-text-preview/* direct-to-backend
  (were bypassing SvelteKit auth middleware)
- docker-compose: Caddy build context + error volume, Watchtower service
  (label-enable mode, 5 min poll), watchtower labels on backend/runner/ui
- Scraper: ScrapeChapterList uses retryGet (9 attempts, Retry-After backoff)
  to fix 429-induced chapter list failures; upTo param stops pagination early
  for range scrapes
- UI: Browse→Catalogue rename (routes, API, links), admin scrape page
  Continue/Retry buttons, +error.svelte branded error page, type cleanup
  (removed dead exports, added BookPreviewMeta/BookPreviewResponse to scraper.ts)
- Meilisearch: meta_updated field, sort=update fix, facet distribution
- Docs: reorganise into docs/d2/ and docs/mermaid/ subdirectories, update
  all diagrams to reflect Caddy/Watchtower/routing changes, add
  api-routing.d2 ownership map with auth-level colour coding, regenerate SVGs
This commit is contained in:
Admin
2026-03-22 21:10:38 +05:00
parent a85636d5db
commit 5b27d501af
60 changed files with 2485 additions and 908 deletions

146
.gitea/workflows/ci-v3.yaml Normal file
View File

@@ -0,0 +1,146 @@
name: CI / v3
on:
push:
branches: ["main", "master"]
paths:
- "v3/**"
- ".gitea/workflows/ci-v3.yaml"
pull_request:
branches: ["main", "master"]
paths:
- "v3/**"
- ".gitea/workflows/ci-v3.yaml"
concurrency:
group: ${{ gitea.workflow }}-${{ gitea.ref }}
cancel-in-progress: true
jobs:
# ── backend: lint & test ─────────────────────────────────────────────────────
test-backend:
name: Test backend
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: v3/backend/go.mod
cache-dependency-path: v3/backend/go.sum
- name: go vet
working-directory: v3/backend
run: go vet ./...
- name: Run tests
working-directory: v3/backend
run: go test -short -race -count=1 -timeout=60s ./...
# ── ui: type-check ───────────────────────────────────────────────────────────
check-ui:
name: Check ui
runs-on: ubuntu-latest
defaults:
run:
working-directory: v3/ui
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: "22"
cache: npm
cache-dependency-path: v3/ui/package-lock.json
- name: Install dependencies
run: npm ci
- name: Type check
run: npm run check
- name: Build
run: npm run build
# ── docker: backend (push to Docker Hub on branch push only) ─────────────────
docker-backend:
name: Docker / backend
runs-on: ubuntu-latest
needs: [test-backend]
if: gitea.event_name == 'push'
steps:
- uses: actions/checkout@v4
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v6
with:
context: v3/backend
target: backend
push: true
tags: |
${{ secrets.DOCKER_USER }}/libnovel-v3-backend:latest
${{ secrets.DOCKER_USER }}/libnovel-v3-backend:${{ gitea.sha }}
build-args: |
VERSION=${{ gitea.sha }}
COMMIT=${{ gitea.sha }}
# ── docker: runner ────────────────────────────────────────────────────────────
docker-runner:
name: Docker / runner
runs-on: ubuntu-latest
needs: [test-backend]
if: gitea.event_name == 'push'
steps:
- uses: actions/checkout@v4
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v6
with:
context: v3/backend
target: runner
push: true
tags: |
${{ secrets.DOCKER_USER }}/libnovel-v3-runner:latest
${{ secrets.DOCKER_USER }}/libnovel-v3-runner:${{ gitea.sha }}
build-args: |
VERSION=${{ gitea.sha }}
COMMIT=${{ gitea.sha }}
# ── docker: ui ────────────────────────────────────────────────────────────────
docker-ui:
name: Docker / ui
runs-on: ubuntu-latest
needs: [check-ui]
if: gitea.event_name == 'push'
steps:
- uses: actions/checkout@v4
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v6
with:
context: v3/ui
push: true
tags: |
${{ secrets.DOCKER_USER }}/libnovel-v3-ui:latest
${{ secrets.DOCKER_USER }}/libnovel-v3-ui:${{ gitea.sha }}
build-args: |
BUILD_VERSION=${{ gitea.sha }}
BUILD_COMMIT=${{ gitea.sha }}

View File

@@ -0,0 +1,177 @@
name: Release / v3
on:
push:
tags:
- "v3/*" # e.g. v3/1.0.0, v3/1.2.3
- "v3-*" # e.g. v3-1.0.0 (alternative convention)
concurrency:
group: ${{ gitea.workflow }}-${{ gitea.ref }}
cancel-in-progress: true
jobs:
# ── backend: lint & test ─────────────────────────────────────────────────────
test-backend:
name: Test backend
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: v3/backend/go.mod
cache-dependency-path: v3/backend/go.sum
- name: go vet
working-directory: v3/backend
run: go vet ./...
- name: Run tests
working-directory: v3/backend
run: go test -short -race -count=1 -timeout=60s ./...
# ── ui: type-check & build ───────────────────────────────────────────────────
check-ui:
name: Check ui
runs-on: ubuntu-latest
defaults:
run:
working-directory: v3/ui
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: "22"
cache: npm
cache-dependency-path: v3/ui/package-lock.json
- name: Install dependencies
run: npm ci
- name: Type check
run: npm run check
- name: Build
run: npm run build
# ── docker: backend ───────────────────────────────────────────────────────────
docker-backend:
name: Docker / backend
runs-on: ubuntu-latest
needs: [test-backend]
steps:
- uses: actions/checkout@v4
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ secrets.DOCKER_USER }}/libnovel-v3-backend
tags: |
type=match,pattern=v3/(.*),group=1
type=raw,value=latest
- name: Build and push
uses: docker/build-push-action@v6
with:
context: v3/backend
target: backend
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
VERSION=${{ steps.meta.outputs.version }}
COMMIT=${{ gitea.sha }}
# ── docker: runner ────────────────────────────────────────────────────────────
docker-runner:
name: Docker / runner
runs-on: ubuntu-latest
needs: [test-backend]
steps:
- uses: actions/checkout@v4
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ secrets.DOCKER_USER }}/libnovel-v3-runner
tags: |
type=match,pattern=v3/(.*),group=1
type=raw,value=latest
- name: Build and push
uses: docker/build-push-action@v6
with:
context: v3/backend
target: runner
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
VERSION=${{ steps.meta.outputs.version }}
COMMIT=${{ gitea.sha }}
# ── docker: ui ────────────────────────────────────────────────────────────────
docker-ui:
name: Docker / ui
runs-on: ubuntu-latest
needs: [check-ui]
steps:
- uses: actions/checkout@v4
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ secrets.DOCKER_USER }}/libnovel-v3-ui
tags: |
type=match,pattern=v3/(.*),group=1
type=raw,value=latest
- name: Build and push
uses: docker/build-push-action@v6
with:
context: v3/ui
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
BUILD_VERSION=${{ steps.meta.outputs.version }}
BUILD_COMMIT=${{ gitea.sha }}
# ── Gitea release ─────────────────────────────────────────────────────────────
release:
name: Gitea Release
runs-on: ubuntu-latest
needs: [docker-backend, docker-runner, docker-ui]
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Create release
uses: actions/gitea-release-action@v1
with:
token: ${{ secrets.GITEA_TOKEN }}
generate_release_notes: true

BIN
backend/bin/runner Executable file

Binary file not shown.

View File

@@ -1,6 +1,7 @@
# v3/Caddyfile # v3/Caddyfile
# #
# Caddy reverse proxy for LibNovel v3. # Caddy reverse proxy for LibNovel v3.
# Custom build includes github.com/mholt/caddy-ratelimit.
# #
# Environment variables consumed (set in docker-compose.yml): # Environment variables consumed (set in docker-compose.yml):
# DOMAIN — public hostname, e.g. libnovel.example.com # DOMAIN — public hostname, e.g. libnovel.example.com
@@ -9,51 +10,106 @@
# #
# Routing rules: # Routing rules:
# /health → backend:8080 (liveness probe) # /health → backend:8080 (liveness probe)
# /scrape* → backend:8080 (scrape task creation) # /scrape* → backend:8080 (Go admin scrape endpoints)
# /api/browse → backend:8080 (MinIO-cached browse pages) # /api/browse → backend:8080 (MinIO-cached browse pages)
# /api/book-preview/* → backend:8080 (live scrape, no store write) # /api/book-preview/* → backend:8080 (live scrape, no store write)
# /api/chapter-text-preview/*/* → backend:8080 (live chapter, no store write) # /api/chapter-text/* → backend:8080 (chapter markdown from MinIO)
# /api/chapter-text/*/* → backend:8080 (chapter markdown from MinIO)
# /api/reindex/* → backend:8080 (rebuild chapter index) # /api/reindex/* → backend:8080 (rebuild chapter index)
# /api/cover/* → backend:8080 (proxy cover image) # /api/cover/* → backend:8080 (proxy cover image)
# /api/audio-proxy/*/* → backend:8080 (proxy generated audio) # /api/audio-proxy/* → backend:8080 (proxy generated audio)
# /api/scrape/* → backend:8080 (scrape job status/tasks) # /avatars/* → minio:9000 (presigned avatar GETs)
# /* (everything else) → ui:3000 (SvelteKit — handles all # /* (everything else) → ui:3000 (SvelteKit — handles all
# remaining /api/* routes too) # remaining /api/* routes)
# #
# The SvelteKit UI itself proxies to the backend for: ranking, voices, search, # Routes intentionally removed from direct-to-backend:
# browse-page, presign, audio, progress, and the Go /api/progress endpoint. # /api/scrape/* — SvelteKit has /api/scrape/ counterparts
# MinIO and PocketBase are NOT exposed publicly. # that enforce auth; routing directly would
# bypass SK middleware.
# /api/chapter-text-preview/* — Same: SvelteKit owns
# /api/chapter-text-preview/[slug]/[n].
{ {
# Email for Let's Encrypt ACME account registration. # Email for Let's Encrypt ACME account registration.
# When CADDY_ACME_EMAIL is set this expands to e.g. "email you@example.com". # When CADDY_ACME_EMAIL is set this expands to e.g. "email you@example.com".
{$CADDY_ACME_EMAIL:} # When unset it expands to an empty string and is silently ignored.
{$CADDY_ACME_EMAIL:}
} }
(security_headers) { (security_headers) {
header {
# Prevent clickjacking
X-Frame-Options "SAMEORIGIN"
# Prevent MIME-type sniffing
X-Content-Type-Options "nosniff"
# Minimal referrer info for cross-origin requests
Referrer-Policy "strict-origin-when-cross-origin"
# Restrict powerful browser features
Permissions-Policy "camera=(), microphone=(), geolocation=(), payment=()"
# Enforce HTTPS for 1 year (includeSubDomains)
Strict-Transport-Security "max-age=31536000; includeSubDomains"
# Enable XSS filter in older browsers
X-XSS-Protection "1; mode=block"
# Remove server identity header
-Server
}
}
{$DOMAIN:localhost} {
import security_headers import security_headers
handle /health {
# ── Rate limiting ─────────────────────────────────────────────────────────
# Auth endpoints: strict — 10 req/min per IP
rate_limit {
zone auth_zone {
match {
path /api/auth/login /api/auth/register /api/auth/change-password
}
key {remote_host}
window 1m
events 10
}
}
# Admin scrape endpoints: moderate — 20 req/min per IP
rate_limit {
zone scrape_zone {
match {
path /scrape*
}
key {remote_host}
window 1m
events 20
}
}
# Global: 300 req/min per IP (covers everything)
rate_limit {
zone global_zone {
key {remote_host}
window 1m
events 300
}
}
# ── Liveness probe ────────────────────────────────────────────────────────
handle /health {
reverse_proxy backend:8080 reverse_proxy backend:8080
} }
# ── Scrape task creation (Go backend only) ──────────────────────────────── # ── Scrape task creation (Go backend only) ────────────────────────────────
handle /scrape* { handle /scrape* {
reverse_proxy backend:8080 reverse_proxy backend:8080
} }
# ── Backend-only API paths ──────────────────────────────────────────────── # ── Backend-only API paths ────────────────────────────────────────────────
# These paths are served exclusively by the Go backend and have no # These paths are served exclusively by the Go backend and have no
# implemented in the SvelteKit UI. # SvelteKit counterpart. Routing them here skips SK intentionally.
handle /api/browse { handle /api/browse {
reverse_proxy backend:8080 reverse_proxy backend:8080
} }
handle /api/book-preview/* { handle /api/book-preview/* {
reverse_proxy backend:8080 reverse_proxy backend:8080
} }
handle /api/chapter-text/* { handle /api/chapter-text/* {
reverse_proxy backend:8080
}
handle /api/chapter-text/* {
reverse_proxy backend:8080 reverse_proxy backend:8080
} }
handle /api/reindex/* { handle /api/reindex/* {
@@ -66,8 +122,10 @@
reverse_proxy backend:8080 reverse_proxy backend:8080
} }
reverse_proxy backend:8080 # ── MinIO avatars bucket (presigned GET only) ─────────────────────────────
} handle /avatars/* {
reverse_proxy minio:9000
}
# ── SvelteKit UI (catch-all — includes all remaining /api/* routes) ─────── # ── SvelteKit UI (catch-all — includes all remaining /api/* routes) ───────
handle { handle {
@@ -75,7 +133,26 @@
} }
# ── Caddy-level error pages ─────────────────────────────────────────────── # ── Caddy-level error pages ───────────────────────────────────────────────
log { # These fire when the upstream (backend or ui) is completely unreachable.
# SvelteKit's own +error.svelte handles application-level errors (404, 500).
handle_errors 502 {
root * /srv/errors
rewrite * /502.html
file_server
}
handle_errors 503 {
root * /srv/errors
rewrite * /503.html
file_server
}
handle_errors 504 {
root * /srv/errors
rewrite * /504.html
file_server
}
# ── Logging ───────────────────────────────────────────────────────────────
log {
output stdout output stdout
format json format json
} }

BIN
v3/backend/bin/runner Executable file

Binary file not shown.

View File

@@ -111,7 +111,8 @@ func run() error {
MaxConcurrentAudio: cfg.Runner.MaxConcurrentAudio, MaxConcurrentAudio: cfg.Runner.MaxConcurrentAudio,
OrchestratorWorkers: workers, OrchestratorWorkers: workers,
MetricsAddr: cfg.Runner.MetricsAddr, MetricsAddr: cfg.Runner.MetricsAddr,
CatalogueRefreshInterval: cfg.Runner.CatalogueRefreshInterval, CatalogueRefreshInterval: cfg.Runner.CatalogueRefreshInterval,
SkipInitialCatalogueRefresh: cfg.Runner.SkipInitialCatalogueRefresh,
} }
deps := runner.Dependencies{ deps := runner.Dependencies{
Consumer: store, Consumer: store,

BIN
v3/backend/healthcheck Executable file

Binary file not shown.

View File

@@ -7,8 +7,7 @@ package backend
// handleScrapeStatus, handleScrapeTasks // handleScrapeStatus, handleScrapeTasks
// handleBrowse, handleSearch // handleBrowse, handleSearch
// handleGetRanking, handleGetCover // handleGetRanking, handleGetCover
// handleBookPreview, handleChapterText, handleReindex // handleBookPreview, handleChapterText, handleChapterTextPreview, handleChapterMarkdown, handleReindex
// handleChapterText, handleReindex
// handleAudioGenerate, handleAudioStatus, handleAudioProxy // handleAudioGenerate, handleAudioStatus, handleAudioProxy
// handleVoices // handleVoices
// handlePresignChapter, handlePresignAudio, handlePresignVoiceSample // handlePresignChapter, handlePresignAudio, handlePresignVoiceSample
@@ -29,6 +28,8 @@ package backend
// by the runner after each catalogue scrape). // by the runner after each catalogue scrape).
// - GET /api/book-preview returns stored data when in library, or enqueues a // - GET /api/book-preview returns stored data when in library, or enqueues a
// scrape task and returns 202 when not. The backend never scrapes directly. // scrape task and returns 202 when not. The backend never scrapes directly.
// - GET /api/chapter-text-preview scrapes a chapter live from novelfire.net
// directly (no runner task, no store writes). Used for unscraped books.
import ( import (
"context" "context"
@@ -45,6 +46,8 @@ import (
"github.com/libnovel/backend/internal/domain" "github.com/libnovel/backend/internal/domain"
"github.com/libnovel/backend/internal/kokoro" "github.com/libnovel/backend/internal/kokoro"
"github.com/libnovel/backend/internal/meili" "github.com/libnovel/backend/internal/meili"
"github.com/libnovel/backend/internal/novelfire/htmlutil"
"github.com/libnovel/backend/internal/scraper"
) )
const ( const (
@@ -502,6 +505,117 @@ func (s *Server) handleChapterMarkdown(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, raw) fmt.Fprint(w, raw)
} }
// handleChapterTextPreview handles GET /api/chapter-text-preview/{slug}/{n}.
//
// Fetches a chapter live from novelfire.net and returns its plain text without
// writing anything to PocketBase or MinIO. This is the preview path used when
// a chapter has not yet been scraped into the library.
//
// Optional query params:
//
// chapter_url — the canonical chapter URL (preferred over constructing one)
// title — hint for the chapter title (used when the page title is empty)
//
// Response: {"slug":string,"number":int,"title":string,"text":string,"url":string}
func (s *Server) handleChapterTextPreview(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
n, err := strconv.Atoi(r.PathValue("n"))
if err != nil || n < 1 || slug == "" {
jsonError(w, http.StatusBadRequest, "invalid slug or chapter number")
return
}
// Determine the chapter URL to fetch.
chapterURL := r.URL.Query().Get("chapter_url")
if chapterURL == "" {
// Best-effort: novelfire chapter URLs follow /book/{slug}/chapter-{n}
chapterURL = fmt.Sprintf("%s/book/%s/chapter-%d", novelFireBase, slug, n)
}
titleHint := r.URL.Query().Get("title")
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
defer cancel()
// Fetch the chapter page.
req, err := http.NewRequestWithContext(ctx, http.MethodGet, chapterURL, nil)
if err != nil {
s.deps.Log.Error("chapter-text-preview: build request failed", "url", chapterURL, "err", err)
jsonError(w, http.StatusInternalServerError, "failed to build request")
return
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; libnovel-backend/2)")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
resp, err := http.DefaultClient.Do(req)
if err != nil {
s.deps.Log.Warn("chapter-text-preview: fetch failed", "url", chapterURL, "err", err)
jsonError(w, http.StatusBadGateway, "failed to fetch chapter")
return
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound {
jsonError(w, http.StatusNotFound, "chapter not found")
return
}
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
s.deps.Log.Warn("chapter-text-preview: upstream error",
"url", chapterURL, "status", resp.StatusCode, "body_snippet", string(body))
jsonError(w, http.StatusBadGateway, fmt.Sprintf("upstream returned %d", resp.StatusCode))
return
}
bodyBytes, err := io.ReadAll(resp.Body)
if err != nil {
s.deps.Log.Error("chapter-text-preview: read body failed", "err", err)
jsonError(w, http.StatusInternalServerError, "failed to read response")
return
}
// Parse HTML and extract the #content node.
root, err := htmlutil.ParseHTML(string(bodyBytes))
if err != nil {
s.deps.Log.Error("chapter-text-preview: html parse failed", "err", err)
jsonError(w, http.StatusInternalServerError, "failed to parse chapter HTML")
return
}
container := htmlutil.FindFirst(root, scraper.Selector{ID: "content"})
if container == nil {
s.deps.Log.Warn("chapter-text-preview: #content not found", "url", chapterURL)
jsonError(w, http.StatusNotFound, "chapter content not found on page")
return
}
markdownText := htmlutil.NodeToMarkdown(container)
plainText := stripMarkdown(markdownText)
// Extract the chapter title from the page <title> or <h1> if not hinted.
chapterTitle := titleHint
if chapterTitle == "" {
// Try <h1 class="chapter-title"> first, then <h2 class="chapter-title">
for _, tag := range []string{"h1", "h2", "h3"} {
if node := htmlutil.FindFirst(root, scraper.Selector{Tag: tag, Class: "chapter-title"}); node != nil {
chapterTitle = strings.TrimSpace(htmlutil.TextContent(node))
break
}
}
}
if chapterTitle == "" {
chapterTitle = fmt.Sprintf("Chapter %d", n)
}
writeJSON(w, 0, map[string]any{
"slug": slug,
"number": n,
"title": chapterTitle,
"text": plainText,
"url": chapterURL,
})
}
// handleReindex handles POST /api/reindex/{slug}. // handleReindex handles POST /api/reindex/{slug}.
// Rebuilds the chapters_idx PocketBase collection for a book from MinIO objects. // Rebuilds the chapters_idx PocketBase collection for a book from MinIO objects.
func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) { func (s *Server) handleReindex(w http.ResponseWriter, r *http.Request) {
@@ -741,6 +855,59 @@ func (s *Server) handlePresignVoiceSample(w http.ResponseWriter, r *http.Request
writeJSON(w, 0, map[string]string{"url": u}) writeJSON(w, 0, map[string]string{"url": u})
} }
// handleAvatarUpload handles PUT /api/avatar-upload/{userId}.
// The request body must be the raw image bytes; Content-Type must be
// image/jpeg, image/png, or image/webp.
//
// This endpoint is called by the SvelteKit server (not the browser directly),
// so MinIO credentials and internal networking are not a concern.
//
// Returns: { "key": "<objectKey>" }
func (s *Server) handleAvatarUpload(w http.ResponseWriter, r *http.Request) {
userID := r.PathValue("userId")
if userID == "" {
jsonError(w, http.StatusBadRequest, "missing userId")
return
}
ct := r.Header.Get("Content-Type")
var ext string
switch {
case strings.HasPrefix(ct, "image/jpeg"):
ext = "jpg"
case strings.HasPrefix(ct, "image/png"):
ext = "png"
case strings.HasPrefix(ct, "image/webp"):
ext = "webp"
default:
jsonError(w, http.StatusBadRequest, "unsupported content-type; use image/jpeg, image/png, or image/webp")
return
}
const maxSize = 5 << 20 // 5 MiB
data, err := io.ReadAll(io.LimitReader(r.Body, maxSize+1))
if err != nil {
jsonError(w, http.StatusBadRequest, "failed to read body")
return
}
if len(data) > maxSize {
jsonError(w, http.StatusRequestEntityTooLarge, "image too large (max 5 MiB)")
return
}
if len(data) == 0 {
jsonError(w, http.StatusBadRequest, "empty body")
return
}
key, err := s.deps.PresignStore.PutAvatar(r.Context(), userID, ext, ct, data)
if err != nil {
s.deps.Log.Error("avatar upload failed", "userId", userID, "err", err)
jsonError(w, http.StatusInternalServerError, "upload failed")
return
}
writeJSON(w, 0, map[string]string{"key": key})
}
// handlePresignAvatarUpload handles GET /api/presign/avatar-upload/{userId}. // handlePresignAvatarUpload handles GET /api/presign/avatar-upload/{userId}.
// Query params: ext (jpg|png|webp, defaults to jpg) // Query params: ext (jpg|png|webp, defaults to jpg)
func (s *Server) handlePresignAvatarUpload(w http.ResponseWriter, r *http.Request) { func (s *Server) handlePresignAvatarUpload(w http.ResponseWriter, r *http.Request) {
@@ -912,7 +1079,7 @@ func (s *Server) handleCatalogue(w http.ResponseWriter, r *http.Request) {
Limit: limit, Limit: limit,
} }
books, total, err := s.deps.SearchIndex.Catalogue(r.Context(), cq) books, total, facets, err := s.deps.SearchIndex.Catalogue(r.Context(), cq)
if err != nil { if err != nil {
s.deps.Log.Error("handleCatalogue: Catalogue query failed", "err", err) s.deps.Log.Error("handleCatalogue: Catalogue query failed", "err", err)
jsonError(w, http.StatusInternalServerError, "search failed") jsonError(w, http.StatusInternalServerError, "search failed")
@@ -928,6 +1095,10 @@ func (s *Server) handleCatalogue(w http.ResponseWriter, r *http.Request) {
"limit": limit, "limit": limit,
"total": total, "total": total,
"has_next": hasNext, "has_next": hasNext,
"facets": map[string]any{
"genres": facets.Genres,
"statuses": facets.Statuses,
},
}) })
} }

View File

@@ -144,6 +144,10 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
// Use this instead of presign+fetch to avoid SvelteKit→MinIO network path. // Use this instead of presign+fetch to avoid SvelteKit→MinIO network path.
mux.HandleFunc("GET /api/chapter-markdown/{slug}/{n}", s.handleChapterMarkdown) mux.HandleFunc("GET /api/chapter-markdown/{slug}/{n}", s.handleChapterMarkdown)
// Chapter text preview — live scrape from novelfire.net, no store writes.
// Used when the chapter is not yet in the library (preview mode).
mux.HandleFunc("GET /api/chapter-text-preview/{slug}/{n}", s.handleChapterTextPreview)
// Reindex chapters_idx from MinIO // Reindex chapters_idx from MinIO
mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex) mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex)
@@ -161,6 +165,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
mux.HandleFunc("GET /api/presign/voice-sample/{voice}", s.handlePresignVoiceSample) mux.HandleFunc("GET /api/presign/voice-sample/{voice}", s.handlePresignVoiceSample)
mux.HandleFunc("GET /api/presign/avatar-upload/{userId}", s.handlePresignAvatarUpload) mux.HandleFunc("GET /api/presign/avatar-upload/{userId}", s.handlePresignAvatarUpload)
mux.HandleFunc("GET /api/presign/avatar/{userId}", s.handlePresignAvatar) mux.HandleFunc("GET /api/presign/avatar/{userId}", s.handlePresignAvatar)
mux.HandleFunc("PUT /api/avatar-upload/{userId}", s.handleAvatarUpload)
// Reading progress // Reading progress
mux.HandleFunc("GET /api/progress", s.handleGetProgress) mux.HandleFunc("GET /api/progress", s.handleGetProgress)

View File

@@ -105,6 +105,10 @@ type PresignStore interface {
// Returns ("", false, nil) when no avatar exists. // Returns ("", false, nil) when no avatar exists.
PresignAvatarURL(ctx context.Context, userID string) (string, bool, error) PresignAvatarURL(ctx context.Context, userID string) (string, bool, error)
// PutAvatar stores raw image bytes for a user avatar directly in MinIO.
// ext should be "jpg", "png", or "webp". Returns the object key.
PutAvatar(ctx context.Context, userID, ext, contentType string, data []byte) (key string, err error)
// DeleteAvatar removes all avatar objects for a user. // DeleteAvatar removes all avatar objects for a user.
DeleteAvatar(ctx context.Context, userID string) error DeleteAvatar(ctx context.Context, userID string) error
} }

View File

@@ -68,6 +68,9 @@ func (m *mockStore) PresignAvatarUpload(_ context.Context, _, _ string) (string,
func (m *mockStore) PresignAvatarURL(_ context.Context, _ string) (string, bool, error) { func (m *mockStore) PresignAvatarURL(_ context.Context, _ string) (string, bool, error) {
return "", false, nil return "", false, nil
} }
func (m *mockStore) PutAvatar(_ context.Context, _, _, _ string, _ []byte) (string, error) {
return "", nil
}
func (m *mockStore) DeleteAvatar(_ context.Context, _ string) error { return nil } func (m *mockStore) DeleteAvatar(_ context.Context, _ string) error { return nil }
// ProgressStore // ProgressStore

View File

@@ -101,6 +101,11 @@ type Runner struct {
// scrapes per-book metadata, downloads covers, and re-indexes in Meilisearch. // scrapes per-book metadata, downloads covers, and re-indexes in Meilisearch.
// Defaults to 24h. Set to 0 to use the default. // Defaults to 24h. Set to 0 to use the default.
CatalogueRefreshInterval time.Duration CatalogueRefreshInterval time.Duration
// SkipInitialCatalogueRefresh prevents the runner from running a full
// catalogue walk on startup. Useful for quick restarts where the catalogue
// is already indexed and a 24h walk would be wasteful.
// Controlled by RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true.
SkipInitialCatalogueRefresh bool
} }
// Config is the top-level configuration struct consumed by both binaries. // Config is the top-level configuration struct consumed by both binaries.
@@ -142,7 +147,7 @@ func Load() Config {
PublicUseSSL: envBool("MINIO_PUBLIC_USE_SSL", true), PublicUseSSL: envBool("MINIO_PUBLIC_USE_SSL", true),
BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"), BucketChapters: envOr("MINIO_BUCKET_CHAPTERS", "libnovel-chapters"),
BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"), BucketAudio: envOr("MINIO_BUCKET_AUDIO", "libnovel-audio"),
BucketAvatars: envOr("MINIO_BUCKET_AVATARS", "libnovel-avatars"), BucketAvatars: envOr("MINIO_BUCKET_AVATARS", "avatars"),
BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"), BucketBrowse: envOr("MINIO_BUCKET_BROWSE", "libnovel-browse"),
}, },
@@ -156,14 +161,15 @@ func Load() Config {
}, },
Runner: Runner{ Runner: Runner{
PollInterval: envDuration("RUNNER_POLL_INTERVAL", 30*time.Second), PollInterval: envDuration("RUNNER_POLL_INTERVAL", 30*time.Second),
MaxConcurrentScrape: envInt("RUNNER_MAX_CONCURRENT_SCRAPE", 1), MaxConcurrentScrape: envInt("RUNNER_MAX_CONCURRENT_SCRAPE", 1),
MaxConcurrentAudio: envInt("RUNNER_MAX_CONCURRENT_AUDIO", 1), MaxConcurrentAudio: envInt("RUNNER_MAX_CONCURRENT_AUDIO", 1),
WorkerID: envOr("RUNNER_WORKER_ID", workerID), WorkerID: envOr("RUNNER_WORKER_ID", workerID),
Workers: envInt("RUNNER_WORKERS", 0), // 0 → runtime.NumCPU() Workers: envInt("RUNNER_WORKERS", 0), // 0 → runtime.NumCPU()
Timeout: envDuration("RUNNER_TIMEOUT", 90*time.Second), Timeout: envDuration("RUNNER_TIMEOUT", 90*time.Second),
MetricsAddr: envOr("RUNNER_METRICS_ADDR", ":9091"), MetricsAddr: envOr("RUNNER_METRICS_ADDR", ":9091"),
CatalogueRefreshInterval: envDuration("RUNNER_CATALOGUE_REFRESH_INTERVAL", 0), CatalogueRefreshInterval: envDuration("RUNNER_CATALOGUE_REFRESH_INTERVAL", 0),
SkipInitialCatalogueRefresh: envBool("RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH", false),
}, },
Meilisearch: Meilisearch{ Meilisearch: Meilisearch{

View File

@@ -20,6 +20,10 @@ type BookMeta struct {
SourceURL string `json:"source_url"` SourceURL string `json:"source_url"`
Ranking int `json:"ranking,omitempty"` Ranking int `json:"ranking,omitempty"`
Rating float64 `json:"rating,omitempty"` Rating float64 `json:"rating,omitempty"`
// MetaUpdated is the Unix timestamp (seconds) when the book record was last
// updated in PocketBase. Populated on read; not sent on write (PocketBase
// manages its own updated field).
MetaUpdated int64 `json:"meta_updated,omitempty"`
} }
// CatalogueEntry is a lightweight book reference returned by catalogue pages. // CatalogueEntry is a lightweight book reference returned by catalogue pages.

View File

@@ -6,7 +6,7 @@
// - Primary key: "slug" // - Primary key: "slug"
// - Searchable attributes: title, author, genres, summary // - Searchable attributes: title, author, genres, summary
// - Filterable attributes: status, genres // - Filterable attributes: status, genres
// - Sortable attributes: rank, rating, total_chapters // - Sortable attributes: rank, rating, total_chapters, meta_updated
// //
// The client is intentionally simple: UpsertBook and Search only. All // The client is intentionally simple: UpsertBook and Search only. All
// Meilisearch-specific details (index management, attribute configuration) // Meilisearch-specific details (index management, attribute configuration)
@@ -32,8 +32,9 @@ type Client interface {
// Search returns up to limit books matching query. // Search returns up to limit books matching query.
Search(ctx context.Context, query string, limit int) ([]domain.BookMeta, error) Search(ctx context.Context, query string, limit int) ([]domain.BookMeta, error)
// Catalogue queries books with optional filters, sort, and pagination. // Catalogue queries books with optional filters, sort, and pagination.
// Returns books and the total hit count for pagination. // Returns books, the total hit count for pagination, and a FacetResult
Catalogue(ctx context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, error) // with available genre and status values from the index.
Catalogue(ctx context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, FacetResult, error)
} }
// CatalogueQuery holds parameters for the /api/catalogue endpoint. // CatalogueQuery holds parameters for the /api/catalogue endpoint.
@@ -41,11 +42,18 @@ type CatalogueQuery struct {
Q string // full-text query (may be empty for browse) Q string // full-text query (may be empty for browse)
Genre string // genre filter, e.g. "fantasy" or "all" Genre string // genre filter, e.g. "fantasy" or "all"
Status string // status filter, e.g. "ongoing", "completed", or "all" Status string // status filter, e.g. "ongoing", "completed", or "all"
Sort string // sort field: "popular", "new", "top-rated", "rank", "" Sort string // sort field: "popular", "new", "update", "top-rated", "rank", ""
Page int // 1-indexed Page int // 1-indexed
Limit int // items per page, default 20 Limit int // items per page, default 20
} }
// FacetResult holds the available filter values discovered from the index.
// Values are sorted alphabetically and include only those present in the index.
type FacetResult struct {
Genres []string // distinct genre values
Statuses []string // distinct status values
}
// MeiliClient wraps the meilisearch-go SDK. // MeiliClient wraps the meilisearch-go SDK.
type MeiliClient struct { type MeiliClient struct {
idx meilisearch.IndexManager idx meilisearch.IndexManager
@@ -93,7 +101,7 @@ func Configure(host, apiKey string) error {
return fmt.Errorf("meili: update filterable attributes: %w", err) return fmt.Errorf("meili: update filterable attributes: %w", err)
} }
sortable := []string{"rank", "rating", "total_chapters"} sortable := []string{"rank", "rating", "total_chapters", "meta_updated"}
if _, err := idx.UpdateSortableAttributes(&sortable); err != nil { if _, err := idx.UpdateSortableAttributes(&sortable); err != nil {
return fmt.Errorf("meili: update sortable attributes: %w", err) return fmt.Errorf("meili: update sortable attributes: %w", err)
} }
@@ -114,6 +122,9 @@ type bookDoc struct {
SourceURL string `json:"source_url"` SourceURL string `json:"source_url"`
Rank int `json:"rank"` Rank int `json:"rank"`
Rating float64 `json:"rating"` Rating float64 `json:"rating"`
// MetaUpdated is the Unix timestamp (seconds) of the last PocketBase update.
// Used for sort=update ("recently updated" ordering).
MetaUpdated int64 `json:"meta_updated"`
} }
func toDoc(b domain.BookMeta) bookDoc { func toDoc(b domain.BookMeta) bookDoc {
@@ -129,6 +140,7 @@ func toDoc(b domain.BookMeta) bookDoc {
SourceURL: b.SourceURL, SourceURL: b.SourceURL,
Rank: b.Ranking, Rank: b.Ranking,
Rating: b.Rating, Rating: b.Rating,
MetaUpdated: b.MetaUpdated,
} }
} }
@@ -145,6 +157,7 @@ func fromDoc(d bookDoc) domain.BookMeta {
SourceURL: d.SourceURL, SourceURL: d.SourceURL,
Ranking: d.Rank, Ranking: d.Rank,
Rating: d.Rating, Rating: d.Rating,
MetaUpdated: d.MetaUpdated,
} }
} }
@@ -188,8 +201,9 @@ func (c *MeiliClient) Search(_ context.Context, query string, limit int) ([]doma
} }
// Catalogue queries books with optional full-text search, genre/status filters, // Catalogue queries books with optional full-text search, genre/status filters,
// sort order, and pagination. Returns matching books and the total estimate. // sort order, and pagination. Returns matching books, the total estimate, and
func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, error) { // a FacetResult containing available genre and status values from the index.
func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.BookMeta, int64, FacetResult, error) {
if q.Limit <= 0 { if q.Limit <= 0 {
q.Limit = 20 q.Limit = 20
} }
@@ -200,6 +214,9 @@ func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.B
req := &meilisearch.SearchRequest{ req := &meilisearch.SearchRequest{
Limit: int64(q.Limit), Limit: int64(q.Limit),
Offset: int64((q.Page - 1) * q.Limit), Offset: int64((q.Page - 1) * q.Limit),
// Request facet distribution so the UI can build filter options
// dynamically without hardcoding genre/status lists.
Facets: []string{"genres", "status"},
} }
// Build filter // Build filter
@@ -214,7 +231,7 @@ func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.B
req.Filter = strings.Join(filters, " AND ") req.Filter = strings.Join(filters, " AND ")
} }
// Map UI sort tokens to Meilisearch sort expressions // Map UI sort tokens to Meilisearch sort expressions.
switch q.Sort { switch q.Sort {
case "rank": case "rank":
req.Sort = []string{"rank:asc"} req.Sort = []string{"rank:asc"}
@@ -222,12 +239,14 @@ func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.B
req.Sort = []string{"rating:desc"} req.Sort = []string{"rating:desc"}
case "new": case "new":
req.Sort = []string{"total_chapters:desc"} req.Sort = []string{"total_chapters:desc"}
case "update":
req.Sort = []string{"meta_updated:desc"}
// "popular" and "" → relevance (no explicit sort) // "popular" and "" → relevance (no explicit sort)
} }
res, err := c.idx.Search(q.Q, req) res, err := c.idx.Search(q.Q, req)
if err != nil { if err != nil {
return nil, 0, fmt.Errorf("meili: catalogue query: %w", err) return nil, 0, FacetResult{}, fmt.Errorf("meili: catalogue query: %w", err)
} }
books := make([]domain.BookMeta, 0, len(res.Hits)) books := make([]domain.BookMeta, 0, len(res.Hits))
@@ -242,7 +261,45 @@ func (c *MeiliClient) Catalogue(_ context.Context, q CatalogueQuery) ([]domain.B
} }
books = append(books, fromDoc(doc)) books = append(books, fromDoc(doc))
} }
return books, res.EstimatedTotalHits, nil
facets := parseFacets(res.FacetDistribution)
return books, res.EstimatedTotalHits, facets, nil
}
// parseFacets extracts sorted genre and status slices from a Meilisearch
// facetDistribution raw JSON value.
// The JSON shape is: {"genres":{"fantasy":12,"action":5},"status":{"ongoing":7}}
func parseFacets(raw json.RawMessage) FacetResult {
var result FacetResult
if len(raw) == 0 {
return result
}
var dist map[string]map[string]int64
if err := json.Unmarshal(raw, &dist); err != nil {
return result
}
if m, ok := dist["genres"]; ok {
for k := range m {
result.Genres = append(result.Genres, k)
}
sortStrings(result.Genres)
}
if m, ok := dist["status"]; ok {
for k := range m {
result.Statuses = append(result.Statuses, k)
}
sortStrings(result.Statuses)
}
return result
}
// sortStrings sorts a slice of strings in place.
func sortStrings(s []string) {
for i := 1; i < len(s); i++ {
for j := i; j > 0 && s[j] < s[j-1]; j-- {
s[j], s[j-1] = s[j-1], s[j]
}
}
} }
// NoopClient is a no-op Client used when Meilisearch is not configured. // NoopClient is a no-op Client used when Meilisearch is not configured.
@@ -252,6 +309,6 @@ func (NoopClient) UpsertBook(_ context.Context, _ domain.BookMeta) error { retur
func (NoopClient) Search(_ context.Context, _ string, _ int) ([]domain.BookMeta, error) { func (NoopClient) Search(_ context.Context, _ string, _ int) ([]domain.BookMeta, error) {
return nil, nil return nil, nil
} }
func (NoopClient) Catalogue(_ context.Context, _ CatalogueQuery) ([]domain.BookMeta, int64, error) { func (NoopClient) Catalogue(_ context.Context, _ CatalogueQuery) ([]domain.BookMeta, int64, FacetResult, error) {
return nil, 0, nil return nil, 0, FacetResult{}, nil
} }

View File

@@ -194,8 +194,12 @@ func (s *Scraper) ScrapeMetadata(ctx context.Context, bookURL string) (domain.Bo
// ── ChapterListProvider ─────────────────────────────────────────────────────── // ── ChapterListProvider ───────────────────────────────────────────────────────
// ScrapeChapterList returns all chapter references for a book, ordered ascending. // ScrapeChapterList returns chapter references for a book, ordered ascending.
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]domain.ChapterRef, error) { // upTo > 0 stops pagination as soon as at least upTo chapter numbers have been
// collected — use this for range scrapes so we don't paginate 100 pages just
// to discover refs we'll never scrape. upTo == 0 fetches all pages.
// Each page fetch uses retryGet with 429-aware exponential backoff.
func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string, upTo int) ([]domain.ChapterRef, error) {
var refs []domain.ChapterRef var refs []domain.ChapterRef
baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters" baseChapterURL := strings.TrimRight(bookURL, "/") + "/chapters"
page := 1 page := 1
@@ -210,7 +214,7 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]doma
pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page) pageURL := fmt.Sprintf("%s?page=%d", baseChapterURL, page)
s.log.Info("scraping chapter list", "page", page, "url", pageURL) s.log.Info("scraping chapter list", "page", page, "url", pageURL)
raw, err := s.client.GetContent(ctx, pageURL) raw, err := retryGet(ctx, s.log, s.client, pageURL, 9, 6*time.Second)
if err != nil { if err != nil {
return refs, fmt.Errorf("chapter list page %d: %w", page, err) return refs, fmt.Errorf("chapter list page %d: %w", page, err)
} }
@@ -255,6 +259,13 @@ func (s *Scraper) ScrapeChapterList(ctx context.Context, bookURL string) ([]doma
}) })
} }
// Early-stop: if we have seen at least upTo chapter numbers, we have
// enough refs to cover the requested range — no need to paginate further.
if upTo > 0 && len(refs) > 0 && refs[len(refs)-1].Number >= upTo {
s.log.Debug("chapter list early-stop reached", "upTo", upTo, "collected", len(refs))
break
}
page++ page++
} }

View File

@@ -106,7 +106,7 @@ func (o *Orchestrator) RunBook(ctx context.Context, task domain.ScrapeTask) doma
o.log.Info("metadata saved", "slug", meta.Slug, "title", meta.Title) o.log.Info("metadata saved", "slug", meta.Slug, "title", meta.Title)
// ── Step 2: Chapter list ────────────────────────────────────────────────── // ── Step 2: Chapter list ──────────────────────────────────────────────────
refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL) refs, err := o.novel.ScrapeChapterList(ctx, task.TargetURL, task.ToChapter)
if err != nil { if err != nil {
o.log.Error("chapter list scrape failed", "slug", meta.Slug, "err", err) o.log.Error("chapter list scrape failed", "slug", meta.Slug, "err", err)
result.ErrorMessage = fmt.Sprintf("chapter list: %v", err) result.ErrorMessage = fmt.Sprintf("chapter list: %v", err)

View File

@@ -34,7 +34,7 @@ func (s *stubScraper) ScrapeMetadata(_ context.Context, _ string) (domain.BookMe
return s.meta, s.metaErr return s.meta, s.metaErr
} }
func (s *stubScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) { func (s *stubScraper) ScrapeChapterList(_ context.Context, _ string, _ int) ([]domain.ChapterRef, error) {
return s.refs, s.refsErr return s.refs, s.refsErr
} }

View File

@@ -17,6 +17,7 @@ import (
"context" "context"
"fmt" "fmt"
"log/slog" "log/slog"
"os"
"sync" "sync"
"sync/atomic" "sync/atomic"
"time" "time"
@@ -55,6 +56,11 @@ type Config struct {
// scrapes per-book metadata, downloads covers, and re-indexes everything in // scrapes per-book metadata, downloads covers, and re-indexes everything in
// Meilisearch. Defaults to 24h (expensive — full catalogue walk). // Meilisearch. Defaults to 24h (expensive — full catalogue walk).
CatalogueRefreshInterval time.Duration CatalogueRefreshInterval time.Duration
// SkipInitialCatalogueRefresh suppresses the immediate catalogue walk that
// otherwise fires at startup. The periodic ticker (CatalogueRefreshInterval)
// still fires normally. Set RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true for
// quick restarts where the catalogue is already up to date.
SkipInitialCatalogueRefresh bool
// MetricsAddr is the HTTP listen address for the /metrics endpoint. // MetricsAddr is the HTTP listen address for the /metrics endpoint.
// Defaults to ":9091". Set to "" to disable. // Defaults to ":9091". Set to "" to disable.
MetricsAddr string MetricsAddr string
@@ -175,8 +181,12 @@ func (r *Runner) Run(ctx context.Context) error {
// Run one browse refresh immediately on startup. // Run one browse refresh immediately on startup.
go r.runBrowseRefresh(ctx) go r.runBrowseRefresh(ctx)
// Run one catalogue refresh immediately on startup. // Run one catalogue refresh immediately on startup (unless skipped by flag).
go r.runCatalogueRefresh(ctx) if !r.cfg.SkipInitialCatalogueRefresh {
go r.runCatalogueRefresh(ctx)
} else {
r.deps.Log.Info("runner: skipping initial catalogue refresh (RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH=true)")
}
// Run one poll immediately on startup, then on each tick. // Run one poll immediately on startup, then on each tick.
for { for {
@@ -208,6 +218,15 @@ func (r *Runner) Run(ctx context.Context) error {
// poll claims all available pending tasks and dispatches them to goroutines. // poll claims all available pending tasks and dispatches them to goroutines.
func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg *sync.WaitGroup) { func (r *Runner) poll(ctx context.Context, scrapeSem, audioSem chan struct{}, wg *sync.WaitGroup) {
// ── Heartbeat file ────────────────────────────────────────────────────
// Touch /tmp/runner.alive so the Docker health check can confirm the
// runner is actively polling. Failure is non-fatal — just log it.
if f, err := os.Create("/tmp/runner.alive"); err != nil {
r.deps.Log.Warn("runner: could not write heartbeat file", "err", err)
} else {
f.Close()
}
// ── Reap orphaned tasks ─────────────────────────────────────────────── // ── Reap orphaned tasks ───────────────────────────────────────────────
if n, err := r.deps.Consumer.ReapStaleTasks(ctx, r.cfg.StaleTaskThreshold); err != nil { if n, err := r.deps.Consumer.ReapStaleTasks(ctx, r.cfg.StaleTaskThreshold); err != nil {
r.deps.Log.Warn("runner: reap stale tasks failed", "err", err) r.deps.Log.Warn("runner: reap stale tasks failed", "err", err)

View File

@@ -146,7 +146,7 @@ func (s *stubNovelScraper) ScrapeMetadata(_ context.Context, _ string) (domain.B
return domain.BookMeta{Slug: "test-book", Title: "Test Book", SourceURL: "https://example.com/book/test-book"}, nil return domain.BookMeta{Slug: "test-book", Title: "Test Book", SourceURL: "https://example.com/book/test-book"}, nil
} }
func (s *stubNovelScraper) ScrapeChapterList(_ context.Context, _ string) ([]domain.ChapterRef, error) { func (s *stubNovelScraper) ScrapeChapterList(_ context.Context, _ string, _ int) ([]domain.ChapterRef, error) {
return s.chapters, nil return s.chapters, nil
} }

View File

@@ -20,8 +20,10 @@ type MetadataProvider interface {
} }
// ChapterListProvider can enumerate all chapters of a book. // ChapterListProvider can enumerate all chapters of a book.
// upTo > 0 stops pagination once at least upTo chapter numbers have been
// collected (early-exit optimisation for range scrapes). upTo == 0 fetches all pages.
type ChapterListProvider interface { type ChapterListProvider interface {
ScrapeChapterList(ctx context.Context, bookURL string) ([]domain.ChapterRef, error) ScrapeChapterList(ctx context.Context, bookURL string, upTo int) ([]domain.ChapterRef, error)
} }
// ChapterTextProvider can extract the readable text from a single chapter page. // ChapterTextProvider can extract the readable text from a single chapter page.

View File

@@ -145,6 +145,10 @@ type pbBook struct {
} }
func (b pbBook) toDomain() domain.BookMeta { func (b pbBook) toDomain() domain.BookMeta {
var metaUpdated int64
if t, err := time.Parse(time.RFC3339, b.Updated); err == nil {
metaUpdated = t.Unix()
}
return domain.BookMeta{ return domain.BookMeta{
Slug: b.Slug, Slug: b.Slug,
Title: b.Title, Title: b.Title,
@@ -157,6 +161,7 @@ func (b pbBook) toDomain() domain.BookMeta {
SourceURL: b.SourceURL, SourceURL: b.SourceURL,
Ranking: b.Ranking, Ranking: b.Ranking,
Rating: b.Rating, Rating: b.Rating,
MetaUpdated: metaUpdated,
} }
} }
@@ -405,6 +410,17 @@ func (s *Store) PresignAvatarURL(ctx context.Context, userID string) (string, bo
return "", false, nil return "", false, nil
} }
func (s *Store) PutAvatar(ctx context.Context, userID, ext, contentType string, data []byte) (string, error) {
// Delete existing avatar objects for this user before writing the new one
// so old extensions don't linger (e.g. old .png after uploading a .jpg).
_ = s.mc.deleteObjects(ctx, s.mc.bucketAvatars, userID+"/")
key := AvatarObjectKey(userID, ext)
if err := s.mc.putObject(ctx, s.mc.bucketAvatars, key, contentType, data); err != nil {
return "", fmt.Errorf("put avatar: %w", err)
}
return key, nil
}
func (s *Store) DeleteAvatar(ctx context.Context, userID string) error { func (s *Store) DeleteAvatar(ctx context.Context, userID string) error {
return s.mc.deleteObjects(ctx, s.mc.bucketAvatars, userID+"/") return s.mc.deleteObjects(ctx, s.mc.bucketAvatars, userID+"/")
} }

7
v3/caddy/Dockerfile Normal file
View File

@@ -0,0 +1,7 @@
FROM caddy:2-builder AS builder
RUN xcaddy build \
--with github.com/mholt/caddy-ratelimit
FROM caddy:2-alpine
COPY --from=builder /usr/bin/caddy /usr/bin/caddy

51
v3/caddy/errors/502.html Normal file
View File

@@ -0,0 +1,51 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>502 — Service Unavailable</title>
<style>
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
body {
min-height: 100svh;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
gap: 1rem;
background: #09090b;
color: #a1a1aa;
font-family: ui-sans-serif, system-ui, sans-serif;
padding: 2rem;
text-align: center;
}
.code {
font-size: clamp(4rem, 20vw, 8rem);
font-weight: 800;
color: #27272a;
line-height: 1;
letter-spacing: -0.04em;
}
h1 { font-size: 1.25rem; font-weight: 600; color: #e4e4e7; }
p { font-size: 0.9rem; max-width: 36ch; line-height: 1.6; }
a {
margin-top: 0.5rem;
display: inline-block;
padding: 0.6rem 1.4rem;
border-radius: 0.5rem;
background: #f59e0b;
color: #000;
font-weight: 600;
font-size: 0.875rem;
text-decoration: none;
}
a:hover { background: #d97706; }
</style>
</head>
<body>
<div class="code">502</div>
<h1>Service Unavailable</h1>
<p>The server is temporarily unreachable. Please try again in a moment.</p>
<a href="/">Go home</a>
</body>
</html>

51
v3/caddy/errors/503.html Normal file
View File

@@ -0,0 +1,51 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>503 — Maintenance</title>
<style>
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
body {
min-height: 100svh;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
gap: 1rem;
background: #09090b;
color: #a1a1aa;
font-family: ui-sans-serif, system-ui, sans-serif;
padding: 2rem;
text-align: center;
}
.code {
font-size: clamp(4rem, 20vw, 8rem);
font-weight: 800;
color: #27272a;
line-height: 1;
letter-spacing: -0.04em;
}
h1 { font-size: 1.25rem; font-weight: 600; color: #e4e4e7; }
p { font-size: 0.9rem; max-width: 36ch; line-height: 1.6; }
a {
margin-top: 0.5rem;
display: inline-block;
padding: 0.6rem 1.4rem;
border-radius: 0.5rem;
background: #f59e0b;
color: #000;
font-weight: 600;
font-size: 0.875rem;
text-decoration: none;
}
a:hover { background: #d97706; }
</style>
</head>
<body>
<div class="code">503</div>
<h1>Under Maintenance</h1>
<p>LibNovel is briefly offline for maintenance. We&rsquo;ll be back shortly.</p>
<a href="/">Try again</a>
</body>
</html>

51
v3/caddy/errors/504.html Normal file
View File

@@ -0,0 +1,51 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>504 — Gateway Timeout</title>
<style>
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
body {
min-height: 100svh;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
gap: 1rem;
background: #09090b;
color: #a1a1aa;
font-family: ui-sans-serif, system-ui, sans-serif;
padding: 2rem;
text-align: center;
}
.code {
font-size: clamp(4rem, 20vw, 8rem);
font-weight: 800;
color: #27272a;
line-height: 1;
letter-spacing: -0.04em;
}
h1 { font-size: 1.25rem; font-weight: 600; color: #e4e4e7; }
p { font-size: 0.9rem; max-width: 36ch; line-height: 1.6; }
a {
margin-top: 0.5rem;
display: inline-block;
padding: 0.6rem 1.4rem;
border-radius: 0.5rem;
background: #f59e0b;
color: #000;
font-weight: 600;
font-size: 0.875rem;
text-decoration: none;
}
a:hover { background: #d97706; }
</style>
</head>
<body>
<div class="code">504</div>
<h1>Gateway Timeout</h1>
<p>The request took too long to complete. Please refresh and try again.</p>
<a href="/">Go home</a>
</body>
</html>

View File

@@ -8,8 +8,8 @@ x-infra-env: &infra-env
MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}" MINIO_ACCESS_KEY: "${MINIO_ROOT_USER:-admin}"
MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}" MINIO_SECRET_KEY: "${MINIO_ROOT_PASSWORD:-changeme123}"
MINIO_USE_SSL: "false" MINIO_USE_SSL: "false"
MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-}" MINIO_PUBLIC_ENDPOINT: "${MINIO_PUBLIC_ENDPOINT:-localhost}"
MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-false}" MINIO_PUBLIC_USE_SSL: "${MINIO_PUBLIC_USE_SSL:-true}"
# PocketBase # PocketBase
POCKETBASE_URL: "http://pocketbase:8090" POCKETBASE_URL: "http://pocketbase:8090"
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}" POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
@@ -53,7 +53,7 @@ services:
mc alias set local http://minio:9000 $${MINIO_ROOT_USER:-admin} $${MINIO_ROOT_PASSWORD:-changeme123}; mc alias set local http://minio:9000 $${MINIO_ROOT_USER:-admin} $${MINIO_ROOT_PASSWORD:-changeme123};
mc mb --ignore-existing local/libnovel-chapters; mc mb --ignore-existing local/libnovel-chapters;
mc mb --ignore-existing local/libnovel-audio; mc mb --ignore-existing local/libnovel-audio;
mc mb --ignore-existing local/libnovel-avatars; mc mb --ignore-existing local/avatars;
mc mb --ignore-existing local/libnovel-browse; mc mb --ignore-existing local/libnovel-browse;
echo 'buckets ready'; echo 'buckets ready';
" "
@@ -135,6 +135,8 @@ services:
args: args:
VERSION: "${GIT_TAG:-dev}" VERSION: "${GIT_TAG:-dev}"
COMMIT: "${GIT_COMMIT:-unknown}" COMMIT: "${GIT_COMMIT:-unknown}"
labels:
com.centurylinklabs.watchtower.enable: "true"
restart: unless-stopped restart: unless-stopped
stop_grace_period: 35s stop_grace_period: 35s
depends_on: depends_on:
@@ -170,6 +172,8 @@ services:
args: args:
VERSION: "${GIT_TAG:-dev}" VERSION: "${GIT_TAG:-dev}"
COMMIT: "${GIT_COMMIT:-unknown}" COMMIT: "${GIT_COMMIT:-unknown}"
labels:
com.centurylinklabs.watchtower.enable: "true"
restart: unless-stopped restart: unless-stopped
stop_grace_period: 135s stop_grace_period: 135s
depends_on: depends_on:
@@ -215,6 +219,8 @@ services:
args: args:
BUILD_VERSION: "${GIT_TAG:-dev}" BUILD_VERSION: "${GIT_TAG:-dev}"
BUILD_COMMIT: "${GIT_COMMIT:-unknown}" BUILD_COMMIT: "${GIT_COMMIT:-unknown}"
labels:
com.centurylinklabs.watchtower.enable: "true"
restart: unless-stopped restart: unless-stopped
stop_grace_period: 35s stop_grace_period: 35s
depends_on: depends_on:
@@ -238,7 +244,7 @@ services:
POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}" POCKETBASE_ADMIN_EMAIL: "${POCKETBASE_ADMIN_EMAIL:-admin@libnovel.local}"
POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}" POCKETBASE_ADMIN_PASSWORD: "${POCKETBASE_ADMIN_PASSWORD:-changeme123}"
AUTH_SECRET: "${AUTH_SECRET:-dev_secret_change_in_production}" AUTH_SECRET: "${AUTH_SECRET:-dev_secret_change_in_production}"
PUBLIC_MINIO_PUBLIC_URL: "${MINIO_PUBLIC_ENDPOINT:-http://localhost:9000}" PUBLIC_MINIO_PUBLIC_URL: "${MINIO_PUBLIC_ENDPOINT:-https://localhost}"
# Valkey # Valkey
VALKEY_ADDR: "valkey:6379" VALKEY_ADDR: "valkey:6379"
healthcheck: healthcheck:
@@ -248,8 +254,11 @@ services:
retries: 3 retries: 3
# ─── Caddy (reverse proxy + automatic HTTPS) ────────────────────────────────── # ─── Caddy (reverse proxy + automatic HTTPS) ──────────────────────────────────
# Custom build includes github.com/mholt/caddy-ratelimit.
caddy: caddy:
image: caddy:2-alpine build:
context: ./caddy
dockerfile: Dockerfile
restart: unless-stopped restart: unless-stopped
depends_on: depends_on:
backend: backend:
@@ -265,9 +274,24 @@ services:
CADDY_ACME_EMAIL: "${CADDY_ACME_EMAIL:-}" CADDY_ACME_EMAIL: "${CADDY_ACME_EMAIL:-}"
volumes: volumes:
- ./Caddyfile:/etc/caddy/Caddyfile:ro - ./Caddyfile:/etc/caddy/Caddyfile:ro
- ./caddy/errors:/srv/errors:ro
- caddy_data:/data - caddy_data:/data
- caddy_config:/config - caddy_config:/config
# ─── Watchtower (auto-redeploy custom services on new images) ────────────────
# Only watches services labelled com.centurylinklabs.watchtower.enable=true.
# Third-party infra images (minio, pocketbase, meilisearch, etc.) are excluded.
watchtower:
image: containrrr/watchtower:latest
restart: unless-stopped
volumes:
- /var/run/docker.sock:/var/run/docker.sock
command: --label-enable --interval 300 --cleanup
environment:
WATCHTOWER_NOTIFICATIONS: "${WATCHTOWER_NOTIFICATIONS:-}"
WATCHTOWER_NOTIFICATION_URL: "${WATCHTOWER_NOTIFICATION_URL:-}"
DOCKER_API_VERSION: "1.44"
volumes: volumes:
minio_data: minio_data:
pb_data: pb_data:

View File

@@ -1,8 +1,8 @@
# API Endpoint Reference # API Endpoint Reference
All endpoints served by the Go **backend** binary on `:8080`. In production all > **Routing ownership map**: see [`docs/d2/api-routing.svg`](d2/api-routing.svg) (source: [`docs/d2/api-routing.d2`](d2/api-routing.d2)) for a visual overview of which paths Caddy sends to the backend directly vs. through SvelteKit, with auth levels colour-coded.
traffic is routed through Caddy — `/api/*` and `/health` are proxied to the
backend; everything else goes to the SvelteKit UI. All traffic enters through **Caddy :443**. Caddy routes a subset of paths directly to the Go backend (bypassing SvelteKit); everything else goes to SvelteKit, which enforces auth before proxying onward.
## Health / Version ## Health / Version

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 51 KiB

201
v3/docs/d2/api-routing.d2 Normal file
View File

@@ -0,0 +1,201 @@
direction: right
# ─── Legend ───────────────────────────────────────────────────────────────────
legend: Legend {
style.fill: "#fafafa"
style.stroke: "#d4d4d8"
pub: public {
style.fill: "#f0fdf4"
style.font-color: "#15803d"
style.stroke: "#86efac"
}
user: user auth {
style.fill: "#eff6ff"
style.font-color: "#1d4ed8"
style.stroke: "#93c5fd"
}
adm: admin only {
style.fill: "#fff7ed"
style.font-color: "#c2410c"
style.stroke: "#fdba74"
}
}
# ─── Client ───────────────────────────────────────────────────────────────────
client: Browser / iOS App {
shape: person
style.fill: "#fff9e6"
}
# ─── Caddy ────────────────────────────────────────────────────────────────────
caddy: Caddy :443 {
shape: rectangle
style.fill: "#f1f5f9"
label: "Caddy :443\ncustom build · caddy-ratelimit\nsecurity headers · rate limiting\nstatic error pages"
}
# ─── SvelteKit UI ─────────────────────────────────────────────────────────────
# Handles: auth enforcement, session, all /api/* routes that have SK counterparts
sk: SvelteKit UI :3000 {
style.fill: "#fef3c7"
auth: Auth {
style.fill: "#fde68a"
style.stroke: "#f59e0b"
label: "POST /api/auth/login\nPOST /api/auth/register\nPOST /api/auth/change-password\nGET /api/auth/session"
}
catalogue_sk: Catalogue {
style.fill: "#f0fdf4"
style.stroke: "#86efac"
label: "GET /api/catalogue-page\nGET /api/search"
}
book_sk: Book {
style.fill: "#f0fdf4"
style.stroke: "#86efac"
label: "GET /api/book/{slug}\nGET /api/chapter/{slug}/{n}\nGET /api/chapter-text-preview/{slug}/{n}"
}
scrape_sk: Scrape (admin) {
style.fill: "#fff7ed"
style.stroke: "#fdba74"
label: "GET /api/scrape/status\nGET /api/scrape/tasks\nPOST /api/scrape\nPOST /api/scrape/range\nPOST /api/scrape/cancel/{id}"
}
audio_sk: Audio {
style.fill: "#f0fdf4"
style.stroke: "#86efac"
label: "POST /api/audio/{slug}/{n}\nGET /api/audio/status/{slug}/{n}\nGET /api/voices"
}
presign_sk: Presigned URLs {
style.fill: "#f0fdf4"
style.stroke: "#86efac"
label: "GET /api/presign/chapter/{slug}/{n}\nGET /api/presign/audio/{slug}/{n}\nGET /api/presign/voice-sample/{voice}"
}
presign_user: Presigned URLs (user) {
style.fill: "#eff6ff"
style.stroke: "#93c5fd"
label: "GET /api/presign/avatar-upload/{userId}\nGET /api/presign/avatar/{userId}"
}
progress_sk: Progress {
style.fill: "#f0fdf4"
style.stroke: "#86efac"
label: "GET /api/progress\nPOST /api/progress/{slug}\nDELETE /api/progress/{slug}"
}
library_sk: Library {
style.fill: "#f0fdf4"
style.stroke: "#86efac"
label: "GET /api/library\nPOST /api/library/{slug}\nDELETE /api/library/{slug}"
}
comments_sk: Comments {
style.fill: "#f0fdf4"
style.stroke: "#86efac"
label: "GET /api/comments/{slug}\nPOST /api/comments/{slug}"
}
}
# ─── Go Backend ───────────────────────────────────────────────────────────────
# Caddy proxies these paths directly — no SvelteKit auth layer
be: Backend API :8080 {
style.fill: "#eef3ff"
health_be: Health {
style.fill: "#f0fdf4"
style.stroke: "#86efac"
label: "GET /health\nGET /api/version"
}
scrape_be: Scrape admin (direct) {
style.fill: "#fff7ed"
style.stroke: "#fdba74"
label: "POST /scrape\nPOST /scrape/book\nPOST /scrape/book/range"
}
catalogue_be: Catalogue {
style.fill: "#f0fdf4"
style.stroke: "#86efac"
label: "GET /api/browse\nGET /api/catalogue\nGET /api/ranking\nGET /api/cover/{domain}/{slug}"
}
book_be: Book / Chapter {
style.fill: "#f0fdf4"
style.stroke: "#86efac"
label: "GET /api/book-preview/{slug}\nGET /api/chapter-text/{slug}/{n}\nGET /api/chapter-markdown/{slug}/{n}\nPOST /api/reindex/{slug} ⚠ admin"
}
audio_be: Audio {
style.fill: "#f0fdf4"
style.stroke: "#86efac"
label: "GET /api/audio-proxy/{slug}/{n}\nGET /api/voices"
}
}
# ─── Storage ──────────────────────────────────────────────────────────────────
storage: Storage {
style.fill: "#eaf7ea"
pb: PocketBase :8090 {
shape: cylinder
label: "auth · books · progress\ncomments · library\nscrape_jobs · audio_cache"
}
mn: MinIO :9000 {
shape: cylinder
label: "chapters · audio\navatars · browse"
}
ms: Meilisearch :7700 {
shape: cylinder
label: "index: books"
}
vk: Valkey :6379 {
shape: cylinder
label: "presign URL cache"
}
}
# ─── Caddy routing ────────────────────────────────────────────────────────────
client -> caddy: HTTPS :443
caddy -> sk: "/* (catch-all)\n→ SvelteKit handles auth"
caddy -> be: "/health /scrape*\n/api/browse /api/book-preview/*\n/api/chapter-text/* /api/chapter-markdown/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/* /api/catalogue /api/ranking"
caddy -> storage.mn: "/avatars/*\n(presigned GETs)"
# ─── SvelteKit → Backend (server-side proxy) ──────────────────────────────────
sk.catalogue_sk -> be.catalogue_be: internal proxy
sk.book_sk -> be.book_be: internal proxy
sk.audio_sk -> be.audio_be: internal proxy
sk.presign_sk -> storage.vk: check cache
sk.presign_sk -> storage.mn: generate presign
sk.presign_user -> storage.mn: generate presign
# ─── SvelteKit → Storage (direct) ────────────────────────────────────────────
sk.auth -> storage.pb: sessions / users
sk.scrape_sk -> storage.pb: scrape job records
sk.progress_sk -> storage.pb
sk.library_sk -> storage.pb
sk.comments_sk -> storage.pb
# ─── Backend → Storage ────────────────────────────────────────────────────────
be.catalogue_be -> storage.ms: full-text search
be.catalogue_be -> storage.pb: ranking records
be.catalogue_be -> storage.mn: cover presign
be.book_be -> storage.mn: chapter objects
be.book_be -> storage.pb: book metadata
be.audio_be -> storage.mn: audio presign
be.audio_be -> storage.vk: presign cache

127
v3/docs/d2/api-routing.svg Normal file

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 57 KiB

View File

@@ -46,7 +46,7 @@ storage: Storage {
minio: MinIO { minio: MinIO {
shape: cylinder shape: cylinder
label: "MinIO :9000\n\nbuckets:\n libnovel-chapters\n libnovel-audio\n libnovel-avatars\n libnovel-browse" label: "MinIO :9000\n\nbuckets:\n libnovel-chapters\n libnovel-audio\n avatars\n libnovel-browse"
} }
pocketbase: PocketBase { pocketbase: PocketBase {
@@ -72,7 +72,7 @@ app: Application {
caddy: caddy { caddy: caddy {
shape: rectangle shape: rectangle
label: "Caddy :443 / :80\n(reverse proxy\nauto-HTTPS via Let's Encrypt)" label: "Caddy :443 / :80\ncustom build + caddy-ratelimit\n\nfeatures:\n auto-HTTPS (Let's Encrypt)\n security headers\n rate limiting (per-IP)\n static error pages (502/503/504)"
} }
backend: backend { backend: backend {
@@ -91,6 +91,17 @@ app: Application {
} }
} }
# ─── Ops ──────────────────────────────────────────────────────────────────────
ops: Ops {
style.fill: "#fef9ec"
watchtower: Watchtower {
shape: rectangle
label: "Watchtower\n(containrrr/watchtower)\n\npolls every 5 min\nautopulls + redeploys:\n backend · runner · ui"
}
}
# ─── Init → Storage deps ────────────────────────────────────────────────────── # ─── Init → Storage deps ──────────────────────────────────────────────────────
init.minio-init -> storage.minio: create buckets {style.stroke-dash: 4} init.minio-init -> storage.minio: create buckets {style.stroke-dash: 4}
@@ -106,17 +117,25 @@ app.runner -> storage.minio: write chapter markdown\n& audio MP3s
app.runner -> storage.pocketbase: read/update scrape jobs\nwrite book records app.runner -> storage.pocketbase: read/update scrape jobs\nwrite book records
app.runner -> storage.meilisearch: index books on\nscrape completion app.runner -> storage.meilisearch: index books on\nscrape completion
app.ui -> storage.valkey: read presigned URL cache\n(replaces in-process Map) app.ui -> storage.valkey: read presigned URL cache
app.ui -> storage.pocketbase: auth, progress,\ncomments, settings
# ─── App internal ───────────────────────────────────────────────────────────── # ─── App internal ─────────────────────────────────────────────────────────────
app.ui -> app.backend: REST API calls\n(server-side) app.ui -> app.backend: REST API calls (server-side)\n/api/catalogue /api/book-preview\n/api/chapter-text /api/audio etc.
# ─── Caddy routing ──────────────────────────────────────────────────────────── # ─── Caddy routing ────────────────────────────────────────────────────────────
# Routes sent directly to backend (no SvelteKit counterpart):
# /health /scrape*
# /api/browse /api/book-preview/* /api/chapter-text/*
# /api/reindex/* /api/cover/* /api/audio-proxy/*
# Routes sent to MinIO:
# /avatars/*
# Everything else → SvelteKit UI (including /api/scrape/*, /api/chapter-text-preview/*)
app.caddy -> app.ui: /* (proxy) app.caddy -> app.ui: "/* (catch-all)\n/api/scrape/*\n/api/chapter-text-preview/*\n→ SvelteKit (auth enforced)"
app.caddy -> app.backend: /api/*, /health (proxy) app.caddy -> app.backend: "/health /scrape*\n/api/browse /api/book-preview/*\n/api/chapter-text/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/*"
app.caddy -> storage.minio: /s3/* (proxy, internal only) app.caddy -> storage.minio: "/avatars/*\n(presigned avatar GETs)"
# ─── External → App ─────────────────────────────────────────────────────────── # ─── External → App ───────────────────────────────────────────────────────────
@@ -124,6 +143,12 @@ app.runner -> novelfire: scrape\n(HTTP GET)
app.runner -> kokoro: TTS generation\n(HTTP POST) app.runner -> kokoro: TTS generation\n(HTTP POST)
app.caddy -> letsencrypt: ACME certificate\n(TLS-ALPN-01) app.caddy -> letsencrypt: ACME certificate\n(TLS-ALPN-01)
# ─── Ops → Docker socket ──────────────────────────────────────────────────────
ops.watchtower -> app.backend: watch (label-enabled)
ops.watchtower -> app.runner: watch (label-enabled)
ops.watchtower -> app.ui: watch (label-enabled)
# ─── Browser ────────────────────────────────────────────────────────────────── # ─── Browser ──────────────────────────────────────────────────────────────────
browser -> app.caddy: HTTPS :443\n(single entry point) browser -> app.caddy: HTTPS :443\n(single entry point)

129
v3/docs/d2/architecture.svg Normal file

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 58 KiB

View File

@@ -1,3 +1,5 @@
# Architecture Overview
```mermaid ```mermaid
graph LR graph LR
%% ── External ────────────────────────────────────────────────────────── %% ── External ──────────────────────────────────────────────────────────
@@ -22,12 +24,17 @@ graph LR
%% ── Application ─────────────────────────────────────────────────────── %% ── Application ───────────────────────────────────────────────────────
subgraph APP["Application"] subgraph APP["Application"]
CD[Caddy :443/:80\nreverse proxy\nauto-HTTPS] CD["Caddy :443/:80\ncustom build + caddy-ratelimit\nauto-HTTPS · security headers\nrate limiting · error pages"]
BE[Backend API :8080\nGo HTTP server] BE[Backend API :8080\nGo HTTP server]
RN[Runner :9091\nGo background worker\n/metrics endpoint] RN[Runner :9091\nGo background worker\n/metrics endpoint]
UI[SvelteKit UI :3000\nadapter-node] UI[SvelteKit UI :3000\nadapter-node]
end end
%% ── Ops ───────────────────────────────────────────────────────────────
subgraph OPS["Ops"]
WT[Watchtower\npolls every 5 min\nautopull + redeploy\nbackend · runner · ui]
end
%% ── Init → Storage ──────────────────────────────────────────────────── %% ── Init → Storage ────────────────────────────────────────────────────
MI -.->|create buckets| MN MI -.->|create buckets| MN
PI -.->|bootstrap schema| PB PI -.->|bootstrap schema| PB
@@ -40,20 +47,26 @@ graph LR
RN -->|read/update jobs & books| PB RN -->|read/update jobs & books| PB
RN -->|index books on scrape| MS RN -->|index books on scrape| MS
UI -->|read presign cache| VK UI -->|read presign cache| VK
UI -->|auth · progress · comments| PB
%% ── App internal ────────────────────────────────────────────────────── %% ── App internal ──────────────────────────────────────────────────────
UI -->|REST API| BE UI -->|"REST API (server-side)\n/api/catalogue /api/book-preview\n/api/chapter-text /api/audio"| BE
%% ── Caddy routing ───────────────────────────────────────────────────── %% ── Caddy routing ─────────────────────────────────────────────────────
CD -->|/* proxy| UI CD -->|"/* catch-all\n/api/scrape/*\n/api/chapter-text-preview/*\n→ SvelteKit (auth enforced)"| UI
CD -->|/api/* /health proxy| BE CD -->|"/health /scrape*\n/api/browse /api/book-preview/*\n/api/chapter-text/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/*"| BE
CD -->|/s3/* proxy internal| MN CD -->|/avatars/* presigned GETs| MN
%% ── Runner → External ───────────────────────────────────────────────── %% ── Runner → External ─────────────────────────────────────────────────
RN -->|scrape HTTP GET| NF RN -->|scrape HTTP GET| NF
RN -->|TTS HTTP POST| KK RN -->|TTS HTTP POST| KK
CD -->|ACME certificate| LE CD -->|ACME certificate| LE
%% ── Ops ───────────────────────────────────────────────────────────────
WT -->|watch label-enabled| BE
WT -->|watch label-enabled| RN
WT -->|watch label-enabled| UI
%% ── Client ──────────────────────────────────────────────────────────── %% ── Client ────────────────────────────────────────────────────────────
CL -->|HTTPS :443 single entry| CD CL -->|HTTPS :443 single entry| CD
``` ```

View File

@@ -18,7 +18,7 @@ flowchart TD
E --> G{New chapters\nfound?} E --> G{New chapters\nfound?}
G -- no --> Z([Done — next book]) G -- no --> Z([Done — next book])
G -- yes --> H G -- yes --> H
F --> H[Scrape chapter list\n→ chapters_idx in PocketBase] F --> H[Scrape chapter list with upTo limit\n→ chapters_idx in PocketBase\nretries on 429 with Retry-After backoff]
H --> I[Worker pool — N goroutines\nRUNNER_MAX_CONCURRENT_SCRAPE] H --> I[Worker pool — N goroutines\nRUNNER_MAX_CONCURRENT_SCRAPE]
I --> J[For each missing chapter:\nGET chapter HTML from novelfire.net] I --> J[For each missing chapter:\nGET chapter HTML from novelfire.net]
J --> K[Parse HTML → Markdown\nhtmlutil.NodeToMarkdown] J --> K[Parse HTML → Markdown\nhtmlutil.NodeToMarkdown]
@@ -31,7 +31,7 @@ flowchart TD
## On-Demand Single-Book Scrape ## On-Demand Single-Book Scrape
Triggered when a user visits `/books/{slug}` and the book is not in PocketBase. Triggered when a user visits `/books/{slug}` and the book is not in PocketBase.
The UI calls `GET /api/book-preview/{slug}` → backend enqueues a task. The UI calls `GET /api/book-preview/{slug}` → backend enqueues a scrape task.
```mermaid ```mermaid
sequenceDiagram sequenceDiagram
@@ -76,23 +76,23 @@ requests poll for completion and then stream from MinIO via presigned URL.
```mermaid ```mermaid
flowchart TD flowchart TD
A([POST /api/audio/{slug}/{n}\nbody: voice=af_bella]) --> B{Audio already\nin MinIO?} A(["POST /api/audio/{slug}/{n}\nbody: voice=af_bella"]) --> B{Audio already\nin MinIO?}
B -- yes --> C[200 status: done] B -- yes --> C[200 status: done]
B -- no --> D{Job already\nin queue?} B -- no --> D{Job already\nin queue?}
D -- yes pending/generating --> E[202 task_id + status] D -- "yes pending/generating" --> E[202 task_id + status]
D -- no --> F[INSERT audio_task\nstatus=pending\nin PocketBase] D -- no --> F[INSERT audio_task\nstatus=pending\nin PocketBase]
F --> E F --> E
G([Runner polls task queue]) --> H[Claim audio_task\nstatus=generating] G([Runner polls task queue]) --> H[Claim audio_task\nstatus=generating]
H --> I[GET /api/chapter-text/{slug}/{n}\nfrom backend — plain text] H --> I["GET /api/chapter-text/{slug}/{n}\nfrom backend — plain text"]
I --> J[POST /v1/audio/speech\nto Kokoro-FastAPI\nbody: text + voice] I --> J[POST /v1/audio/speech\nto Kokoro-FastAPI\nbody: text + voice]
J --> K[Stream MP3 response] J --> K[Stream MP3 response]
K --> L[PUT object to MinIO\nlibnovel-audio/{slug}/{n}/{voice}.mp3] K --> L[PUT object to MinIO\nlibnovel-audio/{slug}/{n}/{voice}.mp3]
L --> M[UPDATE audio_task\nstatus=done] L --> M[UPDATE audio_task\nstatus=done]
N([Client polls\nGET /api/audio/status/{slug}/{n}]) --> O{status?} N(["Client polls\nGET /api/audio/status/{slug}/{n}"]) --> O{status?}
O -- pending/generating --> N O -- "pending/generating" --> N
O -- done --> P[GET /api/presign/audio/{slug}/{n}] O -- done --> P["GET /api/presign/audio/{slug}/{n}"]
P --> Q{Valkey cache hit?} P --> Q{Valkey cache hit?}
Q -- yes --> R[302 → presigned URL] Q -- yes --> R[302 → presigned URL]
Q -- no --> S[GeneratePresignedURL\nfrom MinIO — TTL 1h] Q -- no --> S[GeneratePresignedURL\nfrom MinIO — TTL 1h]

View File

@@ -3,7 +3,7 @@
Two representative request paths through the stack: a **page load** (SSR) and a Two representative request paths through the stack: a **page load** (SSR) and a
**media playback** (presigned URL → direct MinIO stream). **media playback** (presigned URL → direct MinIO stream).
## SSR Page Load — Browse / Book Detail ## SSR Page Load — Catalogue / Book Detail
```mermaid ```mermaid
sequenceDiagram sequenceDiagram
@@ -14,10 +14,9 @@ sequenceDiagram
participant MS as Meilisearch :7700 participant MS as Meilisearch :7700
participant PB as PocketBase :8090 participant PB as PocketBase :8090
participant VK as Valkey :6379 participant VK as Valkey :6379
participant MN as MinIO :9000
C->>CD: HTTPS GET /browse C->>CD: HTTPS GET /catalogue
CD->>UI: proxy /* CD->>UI: proxy /* (SvelteKit catch-all)
UI->>BE: GET /api/catalogue?page=1&sort=popular UI->>BE: GET /api/catalogue?page=1&sort=popular
BE->>MS: search(query, filters, sort) BE->>MS: search(query, filters, sort)
MS-->>BE: [{slug, title, …}, …] MS-->>BE: [{slug, title, …}, …]
@@ -25,9 +24,9 @@ sequenceDiagram
UI-->>CD: SSR HTML UI-->>CD: SSR HTML
CD-->>C: 200 HTML CD-->>C: 200 HTML
Note over C,UI: Infinite scroll — client fetches next page Note over C,UI: Infinite scroll — client fetches next page via SvelteKit API route
C->>CD: HTTPS GET /api/browse-page?page=2 C->>CD: HTTPS GET /api/catalogue-page?page=2
CD->>UI: proxy (SvelteKit API route) CD->>UI: proxy /* (SvelteKit /api/catalogue-page server route)
UI->>BE: GET /api/catalogue?page=2 UI->>BE: GET /api/catalogue?page=2
BE->>MS: search(…) BE->>MS: search(…)
MS-->>BE: next page MS-->>BE: next page
@@ -47,16 +46,19 @@ sequenceDiagram
participant MN as MinIO :9000 participant MN as MinIO :9000
C->>CD: GET /api/presign/audio/{slug}/{n}?voice=af_bella C->>CD: GET /api/presign/audio/{slug}/{n}?voice=af_bella
CD->>BE: proxy /api/* CD->>UI: proxy /* (SvelteKit /api/presign/audio route)
UI->>BE: GET /api/presign/audio/{slug}/{n}?voice=af_bella
BE->>VK: GET presign:audio:{slug}:{n}:{voice} BE->>VK: GET presign:audio:{slug}:{n}:{voice}
alt cache hit alt cache hit
VK-->>BE: presigned URL (TTL remaining) VK-->>BE: presigned URL (TTL remaining)
BE-->>C: 302 redirect → presigned URL BE-->>UI: 302 redirect → presigned URL
UI-->>C: 302 redirect
else cache miss else cache miss
BE->>MN: GeneratePresignedURL(audio-bucket, key, 1h) BE->>MN: GeneratePresignedURL(audio-bucket, key, 1h)
MN-->>BE: presigned URL MN-->>BE: presigned URL
BE->>VK: SET presign:audio:… EX 3500 BE->>VK: SET presign:audio:… EX 3500
BE-->>C: 302 redirect → presigned URL BE-->>UI: 302 redirect → presigned URL
UI-->>C: 302 redirect
end end
C->>MN: GET presigned URL (direct, no proxy) C->>MN: GET presigned URL (direct, no proxy)
MN-->>C: audio/mpeg stream MN-->>C: audio/mpeg stream
@@ -74,14 +76,36 @@ sequenceDiagram
participant MN as MinIO :9000 participant MN as MinIO :9000
C->>CD: HTTPS GET /books/{slug}/chapters/{n} C->>CD: HTTPS GET /books/{slug}/chapters/{n}
CD->>UI: proxy /* CD->>UI: proxy /* (SvelteKit catch-all)
UI->>PB: getBook(slug) + listChapterIdx(slug) UI->>PB: getBook(slug) + listChapterIdx(slug)
PB-->>UI: book meta + chapter list PB-->>UI: book meta + chapter list
UI->>BE: GET /api/chapter-markdown/{slug}/{n} UI->>BE: GET /api/chapter-text/{slug}/{n}
BE->>MN: GetObject(chapters-bucket, {slug}/{n}.md) BE->>MN: GetObject(chapters-bucket, {slug}/{n}.md)
MN-->>BE: markdown text MN-->>BE: markdown text
BE-->>UI: markdown body BE-->>UI: plain text (markdown stripped)
Note over UI: marked() → HTML Note over UI: marked() → HTML
UI-->>CD: SSR HTML UI-->>CD: SSR HTML
CD-->>C: 200 HTML CD-->>C: 200 HTML
``` ```
## Caddy Request Lifecycle
Shows how security hardening applies before a request reaches any upstream.
```mermaid
flowchart TD
A([Incoming HTTPS request]) --> B[TLS termination\nLet's Encrypt cert]
B --> C{Rate limit check\ncaddy-ratelimit}
C -- over limit --> D[429 Too Many Requests]
C -- ok --> E[Add security headers\nX-Frame-Options · X-Content-Type-Options\nReferrer-Policy · Permissions-Policy\nHSTS · X-XSS-Protection\nremove Server header]
E --> F{Route match}
F -- "/health /scrape*\n/api/browse /api/book-preview/*\n/api/chapter-text/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/*" --> G[reverse_proxy → backend:8080]
F -- "/avatars/*" --> H[reverse_proxy → minio:9000]
F -- "/* everything else\n(incl. /api/scrape/*\n/api/chapter-text-preview/*)" --> I[reverse_proxy → ui:3000\nSvelteKit auth middleware runs]
G --> J{Upstream healthy?}
H --> J
I --> J
J -- yes --> K([Response to client])
J -- "502/503/504" --> L[handle_errors\nstatic HTML from /srv/errors/]
L --> K
```

View File

@@ -2,20 +2,7 @@
import { Button } from '$lib/components/ui/button'; import { Button } from '$lib/components/ui/button';
import { Textarea } from '$lib/components/ui/textarea'; import { Textarea } from '$lib/components/ui/textarea';
import { cn } from '$lib/utils'; import { cn } from '$lib/utils';
import type { BookComment } from '$lib/types';
interface BookComment {
id: string;
slug: string;
user_id: string;
username: string;
body: string;
upvotes: number;
downvotes: number;
created: string;
parent_id?: string;
replies?: BookComment[];
}
let { let {
slug, slug,
isLoggedIn = false, isLoggedIn = false,
@@ -312,7 +299,7 @@
</div> </div>
{:else} {:else}
<p class="text-sm text-zinc-500"> <p class="text-sm text-zinc-500">
<a href="/auth/login" class="text-amber-400 hover:text-amber-300 transition-colors">Log in</a> <a href="/login" class="text-amber-400 hover:text-amber-300 transition-colors">Log in</a>
to leave a comment. to leave a comment.
</p> </p>
{/if} {/if}

View File

@@ -0,0 +1,71 @@
/**
* Shared types and helpers for the /api/catalogue backend response.
*
* Imported by both:
* - src/routes/catalogue/+page.server.ts (SSR page load)
* - src/routes/api/catalogue-page/+server.ts (infinite-scroll proxy)
*/
/** Shape of a single book as returned by GET /api/catalogue on the Go backend. */
export interface CatalogueBook {
slug: string;
title: string;
author: string;
cover: string;
status: string;
genres: string[];
summary: string;
total_chapters: number;
source_url: string;
ranking: number;
rating: number;
}
/** Facets returned alongside catalogue results for dynamic filter options. */
export interface CatalogueFacets {
genres: string[];
statuses: string[];
}
/** Full response shape from GET /api/catalogue. */
export interface CatalogueResponse {
books: CatalogueBook[];
page: number;
limit: number;
total: number;
has_next: boolean;
facets?: CatalogueFacets;
}
/** Normalised book shape consumed by the catalogue UI. */
export interface NovelListing {
slug: string;
title: string;
cover: string;
rank: string;
rating: string;
chapters: string;
url: string;
// enriched fields
author?: string;
status?: string;
genres?: string[];
source_url?: string;
}
/** Convert a raw CatalogueBook into the UI NovelListing shape. */
export function bookToListing(book: CatalogueBook): NovelListing {
return {
slug: book.slug,
title: book.title,
cover: book.cover,
rank: book.ranking > 0 ? `#${book.ranking}` : '',
rating: book.rating > 0 ? String(book.rating) : '',
chapters: book.total_chapters > 0 ? `${book.total_chapters} chapters` : '',
url: book.source_url ?? '',
author: book.author,
status: book.status,
genres: book.genres ?? [],
source_url: book.source_url
};
}

View File

@@ -51,7 +51,7 @@ export async function presignAvatarUrl(userId: string): Promise<string | null> {
throw new Error(`presign avatar failed: ${res.status} ${body}`); throw new Error(`presign avatar failed: ${res.status} ${body}`);
} }
const data = (await res.json()) as { url: string }; const data = (await res.json()) as { url: string };
return data.url ?? null; return data.url ? rewriteHost(data.url) : null;
} }
/** /**

View File

@@ -46,7 +46,7 @@ export interface Progress {
updated: string; updated: string;
} }
export interface UserSettings { export interface PBUserSettings {
id?: string; id?: string;
session_id: string; session_id: string;
user_id?: string; user_id?: string;
@@ -583,8 +583,8 @@ function settingsFilter(sessionId: string, userId?: string): string {
export async function getSettings( export async function getSettings(
sessionId: string, sessionId: string,
userId?: string userId?: string
): Promise<UserSettings | null> { ): Promise<PBUserSettings | null> {
return listOne<UserSettings>('user_settings', settingsFilter(sessionId, userId)); return listOne<PBUserSettings>('user_settings', settingsFilter(sessionId, userId));
} }
export async function saveSettings( export async function saveSettings(
@@ -592,12 +592,12 @@ export async function saveSettings(
settings: { autoNext: boolean; voice: string; speed: number }, settings: { autoNext: boolean; voice: string; speed: number },
userId?: string userId?: string
): Promise<void> { ): Promise<void> {
const existing = await listOne<UserSettings & { id: string }>( const existing = await listOne<PBUserSettings & { id: string }>(
'user_settings', 'user_settings',
settingsFilter(sessionId, userId) settingsFilter(sessionId, userId)
); );
const payload: Partial<UserSettings> = { const payload: Partial<PBUserSettings> = {
session_id: sessionId, session_id: sessionId,
auto_next: settings.autoNext, auto_next: settings.autoNext,
voice: settings.voice, voice: settings.voice,
@@ -684,6 +684,8 @@ export interface ScrapingTask {
books_found: number; books_found: number;
chapters_scraped: number; chapters_scraped: number;
chapters_skipped: number; chapters_skipped: number;
from_chapter: number;
to_chapter: number;
errors: number; errors: number;
started: string; started: string;
finished: string; finished: string;
@@ -694,6 +696,10 @@ export async function listScrapingTasks(): Promise<ScrapingTask[]> {
return listAll<ScrapingTask>('scraping_tasks', '', '-started'); return listAll<ScrapingTask>('scraping_tasks', '', '-started');
} }
export async function getScrapingTask(id: string): Promise<ScrapingTask | null> {
return listOne<ScrapingTask>('scraping_tasks', `id="${id}"`);
}
// ─── Audio jobs ─────────────────────────────────────────────────────────────── // ─── Audio jobs ───────────────────────────────────────────────────────────────
export interface AudioJob { export interface AudioJob {
@@ -850,7 +856,7 @@ export async function updateUserAvatarUrl(userId: string, avatarUrl: string): Pr
// ─── Comments ───────────────────────────────────────────────────────────────── // ─── Comments ─────────────────────────────────────────────────────────────────
export interface BookComment { export interface PBBookComment {
id: string; id: string;
slug: string; slug: string;
user_id: string; user_id: string;
@@ -881,7 +887,7 @@ export type CommentSort = 'top' | 'new';
export async function listComments( export async function listComments(
slug: string, slug: string,
sort: CommentSort = 'new' sort: CommentSort = 'new'
): Promise<BookComment[]> { ): Promise<PBBookComment[]> {
const token = await getToken(); const token = await getToken();
const slugEsc = slug.replace(/"/g, '\\"'); const slugEsc = slug.replace(/"/g, '\\"');
// Only top-level comments (parent_id is empty or missing) // Only top-level comments (parent_id is empty or missing)
@@ -896,7 +902,7 @@ export async function listComments(
); );
if (!res.ok) return []; if (!res.ok) return [];
const data = await res.json(); const data = await res.json();
let items = (data.items ?? []) as BookComment[]; let items = (data.items ?? []) as PBBookComment[];
if (sort === 'top') { if (sort === 'top') {
items = items.sort((a, b) => { items = items.sort((a, b) => {
const scoreB = (b.upvotes ?? 0) - (b.downvotes ?? 0); const scoreB = (b.upvotes ?? 0) - (b.downvotes ?? 0);
@@ -913,7 +919,7 @@ export async function listComments(
* List replies (1-level deep) for a single parent comment. * List replies (1-level deep) for a single parent comment.
* Always sorted oldest-first so the conversation reads naturally. * Always sorted oldest-first so the conversation reads naturally.
*/ */
export async function listReplies(parentId: string): Promise<BookComment[]> { export async function listReplies(parentId: string): Promise<PBBookComment[]> {
const token = await getToken(); const token = await getToken();
const filter = encodeURIComponent(`parent_id="${parentId.replace(/"/g, '\\"')}"`); const filter = encodeURIComponent(`parent_id="${parentId.replace(/"/g, '\\"')}"`);
const res = await fetch( const res = await fetch(
@@ -922,7 +928,7 @@ export async function listReplies(parentId: string): Promise<BookComment[]> {
); );
if (!res.ok) return []; if (!res.ok) return [];
const data = await res.json(); const data = await res.json();
return (data.items ?? []) as BookComment[]; return (data.items ?? []) as PBBookComment[];
} }
/** /**
@@ -935,7 +941,7 @@ export async function createComment(
userId: string | undefined, userId: string | undefined,
username: string, username: string,
parentId?: string parentId?: string
): Promise<BookComment> { ): Promise<PBBookComment> {
const token = await getToken(); const token = await getToken();
const res = await fetch(`${PB_URL}/api/collections/book_comments/records`, { const res = await fetch(`${PB_URL}/api/collections/book_comments/records`, {
method: 'POST', method: 'POST',
@@ -955,7 +961,7 @@ export async function createComment(
const text = await res.text().catch(() => ''); const text = await res.text().catch(() => '');
throw new Error(`createComment failed: ${res.status} ${text}`); throw new Error(`createComment failed: ${res.status} ${text}`);
} }
return res.json() as Promise<BookComment>; return res.json() as Promise<PBBookComment>;
} }
/** /**
@@ -971,7 +977,7 @@ export async function deleteComment(commentId: string, userId: string): Promise<
headers: { Authorization: `Bearer ${token}` } headers: { Authorization: `Bearer ${token}` }
}); });
if (!getRes.ok) throw new Error(`Comment not found: ${commentId}`); if (!getRes.ok) throw new Error(`Comment not found: ${commentId}`);
const comment = (await getRes.json()) as BookComment; const comment = (await getRes.json()) as PBBookComment;
if (comment.user_id !== userId) throw new Error('Not authorized to delete this comment'); if (comment.user_id !== userId) throw new Error('Not authorized to delete this comment');
// Delete any replies first // Delete any replies first
@@ -982,7 +988,7 @@ export async function deleteComment(commentId: string, userId: string): Promise<
); );
if (repliesRes.ok) { if (repliesRes.ok) {
const repliesData = await repliesRes.json(); const repliesData = await repliesRes.json();
const replies = (repliesData.items ?? []) as BookComment[]; const replies = (repliesData.items ?? []) as PBBookComment[];
await Promise.all( await Promise.all(
replies.map((r) => replies.map((r) =>
fetch(`${PB_URL}/api/collections/book_comments/records/${r.id}`, { fetch(`${PB_URL}/api/collections/book_comments/records/${r.id}`, {
@@ -1035,7 +1041,7 @@ export async function voteComment(
vote: 'up' | 'down', vote: 'up' | 'down',
sessionId: string, sessionId: string,
userId?: string userId?: string
): Promise<BookComment> { ): Promise<PBBookComment> {
const token = await getToken(); const token = await getToken();
// Fetch current comment // Fetch current comment
@@ -1043,7 +1049,7 @@ export async function voteComment(
headers: { Authorization: `Bearer ${token}` } headers: { Authorization: `Bearer ${token}` }
}); });
if (!commentRes.ok) throw new Error(`Comment not found: ${commentId}`); if (!commentRes.ok) throw new Error(`Comment not found: ${commentId}`);
const comment = (await commentRes.json()) as BookComment; const comment = (await commentRes.json()) as PBBookComment;
const existing = await getCommentVote(commentId, sessionId, userId); const existing = await getCommentVote(commentId, sessionId, userId);
@@ -1086,7 +1092,7 @@ export async function voteComment(
}) })
}); });
if (!patchRes.ok) throw new Error(`Failed to update vote counts on comment ${commentId}`); if (!patchRes.ok) throw new Error(`Failed to update vote counts on comment ${commentId}`);
return patchRes.json() as Promise<BookComment>; return patchRes.json() as Promise<PBBookComment>;
} }
/** /**

View File

@@ -33,4 +33,28 @@ export async function backendFetch(path: string, init?: RequestInit): Promise<Re
} }
} }
// ─── Response types ───────────────────────────────────────────────────────────
/**
* Metadata shape returned inside the 200 response from GET /api/book-preview/{slug}.
* Used in both the SSR page load and the API proxy to avoid duplicating the inline type.
*/
export interface BookPreviewMeta {
slug: string;
title: string;
author: string;
cover: string;
status: string;
genres: string[];
summary: string;
total_chapters: number;
source_url: string;
}
/** Full 200 response from GET /api/book-preview/{slug}. */
export interface BookPreviewResponse {
in_lib: boolean;
meta: BookPreviewMeta;
chapters: { number: number; title: string; date?: string }[];
}

View File

@@ -6,34 +6,6 @@
* safe to import in both server and client code. * safe to import in both server and client code.
*/ */
// ── Auth / User ──────────────────────────────────────────────────────────────
export interface AuthUser {
id: string;
username: string;
email: string;
role: 'admin' | 'user';
avatarUrl?: string;
}
// ── Books ────────────────────────────────────────────────────────────────────
export interface BookSummary {
id: string;
slug: string;
title: string;
author: string;
coverUrl: string;
status: string;
/** Reading progress 0–100, if the user has any. */
progress?: number;
}
export interface ChapterRef {
number: number;
title: string;
}
// ── Comments ───────────────────────────────────────────────────────────────── // ── Comments ─────────────────────────────────────────────────────────────────
export interface BookComment { export interface BookComment {
@@ -49,11 +21,6 @@ export interface BookComment {
replies?: BookComment[]; replies?: BookComment[];
} }
// ── Audio ────────────────────────────────────────────────────────────────────
export type AudioStatus = 'idle' | 'loading' | 'generating' | 'ready' | 'error';
export type NextStatus = 'none' | 'prefetching' | 'prefetched' | 'failed';
// ── User settings ───────────────────────────────────────────────────────────── // ── User settings ─────────────────────────────────────────────────────────────
export interface UserSettings { export interface UserSettings {

View File

@@ -0,0 +1,71 @@
<script lang="ts">
import { page } from '$app/state';
const status = $derived(page.status);
const message = $derived(page.error?.message ?? 'Something went wrong.');
const title = $derived(
status === 404
? 'Page not found'
: status === 403
? 'Access denied'
: status === 429
? 'Too many requests'
: status >= 500
? 'Server error'
: 'Error'
);
const description = $derived(
status === 404
? "The page you're looking for doesn't exist or has been moved."
: status === 403
? "You don't have permission to access this page."
: status === 429
? 'You are sending too many requests. Please slow down and try again shortly.'
: status >= 500
? 'An unexpected error occurred on our end. Try refreshing, or come back in a moment.'
: message
);
const code = $derived(String(status));
</script>
<svelte:head>
<title>{status} — {title} · libnovel</title>
</svelte:head>
<!-- Full-viewport centred error page — no layout nav since this is +error.svelte -->
<div
class="min-h-screen bg-zinc-950 text-zinc-100 flex flex-col items-center justify-center px-6 py-16 font-sans"
>
<!-- Large status code -->
<p class="text-[8rem] sm:text-[11rem] font-black leading-none text-zinc-800 select-none tabular-nums">
{code}
</p>
<!-- Title + description -->
<div class="mt-4 text-center max-w-md space-y-2">
<h1 class="text-2xl sm:text-3xl font-bold text-zinc-100">{title}</h1>
<p class="text-zinc-400 text-sm sm:text-base leading-relaxed">{description}</p>
</div>
<!-- Actions -->
<div class="mt-10 flex flex-wrap gap-3 justify-center">
<a
href="/"
class="px-5 py-2.5 rounded-xl bg-amber-400 text-zinc-900 font-semibold text-sm hover:bg-amber-300 transition-colors"
>
Go home
</a>
<button
onclick={() => history.back()}
class="px-5 py-2.5 rounded-xl bg-zinc-800 border border-zinc-700 text-zinc-200 font-semibold text-sm hover:bg-zinc-700 transition-colors"
>
Go back
</button>
</div>
<!-- Subtle branding -->
<p class="mt-16 text-xs text-zinc-700 tracking-widest uppercase select-none">libnovel</p>
</div>

View File

@@ -233,8 +233,8 @@
Library Library
</a> </a>
<a <a
href="/browse" href="/catalogue"
class="hidden sm:block text-sm transition-colors {page.url.pathname.startsWith('/browse') ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}" class="hidden sm:block text-sm transition-colors {page.url.pathname.startsWith('/catalogue') ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
> >
Discover Discover
</a> </a>
@@ -250,15 +250,9 @@
</a> </a>
<a <a
href="/admin/audio" href="/admin/audio"
class="hidden sm:block text-sm transition-colors {page.url.pathname === '/admin/audio' ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}" class="hidden sm:block text-sm transition-colors {page.url.pathname.startsWith('/admin/audio') ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
> >
Audio cache Audio
</a>
<a
href="/admin/audio-jobs"
class="hidden sm:block text-sm transition-colors {page.url.pathname.startsWith('/admin/audio-jobs') ? 'text-zinc-100 font-medium' : 'text-zinc-400 hover:text-zinc-100'}"
>
Audio jobs
</a> </a>
{/if} {/if}
<a <a
@@ -318,9 +312,9 @@
Library Library
</a> </a>
<a <a
href="/browse" href="/catalogue"
onclick={() => (menuOpen = false)} onclick={() => (menuOpen = false)}
class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/browse') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}" class="px-3 py-2.5 rounded-lg text-sm font-medium transition-colors {page.url.pathname.startsWith('/catalogue') ? 'bg-zinc-800 text-zinc-100' : 'text-zinc-400 hover:bg-zinc-800 hover:text-zinc-100'}"
> >
Discover Discover
</a> </a>
@@ -381,7 +375,7 @@
<!-- Top row: site links --> <!-- Top row: site links -->
<nav class="flex flex-wrap items-center justify-center gap-x-5 gap-y-2"> <nav class="flex flex-wrap items-center justify-center gap-x-5 gap-y-2">
<a href="/books" class="hover:text-zinc-400 transition-colors">Library</a> <a href="/books" class="hover:text-zinc-400 transition-colors">Library</a>
<a href="/browse" class="hover:text-zinc-400 transition-colors">Discover</a> <a href="/catalogue" class="hover:text-zinc-400 transition-colors">Discover</a>
<a <a
href="https://novelfire.net" href="https://novelfire.net"
target="_blank" target="_blank"

View File

@@ -140,7 +140,7 @@
<p class="text-lg font-semibold text-zinc-300 mb-2">Your library is empty</p> <p class="text-lg font-semibold text-zinc-300 mb-2">Your library is empty</p>
<p class="text-sm mb-6">Discover novels and scrape them into your library.</p> <p class="text-sm mb-6">Discover novels and scrape them into your library.</p>
<a <a
href="/browse" href="/catalogue"
class="inline-block px-6 py-3 bg-amber-400 text-zinc-900 font-semibold rounded hover:bg-amber-300 transition-colors" class="inline-block px-6 py-3 bg-amber-400 text-zinc-900 font-semibold rounded hover:bg-amber-300 transition-colors"
> >
Discover Novels Discover Novels

View File

@@ -1,17 +1,6 @@
import { redirect } from '@sveltejs/kit'; import { redirect } from '@sveltejs/kit';
import type { PageServerLoad } from './$types'; import type { PageServerLoad } from './$types';
import { listAudioJobs } from '$lib/server/pocketbase';
import { log } from '$lib/server/logger';
export const load: PageServerLoad = async ({ locals }) => { export const load: PageServerLoad = async () => {
if (locals.user?.role !== 'admin') { redirect(301, '/admin/audio');
redirect(302, '/');
}
const jobs = await listAudioJobs().catch((e) => {
log.warn('admin/audio-jobs', 'failed to load audio jobs', { err: String(e) });
return [];
});
return { jobs };
}; };

View File

@@ -1,153 +0,0 @@
<script lang="ts">
import { untrack } from 'svelte';
import type { PageData } from './$types';
let { data }: { data: PageData } = $props();
let jobs = $state(untrack(() => data.jobs));
// ── Live-poll: refresh while any job is in-flight ────────────────────────────
let hasInFlight = $derived(jobs.some((j) => j.status === 'pending' || j.status === 'generating'));
$effect(() => {
if (!hasInFlight) return;
const id = setInterval(async () => {
const res = await fetch('/admin/audio-jobs?__data=1').catch(() => null);
if (res?.ok) {
// SvelteKit invalidateAll is cleaner — just trigger a soft navigation reload.
import('$app/navigation').then(({ invalidateAll }) => invalidateAll());
}
}, 3000);
return () => clearInterval(id);
});
// Keep local state in sync when server re-loads
$effect(() => {
jobs = data.jobs;
});
// ── Helpers ──────────────────────────────────────────────────────────────────
function statusColor(status: string) {
if (status === 'done') return 'text-green-400';
if (status === 'generating') return 'text-amber-400 animate-pulse';
if (status === 'pending') return 'text-sky-400 animate-pulse';
if (status === 'failed') return 'text-red-400';
return 'text-zinc-300';
}
function fmtDate(s: string) {
if (!s) return '—';
return new Date(s).toLocaleString(undefined, {
month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit'
});
}
function duration(started: string, finished: string) {
if (!started || !finished) return '—';
const ms = new Date(finished).getTime() - new Date(started).getTime();
if (ms < 0) return '—';
const s = Math.floor(ms / 1000);
if (s < 60) return `${s}s`;
const m = Math.floor(s / 60);
return `${m}m ${s % 60}s`;
}
// ── Search ───────────────────────────────────────────────────────────────────
let q = $state('');
let filtered = $derived(
q.trim()
? jobs.filter(
(j) =>
j.slug.toLowerCase().includes(q.toLowerCase().trim()) ||
j.voice.toLowerCase().includes(q.toLowerCase().trim()) ||
j.status.toLowerCase().includes(q.toLowerCase().trim())
)
: jobs
);
// ── Stats ────────────────────────────────────────────────────────────────────
let stats = $derived({
total: jobs.length,
done: jobs.filter((j) => j.status === 'done').length,
failed: jobs.filter((j) => j.status === 'failed').length,
inFlight: jobs.filter((j) => j.status === 'pending' || j.status === 'generating').length
});
</script>
<svelte:head>
<title>Audio jobs — libnovel admin</title>
</svelte:head>
<div class="space-y-6">
<div class="flex items-start justify-between flex-wrap gap-3">
<div>
<h1 class="text-2xl font-bold text-zinc-100">Audio jobs</h1>
<p class="text-zinc-400 text-sm mt-1">
{stats.total} total &middot;
<span class="text-green-400">{stats.done} done</span> &middot;
{#if stats.failed > 0}
<span class="text-red-400">{stats.failed} failed</span> &middot;
{/if}
{#if stats.inFlight > 0}
<span class="text-amber-400 animate-pulse">{stats.inFlight} in-flight</span>
{:else}
<span class="text-zinc-500">0 in-flight</span>
{/if}
</p>
</div>
</div>
<!-- Search -->
<input
type="search"
bind:value={q}
placeholder="Filter by slug, voice or status…"
class="w-full max-w-sm bg-zinc-800 border border-zinc-700 rounded-lg px-3 py-2 text-zinc-100 text-sm placeholder-zinc-500 focus:outline-none focus:ring-2 focus:ring-amber-400"
/>
{#if filtered.length === 0}
<p class="text-zinc-500 text-sm py-8 text-center">
{q.trim() ? 'No results.' : 'No audio jobs yet.'}
</p>
{:else}
<div class="overflow-x-auto rounded-xl border border-zinc-700">
<table class="w-full text-sm">
<thead class="bg-zinc-800 text-zinc-400 text-xs uppercase tracking-wide">
<tr>
<th class="px-4 py-3 text-left">Book</th>
<th class="px-4 py-3 text-right">Ch.</th>
<th class="px-4 py-3 text-left">Voice</th>
<th class="px-4 py-3 text-left">Status</th>
<th class="px-4 py-3 text-left">Started</th>
<th class="px-4 py-3 text-left">Duration</th>
</tr>
</thead>
<tbody class="divide-y divide-zinc-700/50">
{#each filtered as job}
<tr class="bg-zinc-900 hover:bg-zinc-800/50 transition-colors">
<td class="px-4 py-3 text-zinc-200 font-medium">
<a href="/books/{job.slug}" class="hover:text-amber-400 transition-colors">
{job.slug}
</a>
</td>
<td class="px-4 py-3 text-right text-zinc-400">{job.chapter}</td>
<td class="px-4 py-3 text-zinc-400 font-mono text-xs">{job.voice}</td>
<td class="px-4 py-3">
<span class="font-medium {statusColor(job.status)}">{job.status}</span>
</td>
<td class="px-4 py-3 text-zinc-400">{fmtDate(job.started)}</td>
<td class="px-4 py-3 text-zinc-400">{duration(job.started, job.finished)}</td>
</tr>
{#if job.error_message}
<tr class="bg-red-950/20">
<td colspan="6" class="px-4 py-2 text-xs text-red-400 font-mono"
>{job.error_message}</td
>
</tr>
{/if}
{/each}
</tbody>
</table>
</div>
{/if}
</div>

View File

@@ -1,6 +1,6 @@
import { redirect } from '@sveltejs/kit'; import { redirect } from '@sveltejs/kit';
import type { PageServerLoad } from './$types'; import type { PageServerLoad } from './$types';
import { listAudioCache } from '$lib/server/pocketbase'; import { listAudioCache, listAudioJobs, type AudioCacheEntry, type AudioJob } from '$lib/server/pocketbase';
import { log } from '$lib/server/logger'; import { log } from '$lib/server/logger';
export const load: PageServerLoad = async ({ locals }) => { export const load: PageServerLoad = async ({ locals }) => {
@@ -8,10 +8,16 @@ export const load: PageServerLoad = async ({ locals }) => {
redirect(302, '/'); redirect(302, '/');
} }
const entries = await listAudioCache().catch((e) => { const [entries, jobs] = await Promise.all([
log.warn('admin/audio', 'failed to load audio cache', { err: String(e) }); listAudioCache().catch((e): AudioCacheEntry[] => {
return []; log.warn('admin/audio', 'failed to load audio cache', { err: String(e) });
}); return [];
}),
listAudioJobs().catch((e): AudioJob[] => {
log.warn('admin/audio', 'failed to load audio jobs', { err: String(e) });
return [];
})
]);
return { entries }; return { entries, jobs };
}; };

View File

@@ -1,19 +1,42 @@
<script lang="ts"> <script lang="ts">
import { untrack } from 'svelte'; import { untrack } from 'svelte';
import { invalidateAll } from '$app/navigation';
import type { PageData } from './$types'; import type { PageData } from './$types';
import type { AudioJob, AudioCacheEntry } from '$lib/server/pocketbase';
let { data }: { data: PageData } = $props(); let { data }: { data: PageData } = $props();
let entries = $state(untrack(() => data.entries)); let entries = $state<AudioCacheEntry[]>(untrack(() => data.entries));
let jobs = $state<AudioJob[]>(untrack(() => data.jobs));
// ── Parse cache_key ───────────────────────────────────────────────────────── // Keep in sync on server reloads
// cache_key format: "slug/chapter/voice" $effect(() => {
function parseKey(key: string) { entries = data.entries;
const parts = key.split('/'); jobs = data.jobs;
if (parts.length >= 3) { });
return { slug: parts[0], chapter: parts[1], voice: parts.slice(2).join('/') };
} // ── Live-poll while any job is in-flight ─────────────────────────────────────
return { slug: key, chapter: '—', voice: '—' }; let hasInFlight = $derived(jobs.some((j) => j.status === 'pending' || j.status === 'generating'));
$effect(() => {
if (!hasInFlight) return;
const id = setInterval(() => {
invalidateAll();
}, 3000);
return () => clearInterval(id);
});
// ── Tabs ─────────────────────────────────────────────────────────────────────
type Tab = 'jobs' | 'cache';
let activeTab = $state<Tab>('jobs');
// ── Helpers ──────────────────────────────────────────────────────────────────
function jobStatusColor(status: string) {
if (status === 'done') return 'text-green-400';
if (status === 'generating') return 'text-amber-400 animate-pulse';
if (status === 'pending') return 'text-sky-400 animate-pulse';
if (status === 'failed') return 'text-red-400';
return 'text-zinc-300';
} }
function fmtDate(s: string) { function fmtDate(s: string) {
@@ -23,71 +46,237 @@
}); });
} }
// ── Search ────────────────────────────────────────────────────────────────── function duration(started: string, finished: string) {
let q = $state(''); if (!started || !finished) return '—';
let filtered = $derived( const ms = new Date(finished).getTime() - new Date(started).getTime();
q.trim() if (ms < 0) return '—';
? entries.filter((e) => e.cache_key.toLowerCase().includes(q.toLowerCase().trim())) const s = Math.floor(ms / 1000);
if (s < 60) return `${s}s`;
const m = Math.floor(s / 60);
return `${m}m ${s % 60}s`;
}
// ── Audio jobs stats + filter ────────────────────────────────────────────────
let jobsQ = $state('');
let filteredJobs = $derived(
jobsQ.trim()
? jobs.filter(
(j) =>
j.slug.toLowerCase().includes(jobsQ.toLowerCase().trim()) ||
j.voice.toLowerCase().includes(jobsQ.toLowerCase().trim()) ||
j.status.toLowerCase().includes(jobsQ.toLowerCase().trim())
)
: jobs
);
let stats = $derived({
total: jobs.length,
done: jobs.filter((j) => j.status === 'done').length,
failed: jobs.filter((j) => j.status === 'failed').length,
inFlight: jobs.filter((j) => j.status === 'pending' || j.status === 'generating').length
});
// ── Audio cache filter ───────────────────────────────────────────────────────
function parseCacheKey(key: string) {
const parts = key.split('/');
if (parts.length >= 3) {
return { slug: parts[0], chapter: parts[1], voice: parts.slice(2).join('/') };
}
return { slug: key, chapter: '—', voice: '—' };
}
let cacheQ = $state('');
let filteredCache = $derived(
cacheQ.trim()
? entries.filter((e) => e.cache_key.toLowerCase().includes(cacheQ.toLowerCase().trim()))
: entries : entries
); );
</script> </script>
<svelte:head> <svelte:head>
<title>Audio cache — libnovel admin</title> <title>Audio — libnovel admin</title>
</svelte:head> </svelte:head>
<div class="space-y-6"> <div class="space-y-6">
<!-- Header -->
<div> <div>
<h1 class="text-2xl font-bold text-zinc-100">Audio cache</h1> <h1 class="text-2xl font-bold text-zinc-100">Audio</h1>
<p class="text-zinc-400 text-sm mt-1">{entries.length} cached audio file{entries.length !== 1 ? 's' : ''}</p> <p class="text-zinc-400 text-sm mt-1">
{stats.total} job{stats.total !== 1 ? 's' : ''} &middot;
<span class="text-green-400">{stats.done} done</span>
{#if stats.failed > 0}
&middot; <span class="text-red-400">{stats.failed} failed</span>
{/if}
{#if stats.inFlight > 0}
&middot; <span class="text-amber-400 animate-pulse">{stats.inFlight} in-flight</span>
{/if}
&middot; {entries.length} cached file{entries.length !== 1 ? 's' : ''}
</p>
</div> </div>
<!-- Search --> <!-- Tabs -->
<input <div class="flex gap-1 bg-zinc-800 rounded-lg p-1 w-fit border border-zinc-700">
type="search" <button
bind:value={q} onclick={() => (activeTab = 'jobs')}
placeholder="Filter by slug, chapter or voice…" class="px-4 py-1.5 rounded-md text-sm font-medium transition-colors
class="w-full max-w-sm bg-zinc-800 border border-zinc-700 rounded-lg px-3 py-2 text-zinc-100 text-sm placeholder-zinc-500 focus:outline-none focus:ring-2 focus:ring-amber-400" {activeTab === 'jobs' ? 'bg-zinc-700 text-zinc-100' : 'text-zinc-400 hover:text-zinc-200'}"
/> >
Jobs
{#if stats.inFlight > 0}
<span class="ml-1.5 inline-flex items-center justify-center w-4 h-4 rounded-full bg-amber-400 text-zinc-900 text-[10px] font-bold">
{stats.inFlight}
</span>
{/if}
</button>
<button
onclick={() => (activeTab = 'cache')}
class="px-4 py-1.5 rounded-md text-sm font-medium transition-colors
{activeTab === 'cache' ? 'bg-zinc-700 text-zinc-100' : 'text-zinc-400 hover:text-zinc-200'}"
>
Cache
</button>
</div>
{#if filtered.length === 0} <!-- ── Audio Jobs tab ─────────────────────────────────────────────────────── -->
<p class="text-zinc-500 text-sm py-8 text-center"> {#if activeTab === 'jobs'}
{q.trim() ? 'No results.' : 'Audio cache is empty.'} <input
</p> type="search"
{:else} bind:value={jobsQ}
<div class="overflow-x-auto rounded-xl border border-zinc-700"> placeholder="Filter by slug, voice or status…"
<table class="w-full text-sm"> class="w-full max-w-sm bg-zinc-800 border border-zinc-700 rounded-lg px-3 py-2 text-zinc-100 text-sm placeholder-zinc-500 focus:outline-none focus:ring-2 focus:ring-amber-400"
<thead class="bg-zinc-800 text-zinc-400 text-xs uppercase tracking-wide"> />
<tr>
<th class="px-4 py-3 text-left">Book</th> {#if filteredJobs.length === 0}
<th class="px-4 py-3 text-left">Chapter</th> <p class="text-zinc-500 text-sm py-8 text-center">
<th class="px-4 py-3 text-left">Voice</th> {jobsQ.trim() ? 'No matching jobs.' : 'No audio jobs yet.'}
<th class="px-4 py-3 text-left">Filename</th> </p>
<th class="px-4 py-3 text-left">Updated</th> {:else}
</tr> <!-- Desktop table -->
</thead> <div class="hidden sm:block overflow-x-auto rounded-xl border border-zinc-700">
<tbody class="divide-y divide-zinc-700/50"> <table class="w-full text-sm">
{#each filtered as entry} <thead class="bg-zinc-800 text-zinc-400 text-xs uppercase tracking-wide">
{@const parts = parseKey(entry.cache_key)} <tr>
<tr class="bg-zinc-900 hover:bg-zinc-800/50 transition-colors"> <th class="px-4 py-3 text-left">Book</th>
<td class="px-4 py-3 text-zinc-200 font-medium"> <th class="px-4 py-3 text-right">Ch.</th>
<a <th class="px-4 py-3 text-left">Voice</th>
href="/books/{parts.slug}" <th class="px-4 py-3 text-left">Status</th>
class="hover:text-amber-400 transition-colors" <th class="px-4 py-3 text-left">Started</th>
> <th class="px-4 py-3 text-left">Duration</th>
{parts.slug}
</a>
</td>
<td class="px-4 py-3 text-zinc-400">{parts.chapter}</td>
<td class="px-4 py-3 text-zinc-400 font-mono text-xs">{parts.voice}</td>
<td class="px-4 py-3 text-zinc-500 font-mono text-xs truncate max-w-[14rem]" title={entry.filename}>
{entry.filename}
</td>
<td class="px-4 py-3 text-zinc-400">{fmtDate(entry.updated)}</td>
</tr> </tr>
{/each} </thead>
</tbody> <tbody class="divide-y divide-zinc-700/50">
</table> {#each filteredJobs as job}
</div> <tr class="bg-zinc-900 hover:bg-zinc-800/50 transition-colors">
<td class="px-4 py-3 text-zinc-200 font-medium">
<a href="/books/{job.slug}" class="hover:text-amber-400 transition-colors">{job.slug}</a>
</td>
<td class="px-4 py-3 text-right text-zinc-400">{job.chapter}</td>
<td class="px-4 py-3 text-zinc-400 font-mono text-xs">{job.voice}</td>
<td class="px-4 py-3">
<span class="font-medium {jobStatusColor(job.status)}">{job.status}</span>
</td>
<td class="px-4 py-3 text-zinc-400 whitespace-nowrap">{fmtDate(job.started)}</td>
<td class="px-4 py-3 text-zinc-400 whitespace-nowrap">{duration(job.started, job.finished)}</td>
</tr>
{#if job.error_message}
<tr class="bg-red-950/20">
<td colspan="6" class="px-4 py-2 text-xs text-red-400 font-mono">{job.error_message}</td>
</tr>
{/if}
{/each}
</tbody>
</table>
</div>
<!-- Mobile cards -->
<div class="sm:hidden space-y-3">
{#each filteredJobs as job}
<div class="bg-zinc-900 rounded-xl border border-zinc-700 p-4 space-y-2">
<div class="flex items-start justify-between gap-2">
<a href="/books/{job.slug}" class="text-zinc-200 font-medium hover:text-amber-400 transition-colors truncate">
{job.slug}
</a>
<span class="shrink-0 text-xs font-semibold {jobStatusColor(job.status)}">{job.status}</span>
</div>
<div class="grid grid-cols-2 gap-1 text-xs">
<span class="text-zinc-500">Chapter</span><span class="text-zinc-400 text-right">{job.chapter}</span>
<span class="text-zinc-500">Voice</span><span class="text-zinc-400 font-mono text-right truncate">{job.voice}</span>
<span class="text-zinc-500">Started</span><span class="text-zinc-400 text-right">{fmtDate(job.started)}</span>
<span class="text-zinc-500">Duration</span><span class="text-zinc-400 text-right">{duration(job.started, job.finished)}</span>
</div>
{#if job.error_message}
<p class="text-xs text-red-400 font-mono break-all">{job.error_message}</p>
{/if}
</div>
{/each}
</div>
{/if}
{/if}
<!-- ── Audio Cache tab ───────────────────────────────────────────────────── -->
{#if activeTab === 'cache'}
<input
type="search"
bind:value={cacheQ}
placeholder="Filter by slug, chapter or voice…"
class="w-full max-w-sm bg-zinc-800 border border-zinc-700 rounded-lg px-3 py-2 text-zinc-100 text-sm placeholder-zinc-500 focus:outline-none focus:ring-2 focus:ring-amber-400"
/>
{#if filteredCache.length === 0}
<p class="text-zinc-500 text-sm py-8 text-center">
{cacheQ.trim() ? 'No results.' : 'Audio cache is empty.'}
</p>
{:else}
<!-- Desktop table -->
<div class="hidden sm:block overflow-x-auto rounded-xl border border-zinc-700">
<table class="w-full text-sm">
<thead class="bg-zinc-800 text-zinc-400 text-xs uppercase tracking-wide">
<tr>
<th class="px-4 py-3 text-left">Book</th>
<th class="px-4 py-3 text-left">Chapter</th>
<th class="px-4 py-3 text-left">Voice</th>
<th class="px-4 py-3 text-left">Filename</th>
<th class="px-4 py-3 text-left">Updated</th>
</tr>
</thead>
<tbody class="divide-y divide-zinc-700/50">
{#each filteredCache as entry}
{@const parts = parseCacheKey(entry.cache_key)}
<tr class="bg-zinc-900 hover:bg-zinc-800/50 transition-colors">
<td class="px-4 py-3 text-zinc-200 font-medium">
<a href="/books/{parts.slug}" class="hover:text-amber-400 transition-colors">{parts.slug}</a>
</td>
<td class="px-4 py-3 text-zinc-400">{parts.chapter}</td>
<td class="px-4 py-3 text-zinc-400 font-mono text-xs">{parts.voice}</td>
<td class="px-4 py-3 text-zinc-500 font-mono text-xs truncate max-w-[14rem]" title={entry.filename}>
{entry.filename}
</td>
<td class="px-4 py-3 text-zinc-400 whitespace-nowrap">{fmtDate(entry.updated)}</td>
</tr>
{/each}
</tbody>
</table>
</div>
<!-- Mobile cards -->
<div class="sm:hidden space-y-3">
{#each filteredCache as entry}
{@const parts = parseCacheKey(entry.cache_key)}
<div class="bg-zinc-900 rounded-xl border border-zinc-700 p-4 space-y-2">
<a href="/books/{parts.slug}" class="text-zinc-200 font-medium hover:text-amber-400 transition-colors block truncate">
{parts.slug}
</a>
<div class="grid grid-cols-2 gap-1 text-xs">
<span class="text-zinc-500">Chapter</span><span class="text-zinc-400 text-right">{parts.chapter}</span>
<span class="text-zinc-500">Voice</span><span class="text-zinc-400 font-mono text-right truncate">{parts.voice}</span>
<span class="text-zinc-500">Updated</span><span class="text-zinc-400 text-right">{fmtDate(entry.updated)}</span>
</div>
{#if entry.filename}
<p class="text-xs text-zinc-500 font-mono truncate" title={entry.filename}>{entry.filename}</p>
{/if}
</div>
{/each}
</div>
{/if}
{/if} {/if}
</div> </div>

View File

@@ -9,7 +9,6 @@
// ── Live-poll status ──────────────────────────────────────────────────────── // ── Live-poll status ────────────────────────────────────────────────────────
let running = $state(untrack(() => data.running)); let running = $state(untrack(() => data.running));
let tasks = $state(untrack(() => data.tasks)); let tasks = $state(untrack(() => data.tasks));
let polling = $state(false);
// Poll every 5 s while a job is running // Poll every 5 s while a job is running
$effect(() => { $effect(() => {
@@ -20,7 +19,6 @@
const body = await res.json().catch(() => null); const body = await res.json().catch(() => null);
running = body?.running ?? false; running = body?.running ?? false;
if (!running) { if (!running) {
// Refresh tasks list once job finishes
await invalidateAll(); await invalidateAll();
} }
} }
@@ -34,28 +32,27 @@
tasks = data.tasks; tasks = data.tasks;
}); });
// ── Trigger scrape ────────────────────────────────────────────────────────── // ── Single book scrape ──────────────────────────────────────────────────────
let scrapeUrl = $state(''); let scrapeUrl = $state('');
let scrapeError = $state(''); let scrapeError = $state('');
let scraping = $state(false); let scraping = $state(false);
async function triggerScrape(url?: string) { async function triggerBookScrape(url: string) {
if (running || scraping) return; if (running || scraping || !url.trim()) return;
scraping = true; scraping = true;
scrapeError = ''; scrapeError = '';
try { try {
const body = url ? { url } : {};
const res = await fetch('/api/scrape', { const res = await fetch('/api/scrape', {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body) body: JSON.stringify({ url: url.trim() })
}); });
if (!res.ok) { if (!res.ok) {
const data = await res.json().catch(() => ({})); const d = await res.json().catch(() => ({}));
scrapeError = data.error ?? data.message ?? `Error ${res.status}`; scrapeError = d.error ?? d.message ?? `Error ${res.status}`;
} else { } else {
running = true; running = true;
if (url) scrapeUrl = ''; scrapeUrl = '';
} }
} catch { } catch {
scrapeError = 'Network error.'; scrapeError = 'Network error.';
@@ -64,8 +61,72 @@
} }
} }
// ── Range scrape ────────────────────────────────────────────────────────────
let rangeUrl = $state('');
let rangeFrom = $state<number | null>(null);
let rangeTo = $state<number | null>(null);
let rangeError = $state('');
let ranging = $state(false);
async function triggerRangeScrape() {
if (running || ranging || !rangeUrl.trim() || rangeFrom === null) return;
ranging = true;
rangeError = '';
try {
const body: Record<string, unknown> = { url: rangeUrl.trim(), from: rangeFrom };
if (rangeTo !== null) body.to = rangeTo;
const res = await fetch('/api/scrape/range', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body)
});
if (!res.ok) {
const d = await res.json().catch(() => ({}));
rangeError = d.error ?? d.message ?? `Error ${res.status}`;
} else {
running = true;
rangeUrl = '';
rangeFrom = null;
rangeTo = null;
}
} catch {
rangeError = 'Network error.';
} finally {
ranging = false;
}
}
// ── Continue / Retry task ───────────────────────────────────────────────────
function scrollToRangeForm() {
document.getElementById('range-form')?.scrollIntoView({ behavior: 'smooth', block: 'center' });
}
function scrollToBookForm() {
document.getElementById('book-form')?.scrollIntoView({ behavior: 'smooth', block: 'center' });
}
function continueTask(task: ScrapingTask) {
// Re-enqueue a book_range from where it left off
rangeUrl = task.target_url ?? '';
rangeFrom = (task.from_chapter ?? 1) + (task.chapters_scraped ?? 0);
rangeTo = task.to_chapter > 0 ? task.to_chapter : null;
scrollToRangeForm();
}
function retryTask(task: ScrapingTask) {
if (task.kind === 'book_range') {
rangeUrl = task.target_url ?? '';
rangeFrom = task.from_chapter ?? 1;
rangeTo = task.to_chapter > 0 ? task.to_chapter : null;
scrollToRangeForm();
} else {
// book / catalogue — just pre-fill the single-book form
scrapeUrl = task.target_url ?? '';
scrollToBookForm();
}
}
// ── Cancel task ───────────────────────────────────────────────────────────── // ── Cancel task ─────────────────────────────────────────────────────────────
// Tracks which task IDs are currently being cancelled (to disable the button).
let cancellingIds = $state(new Set<string>()); let cancellingIds = $state(new Set<string>());
let cancelErrors: Record<string, string> = $state({}); let cancelErrors: Record<string, string> = $state({});
@@ -79,7 +140,6 @@
const body = await res.json().catch(() => ({})); const body = await res.json().catch(() => ({}));
cancelErrors = { ...cancelErrors, [id]: body.error ?? body.message ?? `Error ${res.status}` }; cancelErrors = { ...cancelErrors, [id]: body.error ?? body.message ?? `Error ${res.status}` };
} else { } else {
// Optimistically flip status in the local list so the button disappears immediately.
tasks = tasks.map((t: ScrapingTask) => (t.id === id ? { ...t, status: 'cancelled' } : t)); tasks = tasks.map((t: ScrapingTask) => (t.id === id ? { ...t, status: 'cancelled' } : t));
} }
} catch { } catch {
@@ -89,6 +149,19 @@
} }
} }
// ── Table filter ────────────────────────────────────────────────────────────
let q = $state('');
let filtered = $derived(
q.trim()
? tasks.filter(
(t: ScrapingTask) =>
t.kind.toLowerCase().includes(q.toLowerCase()) ||
t.status.toLowerCase().includes(q.toLowerCase()) ||
(t.target_url ?? '').toLowerCase().includes(q.toLowerCase())
)
: tasks
);
// ── Helpers ───────────────────────────────────────────────────────────────── // ── Helpers ─────────────────────────────────────────────────────────────────
function statusColor(status: string) { function statusColor(status: string) {
if (status === 'done') return 'text-green-400'; if (status === 'done') return 'text-green-400';
@@ -114,6 +187,16 @@
const m = Math.floor(s / 60); const m = Math.floor(s / 60);
return `${m}m ${s % 60}s`; return `${m}m ${s % 60}s`;
} }
// Popular novelfire genres for quick-scrape links
const quickScrapes = [
{ label: 'Action', url: 'https://novelfire.net/genre/action' },
{ label: 'Fantasy', url: 'https://novelfire.net/genre/fantasy' },
{ label: 'Romance', url: 'https://novelfire.net/genre/romance' },
{ label: 'System', url: 'https://novelfire.net/genre/system' },
{ label: 'Isekai', url: 'https://novelfire.net/genre/isekai' },
{ label: 'Martial Arts', url: 'https://novelfire.net/genre/martial-arts' },
];
</script> </script>
<svelte:head> <svelte:head>
@@ -121,6 +204,7 @@
</svelte:head> </svelte:head>
<div class="space-y-8"> <div class="space-y-8">
<!-- Header -->
<div class="flex items-center justify-between flex-wrap gap-3"> <div class="flex items-center justify-between flex-wrap gap-3">
<div> <div>
<h1 class="text-2xl font-bold text-zinc-100">Scrape tasks</h1> <h1 class="text-2xl font-bold text-zinc-100">Scrape tasks</h1>
@@ -133,106 +217,251 @@
{/if} {/if}
</p> </p>
</div> </div>
</div>
<!-- Trigger controls --> <!-- Scrape controls -->
<div class="flex flex-wrap gap-3 items-start"> <div class="grid gap-4 sm:grid-cols-2">
<button <!-- Single book -->
onclick={() => triggerScrape()} <div id="book-form" class="bg-zinc-800 rounded-xl border border-zinc-700 p-5 space-y-3">
disabled={running || scraping} <h2 class="text-sm font-semibold text-zinc-300">Scrape a single book</h2>
class="px-4 py-2 rounded-lg bg-amber-400 text-zinc-900 font-semibold text-sm hover:bg-amber-300 transition-colors disabled:opacity-50" <div class="flex gap-2">
> <input
Full catalogue scrape type="url"
</button> bind:value={scrapeUrl}
placeholder="https://novelfire.net/book/…"
class="flex-1 min-w-0 bg-zinc-700 border border-zinc-600 rounded-lg px-3 py-2 text-zinc-100 text-sm placeholder-zinc-500 focus:outline-none focus:ring-2 focus:ring-amber-400"
/>
<button
onclick={() => triggerBookScrape(scrapeUrl)}
disabled={!scrapeUrl.trim() || running || scraping}
class="shrink-0 px-4 py-2 rounded-lg bg-zinc-600 text-zinc-100 font-semibold text-sm hover:bg-zinc-500 transition-colors disabled:opacity-50"
>
{scraping ? 'Queuing…' : 'Scrape'}
</button>
</div>
{#if scrapeError}
<p class="text-sm text-red-400">{scrapeError}</p>
{/if}
</div>
<!-- Range scrape -->
<div id="range-form" class="bg-zinc-800 rounded-xl border border-zinc-700 p-5 space-y-3">
<h2 class="text-sm font-semibold text-zinc-300">Scrape chapter range</h2>
<input
type="url"
bind:value={rangeUrl}
placeholder="https://novelfire.net/book/…"
class="w-full bg-zinc-700 border border-zinc-600 rounded-lg px-3 py-2 text-zinc-100 text-sm placeholder-zinc-500 focus:outline-none focus:ring-2 focus:ring-amber-400"
/>
<div class="flex gap-2">
<input
type="number"
bind:value={rangeFrom}
min="1"
placeholder="From ch."
class="w-full bg-zinc-700 border border-zinc-600 rounded-lg px-3 py-2 text-zinc-100 text-sm placeholder-zinc-500 focus:outline-none focus:ring-2 focus:ring-amber-400"
/>
<input
type="number"
bind:value={rangeTo}
min="1"
placeholder="To ch. (opt)"
class="w-full bg-zinc-700 border border-zinc-600 rounded-lg px-3 py-2 text-zinc-100 text-sm placeholder-zinc-500 focus:outline-none focus:ring-2 focus:ring-amber-400"
/>
<button
onclick={triggerRangeScrape}
disabled={!rangeUrl.trim() || rangeFrom === null || running || ranging}
class="shrink-0 px-4 py-2 rounded-lg bg-zinc-600 text-zinc-100 font-semibold text-sm hover:bg-zinc-500 transition-colors disabled:opacity-50"
>
{ranging ? 'Queuing…' : 'Go'}
</button>
</div>
{#if rangeError}
<p class="text-sm text-red-400">{rangeError}</p>
{/if}
</div> </div>
</div> </div>
<!-- Single book scrape --> <!-- Quick-scrape genre links -->
<div class="bg-zinc-800 rounded-xl border border-zinc-700 p-5 space-y-3"> <div class="bg-zinc-800 rounded-xl border border-zinc-700 p-5 space-y-3">
<h2 class="text-sm font-semibold text-zinc-300">Scrape a single book</h2> <h2 class="text-sm font-semibold text-zinc-300">Quick genre refresh</h2>
<div class="flex gap-2"> <p class="text-xs text-zinc-500">Paste one of these into the single-book scraper to re-index a genre, or use them as starting points for range scrapes.</p>
<input <div class="flex flex-wrap gap-2">
type="url" {#each quickScrapes as qs}
bind:value={scrapeUrl} <button
placeholder="https://novelfire.net/book/..." onclick={() => { scrapeUrl = qs.url; }}
class="flex-1 bg-zinc-700 border border-zinc-600 rounded-lg px-3 py-2 text-zinc-100 text-sm placeholder-zinc-500 focus:outline-none focus:ring-2 focus:ring-amber-400" class="px-3 py-1.5 rounded-lg text-xs font-medium bg-zinc-700 text-zinc-300 border border-zinc-600 hover:border-amber-400/60 hover:text-amber-300 transition-colors"
/> >
<button {qs.label}
onclick={() => triggerScrape(scrapeUrl.trim() || undefined)} </button>
disabled={!scrapeUrl.trim() || running || scraping} {/each}
class="px-4 py-2 rounded-lg bg-zinc-600 text-zinc-100 font-semibold text-sm hover:bg-zinc-500 transition-colors disabled:opacity-50" <a
href="https://novelfire.net"
target="_blank"
rel="noopener noreferrer"
class="px-3 py-1.5 rounded-lg text-xs font-medium bg-zinc-700/50 text-zinc-400 border border-zinc-600/50 hover:text-amber-300 hover:border-amber-400/40 transition-colors"
> >
Scrape Browse novelfire.net ↗
</button> </a>
</div> </div>
{#if scrapeError}
<p class="text-sm text-red-400">{scrapeError}</p>
{/if}
</div> </div>
<!-- Tasks table --> <!-- Tasks table -->
{#if tasks.length === 0} <div class="space-y-3">
<p class="text-zinc-500 text-sm py-8 text-center">No scrape tasks yet.</p> <div class="flex items-center gap-3 flex-wrap">
{:else} <h2 class="text-lg font-semibold text-zinc-100 flex-1">Task history</h2>
<div class="overflow-x-auto rounded-xl border border-zinc-700"> <input
<table class="w-full text-sm"> type="search"
<thead class="bg-zinc-800 text-zinc-400 text-xs uppercase tracking-wide"> bind:value={q}
<tr> placeholder="Filter by kind, status or URL…"
<th class="px-4 py-3 text-left">Kind</th> class="w-full max-w-xs bg-zinc-800 border border-zinc-700 rounded-lg px-3 py-2 text-zinc-100 text-sm placeholder-zinc-500 focus:outline-none focus:ring-2 focus:ring-amber-400"
<th class="px-4 py-3 text-left">Status</th> />
<th class="px-4 py-3 text-right">Books</th>
<th class="px-4 py-3 text-right">Chapters</th>
<th class="px-4 py-3 text-right">Skipped</th>
<th class="px-4 py-3 text-right">Errors</th>
<th class="px-4 py-3 text-left">Started</th>
<th class="px-4 py-3 text-left">Duration</th>
<th class="px-4 py-3 text-left">Actions</th>
</tr>
</thead>
<tbody class="divide-y divide-zinc-700/50">
{#each tasks as task}
<tr class="bg-zinc-900 hover:bg-zinc-800/50 transition-colors">
<td class="px-4 py-3 font-mono text-xs text-zinc-300">
{task.kind}
{#if task.target_url}
<br />
<span class="text-zinc-500 truncate max-w-[16rem] block" title={task.target_url}>
{task.target_url.replace('https://novelfire.net/book/', '')}
</span>
{/if}
</td>
<td class="px-4 py-3">
<span class="font-medium {statusColor(task.status)}">{task.status}</span>
</td>
<td class="px-4 py-3 text-right text-zinc-300">{task.books_found ?? 0}</td>
<td class="px-4 py-3 text-right text-zinc-300">{task.chapters_scraped ?? 0}</td>
<td class="px-4 py-3 text-right text-zinc-400">{task.chapters_skipped ?? 0}</td>
<td class="px-4 py-3 text-right {task.errors > 0 ? 'text-red-400' : 'text-zinc-400'}">{task.errors ?? 0}</td>
<td class="px-4 py-3 text-zinc-400">{fmtDate(task.started)}</td>
<td class="px-4 py-3 text-zinc-400">{duration(task.started, task.finished)}</td>
<td class="px-4 py-3">
{#if task.status === 'pending'}
<button
onclick={() => cancelTask(task.id)}
disabled={cancellingIds.has(task.id)}
class="px-2 py-1 rounded text-xs font-medium bg-zinc-700 text-zinc-300 hover:bg-red-900 hover:text-red-300 transition-colors disabled:opacity-50"
title="Cancel this task"
>
{cancellingIds.has(task.id) ? 'Cancelling…' : 'Cancel'}
</button>
{#if cancelErrors[task.id]}
<p class="text-xs text-red-400 mt-1">{cancelErrors[task.id]}</p>
{/if}
{/if}
</td>
</tr>
{#if task.error_message}
<tr class="bg-red-950/20">
<td colspan="9" class="px-4 py-2 text-xs text-red-400 font-mono">{task.error_message}</td>
</tr>
{/if}
{/each}
</tbody>
</table>
</div> </div>
{/if}
{#if filtered.length === 0}
<p class="text-zinc-500 text-sm py-8 text-center">
{q.trim() ? 'No matching tasks.' : 'No scrape tasks yet.'}
</p>
{:else}
<!-- Desktop table -->
<div class="hidden sm:block overflow-x-auto rounded-xl border border-zinc-700">
<table class="w-full text-sm">
<thead class="bg-zinc-800 text-zinc-400 text-xs uppercase tracking-wide">
<tr>
<th class="px-4 py-3 text-left">Kind / URL</th>
<th class="px-4 py-3 text-left">Status</th>
<th class="px-4 py-3 text-right">Books</th>
<th class="px-4 py-3 text-right">Chapters</th>
<th class="px-4 py-3 text-right">Skipped</th>
<th class="px-4 py-3 text-right">Errors</th>
<th class="px-4 py-3 text-left">Started</th>
<th class="px-4 py-3 text-left">Duration</th>
<th class="px-4 py-3 text-left">Actions</th>
</tr>
</thead>
<tbody class="divide-y divide-zinc-700/50">
{#each filtered as task}
<tr class="bg-zinc-900 hover:bg-zinc-800/50 transition-colors">
<td class="px-4 py-3 font-mono text-xs text-zinc-300">
{task.kind}
{#if task.target_url}
<br />
<span class="text-zinc-500 truncate max-w-[16rem] block" title={task.target_url}>
{task.target_url.replace('https://novelfire.net/book/', '')}
</span>
{/if}
</td>
<td class="px-4 py-3">
<span class="font-medium {statusColor(task.status)}">{task.status}</span>
</td>
<td class="px-4 py-3 text-right text-zinc-300">{task.books_found ?? 0}</td>
<td class="px-4 py-3 text-right text-zinc-300">{task.chapters_scraped ?? 0}</td>
<td class="px-4 py-3 text-right text-zinc-400">{task.chapters_skipped ?? 0}</td>
<td class="px-4 py-3 text-right {task.errors > 0 ? 'text-red-400' : 'text-zinc-400'}">{task.errors ?? 0}</td>
<td class="px-4 py-3 text-zinc-400 whitespace-nowrap">{fmtDate(task.started)}</td>
<td class="px-4 py-3 text-zinc-400 whitespace-nowrap">{duration(task.started, task.finished)}</td>
<td class="px-4 py-3">
<div class="flex flex-wrap gap-1.5">
{#if task.status === 'pending'}
<button
onclick={() => cancelTask(task.id)}
disabled={cancellingIds.has(task.id)}
class="px-2 py-1 rounded text-xs font-medium bg-zinc-700 text-zinc-300 hover:bg-red-900 hover:text-red-300 transition-colors disabled:opacity-50"
>
{cancellingIds.has(task.id) ? 'Cancelling…' : 'Cancel'}
</button>
{/if}
{#if task.kind === 'book_range' && task.status !== 'pending' && task.status !== 'running' && (task.chapters_scraped ?? 0) > 0}
<button
onclick={() => continueTask(task)}
class="px-2 py-1 rounded text-xs font-medium bg-amber-900/60 text-amber-300 hover:bg-amber-800/60 transition-colors"
>
Continue ▶
</button>
{/if}
{#if (task.status === 'failed' || task.status === 'cancelled') && task.kind !== 'catalogue'}
<button
onclick={() => retryTask(task)}
class="px-2 py-1 rounded text-xs font-medium bg-sky-900/60 text-sky-300 hover:bg-sky-800/60 transition-colors"
>
Retry ↺
</button>
{/if}
{#if cancelErrors[task.id]}
<p class="text-xs text-red-400 mt-1 w-full">{cancelErrors[task.id]}</p>
{/if}
</div>
</td>
</tr>
{#if task.error_message}
<tr class="bg-red-950/20">
<td colspan="9" class="px-4 py-2 text-xs text-red-400 font-mono">{task.error_message}</td>
</tr>
{/if}
{/each}
</tbody>
</table>
</div>
<!-- Mobile cards -->
<div class="sm:hidden space-y-3">
{#each filtered as task}
<div class="bg-zinc-900 rounded-xl border border-zinc-700 p-4 space-y-2">
<div class="flex items-start justify-between gap-2">
<div class="min-w-0">
<span class="font-mono text-xs text-zinc-300">{task.kind}</span>
{#if task.target_url}
<p class="text-xs text-zinc-500 truncate mt-0.5" title={task.target_url}>
{task.target_url.replace('https://novelfire.net/book/', '')}
</p>
{/if}
</div>
<span class="shrink-0 text-xs font-semibold {statusColor(task.status)}">{task.status}</span>
</div>
<div class="grid grid-cols-2 gap-1 text-xs">
<span class="text-zinc-500">Books</span><span class="text-zinc-300 text-right">{task.books_found ?? 0}</span>
<span class="text-zinc-500">Chapters</span><span class="text-zinc-300 text-right">{task.chapters_scraped ?? 0}</span>
<span class="text-zinc-500">Skipped</span><span class="text-zinc-400 text-right">{task.chapters_skipped ?? 0}</span>
<span class="text-zinc-500">Errors</span><span class="{task.errors > 0 ? 'text-red-400' : 'text-zinc-400'} text-right">{task.errors ?? 0}</span>
<span class="text-zinc-500">Started</span><span class="text-zinc-400 text-right">{fmtDate(task.started)}</span>
<span class="text-zinc-500">Duration</span><span class="text-zinc-400 text-right">{duration(task.started, task.finished)}</span>
</div>
{#if task.error_message}
<p class="text-xs text-red-400 font-mono break-all">{task.error_message}</p>
{/if}
<div class="flex flex-wrap gap-2">
{#if task.status === 'pending'}
<button
onclick={() => cancelTask(task.id)}
disabled={cancellingIds.has(task.id)}
class="flex-1 px-3 py-1.5 rounded-lg text-xs font-medium bg-zinc-700 text-zinc-300 hover:bg-red-900 hover:text-red-300 transition-colors disabled:opacity-50"
>
{cancellingIds.has(task.id) ? 'Cancelling…' : 'Cancel task'}
</button>
{/if}
{#if task.kind === 'book_range' && task.status !== 'pending' && task.status !== 'running' && (task.chapters_scraped ?? 0) > 0}
<button
onclick={() => continueTask(task)}
class="flex-1 px-3 py-1.5 rounded-lg text-xs font-medium bg-amber-900/60 text-amber-300 hover:bg-amber-800/60 transition-colors"
>
Continue ▶
</button>
{/if}
{#if (task.status === 'failed' || task.status === 'cancelled') && task.kind !== 'catalogue'}
<button
onclick={() => retryTask(task)}
class="flex-1 px-3 py-1.5 rounded-lg text-xs font-medium bg-sky-900/60 text-sky-300 hover:bg-sky-800/60 transition-colors"
>
Retry ↺
</button>
{/if}
{#if cancelErrors[task.id]}
<p class="text-xs text-red-400 w-full">{cancelErrors[task.id]}</p>
{/if}
</div>
</div>
{/each}
</div>
{/if}
</div>
</div> </div>

View File

@@ -2,7 +2,7 @@ import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types'; import type { RequestHandler } from './$types';
import { getBook, listChapterIdx, getProgress, isBookSaved } from '$lib/server/pocketbase'; import { getBook, listChapterIdx, getProgress, isBookSaved } from '$lib/server/pocketbase';
import { log } from '$lib/server/logger'; import { log } from '$lib/server/logger';
import { backendFetch } from '$lib/server/scraper'; import { backendFetch, type BookPreviewResponse } from '$lib/server/scraper';
/** /**
* GET /api/book/[slug] * GET /api/book/[slug]
@@ -61,21 +61,7 @@ export const GET: RequestHandler = async ({ params, locals }) => {
} }
// 200 — book was already in library // 200 — book was already in library
const preview: { const preview: BookPreviewResponse = await res.json();
in_lib: boolean;
meta: {
slug: string;
title: string;
author: string;
cover: string;
status: string;
genres: string[];
summary: string;
total_chapters: number;
source_url: string;
};
chapters: { number: number; title: string; date?: string }[];
} = await res.json();
const previewBook = { const previewBook = {
id: '', id: '',

View File

@@ -1,76 +0,0 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { log } from '$lib/server/logger';
import { backendFetch } from '$lib/server/scraper';
/**
* GET /api/browse-page?page=2&genre=all&sort=popular&status=all&q=
*
* Thin proxy to the Go backend's /api/catalogue endpoint.
* Used by the infinite-scroll browse page to append subsequent pages
* without a full SSR navigation.
*
* Normalises the catalogue response to the { novels, page, hasNext } shape
* expected by the client-side infinite scroll in +page.svelte.
*/
export const GET: RequestHandler = async ({ url }) => {
const page = url.searchParams.get('page') ?? '1';
const genre = url.searchParams.get('genre') ?? 'all';
const sort = url.searchParams.get('sort') ?? 'popular';
const status = url.searchParams.get('status') ?? 'all';
const q = url.searchParams.get('q') ?? '';
const params = new URLSearchParams({ page, genre, sort, status });
if (q.trim().length >= 2) {
params.set('q', q.trim());
}
try {
const res = await backendFetch(`/api/catalogue?${params.toString()}`);
if (!res.ok) {
log.error('browse-page', 'backend returned error', { status: res.status });
throw error(502, `Browse fetch failed: ${res.status}`);
}
const data: {
books: Array<{
slug: string;
title: string;
author: string;
cover: string;
status: string;
genres: string[];
total_chapters: number;
source_url: string;
ranking: number;
rating: number;
}>;
page: number;
has_next: boolean;
} = await res.json();
// Normalise to the shape the +page.svelte client-side fetch expects.
const novels = (data.books ?? []).map((book) => ({
slug: book.slug,
title: book.title,
cover: book.cover,
rank: book.ranking > 0 ? `#${book.ranking}` : '',
rating: book.rating > 0 ? String(book.rating) : '',
chapters: book.total_chapters > 0 ? `${book.total_chapters} chapters` : '',
url: book.source_url ?? '',
author: book.author,
status: book.status,
genres: book.genres ?? [],
source_url: book.source_url
}));
return json({
novels,
page: data.page ?? (parseInt(page, 10) || 1),
hasNext: data.has_next ?? false
});
} catch (e) {
if (e instanceof Error && 'status' in e) throw e;
log.error('browse-page', 'network error', { err: String(e) });
throw error(502, 'Could not reach browse service');
}
};

View File

@@ -0,0 +1,47 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { log } from '$lib/server/logger';
import { backendFetch } from '$lib/server/scraper';
import { bookToListing, type CatalogueResponse } from '$lib/server/catalogue';
/**
* GET /api/catalogue-page?page=2&genre=all&sort=popular&status=all&q=
*
* Thin proxy to the Go backend's /api/catalogue endpoint.
* Used by the infinite-scroll catalogue page to append subsequent pages
* without a full SSR navigation.
*
* Returns { novels, page, hasNext } — the shape expected by the client-side
* infinite scroll in +page.svelte.
*/
export const GET: RequestHandler = async ({ url }) => {
const page = url.searchParams.get('page') ?? '1';
const genre = url.searchParams.get('genre') ?? 'all';
const sort = url.searchParams.get('sort') ?? 'popular';
const status = url.searchParams.get('status') ?? 'all';
const q = url.searchParams.get('q') ?? '';
const params = new URLSearchParams({ page, genre, sort, status });
if (q.trim().length >= 2) {
params.set('q', q.trim());
}
try {
const res = await backendFetch(`/api/catalogue?${params.toString()}`);
if (!res.ok) {
log.error('catalogue-page', 'backend returned error', { status: res.status });
throw error(502, `Catalogue fetch failed: ${res.status}`);
}
const data: CatalogueResponse = await res.json();
return json({
novels: (data.books ?? []).map(bookToListing),
page: data.page ?? (parseInt(page, 10) || 1),
hasNext: data.has_next ?? false
});
} catch (e) {
if (e instanceof Error && 'status' in e) throw e;
log.error('catalogue-page', 'network error', { err: String(e) });
throw error(502, 'Could not reach catalogue service');
}
};

View File

@@ -1,65 +1,56 @@
import { json, error } from '@sveltejs/kit'; import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types'; import type { RequestHandler } from './$types';
import { presignAvatarUploadUrl, presignAvatarUrl } from '$lib/server/minio'; import { presignAvatarUrl } from '$lib/server/minio';
import { updateUserAvatarUrl, getUserByUsername } from '$lib/server/pocketbase'; import { updateUserAvatarUrl, getUserByUsername } from '$lib/server/pocketbase';
import { backendFetch } from '$lib/server/scraper';
const ALLOWED_TYPES = ['image/jpeg', 'image/png', 'image/webp']; const ALLOWED_TYPES = ['image/jpeg', 'image/png', 'image/webp'];
/** /**
* POST /api/profile/avatar * POST /api/profile/avatar
* Body: JSON { mime_type: "image/jpeg" | "image/png" | "image/webp" } * Body: raw image bytes (Content-Type: image/jpeg | image/png | image/webp)
* *
* Returns a short-lived presigned PUT URL pointing at MinIO (public endpoint) * Uploads the image to MinIO via the Go backend (server-to-server, no browser
* so the client can upload the image bytes directly, bypassing the server. * → MinIO direct upload), records the key in PocketBase, and returns a fresh
* After the PUT completes, the client must call PATCH /api/profile/avatar
* with the returned key to record it in PocketBase.
*
* Returns: { upload_url: string, key: string }
*/
export const POST: RequestHandler = async ({ request, locals }) => {
if (!locals.user) error(401, 'Not authenticated');
let mimeType = 'image/jpeg';
try {
const body = await request.json();
if (body?.mime_type) mimeType = body.mime_type;
} catch {
// default to jpeg if body is missing/invalid
}
if (!ALLOWED_TYPES.includes(mimeType)) {
error(400, `Unsupported image type: ${mimeType}. Allowed: jpeg, png, webp`);
}
const { uploadUrl, key } = await presignAvatarUploadUrl(locals.user.id, mimeType);
return json({ upload_url: uploadUrl, key });
};
/**
* PATCH /api/profile/avatar
* Body: JSON { key: string }
*
* Called after the client has successfully PUT the image to MinIO via the
* presigned URL. Records the object key in PocketBase and returns a fresh
* presigned GET URL for immediate display. * presigned GET URL for immediate display.
* *
* Returns: { avatar_url: string | null } * Returns: { avatar_url: string | null }
*/ */
export const PATCH: RequestHandler = async ({ request, locals }) => { export const POST: RequestHandler = async ({ request, locals }) => {
if (!locals.user) error(401, 'Not authenticated'); if (!locals.user) error(401, 'Not authenticated');
let key: string | undefined; const ct = request.headers.get('Content-Type') ?? '';
try { // Strip parameters (e.g. "image/jpeg; charset=utf-8" → "image/jpeg")
const body = await request.json(); const mimeType = ct.split(';')[0].trim();
if (typeof body?.key === 'string') key = body.key;
} catch { if (!ALLOWED_TYPES.includes(mimeType)) {
error(400, 'Invalid JSON body'); error(400, `Unsupported image type. Allowed: image/jpeg, image/png, image/webp`);
} }
if (!key) error(400, 'Missing "key" field'); // Read the raw body
const blob = await request.arrayBuffer();
if (blob.byteLength === 0) error(400, 'Empty image body');
if (blob.byteLength > 5 * 1024 * 1024) error(413, 'Image too large (max 5 MiB)');
// Forward directly to Go backend — server-to-server, so internal MinIO is reachable.
const uploadRes = await backendFetch(
`/api/avatar-upload/${encodeURIComponent(locals.user.id)}`,
{
method: 'PUT',
headers: { 'Content-Type': mimeType },
body: blob
}
);
if (!uploadRes.ok) {
const text = await uploadRes.text().catch(() => '');
error(uploadRes.status as 400 | 500, `Upload failed: ${text || uploadRes.statusText}`);
}
const { key } = (await uploadRes.json()) as { key: string };
// Record object key in PocketBase.
await updateUserAvatarUrl(locals.user.id, key); await updateUserAvatarUrl(locals.user.id, key);
// Return a fresh presigned GET URL for immediate display.
const avatarUrl = await presignAvatarUrl(locals.user.id); const avatarUrl = await presignAvatarUrl(locals.user.id);
return json({ avatar_url: avatarUrl }); return json({ avatar_url: avatarUrl });
}; };

View File

@@ -0,0 +1,19 @@
import { json, error } from '@sveltejs/kit';
import type { RequestHandler } from './$types';
import { getScrapingTask } from '$lib/server/pocketbase';
/**
* GET /api/scrape/task/[id]
*
* Returns { id, status, error_message } for a single scraping task.
* Used by the book detail page to poll for task completion.
*/
export const GET: RequestHandler = async ({ params }) => {
const { id } = params;
if (!id) throw error(400, 'Missing task id');
const task = await getScrapingTask(id).catch(() => null);
if (!task) throw error(404, 'Task not found');
return json({ id: task.id, status: task.status, error_message: task.error_message ?? '' });
};

View File

@@ -31,7 +31,7 @@
<p class="text-lg">Your library is empty.</p> <p class="text-lg">Your library is empty.</p>
<p class="text-sm mt-2"> <p class="text-sm mt-2">
Books you start reading or save from Books you start reading or save from
<a href="/browse" class="text-amber-400 hover:text-amber-300 transition-colors">Discover</a> <a href="/catalogue" class="text-amber-400 hover:text-amber-300 transition-colors">Discover</a>
will appear here. will appear here.
</p> </p>
</div> </div>

View File

@@ -2,7 +2,7 @@ import { error } from '@sveltejs/kit';
import type { PageServerLoad } from './$types'; import type { PageServerLoad } from './$types';
import { getBook, listChapterIdx, getProgress, isBookSaved } from '$lib/server/pocketbase'; import { getBook, listChapterIdx, getProgress, isBookSaved } from '$lib/server/pocketbase';
import { log } from '$lib/server/logger'; import { log } from '$lib/server/logger';
import { backendFetch } from '$lib/server/scraper'; import { backendFetch, type BookPreviewResponse } from '$lib/server/scraper';
export const load: PageServerLoad = async ({ params, locals }) => { export const load: PageServerLoad = async ({ params, locals }) => {
const { slug } = params; const { slug } = params;
@@ -70,21 +70,7 @@ export const load: PageServerLoad = async ({ params, locals }) => {
} }
// 200 — book was already in library when backend checked // 200 — book was already in library when backend checked
const preview: { const preview: BookPreviewResponse = await res.json();
in_lib: boolean;
meta: {
slug: string;
title: string;
author: string;
cover: string;
status: string;
genres: string[];
summary: string;
total_chapters: number;
source_url: string;
};
chapters: { number: number; title: string; date?: string }[];
} = await res.json();
const previewBook = { const previewBook = {
id: '', id: '',

View File

@@ -1,5 +1,6 @@
<script lang="ts"> <script lang="ts">
import { untrack } from 'svelte'; import { untrack } from 'svelte';
import { invalidateAll } from '$app/navigation';
import type { PageData } from './$types'; import type { PageData } from './$types';
import CommentsSection from '$lib/components/CommentsSection.svelte'; import CommentsSection from '$lib/components/CommentsSection.svelte';
@@ -93,6 +94,33 @@
// ── Admin panel expand/collapse ─────────────────────────────────────────── // ── Admin panel expand/collapse ───────────────────────────────────────────
let adminOpen = $state(false); let adminOpen = $state(false);
// ── Auto-poll when scrape task is in flight ───────────────────────────────
// When the backend enqueues a scrape for an unseen book, the page shows a
// spinner. Poll every 3 s until the task reaches "done" or "failed", then
// reload the full page data so chapters appear automatically.
$effect(() => {
if (!data.scraping || !data.taskId) return;
const INTERVAL_MS = 3000;
const id = data.taskId;
const timer = setInterval(async () => {
try {
const res = await fetch(`/api/scrape/task/${encodeURIComponent(id)}`);
if (!res.ok) return;
const body: { id: string; status: string } = await res.json();
if (body.status === 'done' || body.status === 'failed') {
clearInterval(timer);
await invalidateAll();
}
} catch {
// network blip — try again next tick
}
}, INTERVAL_MS);
return () => clearInterval(timer);
});
</script> </script>
<svelte:head> <svelte:head>
@@ -109,7 +137,7 @@
<div> <div>
<p class="text-zinc-200 font-semibold text-lg">Scraping in progress…</p> <p class="text-zinc-200 font-semibold text-lg">Scraping in progress…</p>
<p class="text-zinc-500 text-sm mt-1"> <p class="text-zinc-500 text-sm mt-1">
Fetching the first 20 chapters. Refresh the page in a minute. Fetching the first 20 chapters. This page will refresh automatically.
</p> </p>
{#if data.taskId} {#if data.taskId}
<p class="text-zinc-600 text-xs mt-2 font-mono">task: {data.taskId}</p> <p class="text-zinc-600 text-xs mt-2 font-mono">task: {data.taskId}</p>

View File

@@ -2,60 +2,13 @@ import { error } from '@sveltejs/kit';
import type { PageServerLoad, Actions } from './$types'; import type { PageServerLoad, Actions } from './$types';
import { log } from '$lib/server/logger'; import { log } from '$lib/server/logger';
import { backendFetch } from '$lib/server/scraper'; import { backendFetch } from '$lib/server/scraper';
import {
bookToListing,
type CatalogueResponse,
type NovelListing
} from '$lib/server/catalogue';
export interface NovelListing { export type { NovelListing };
slug: string;
title: string;
cover: string;
rank: string;
rating: string;
chapters: string;
url: string;
// enriched fields
author?: string;
status?: string;
genres?: string[];
source_url?: string;
}
// Shape returned by GET /api/catalogue on the Go scraper.
interface CatalogueBook {
slug: string;
title: string;
author: string;
cover: string;
status: string;
genres: string[];
summary: string;
total_chapters: number;
source_url: string;
ranking: number;
rating: number;
}
interface CatalogueResponse {
books: CatalogueBook[];
page: number;
limit: number;
total: number;
has_next: boolean;
}
function bookToListing(book: CatalogueBook): NovelListing {
return {
slug: book.slug,
title: book.title,
cover: book.cover,
rank: book.ranking > 0 ? `#${book.ranking}` : '',
rating: book.rating > 0 ? String(book.rating) : '',
chapters: book.total_chapters > 0 ? `${book.total_chapters} chapters` : '',
url: book.source_url ?? '',
author: book.author,
status: book.status,
genres: book.genres ?? [],
source_url: book.source_url
};
}
export const load: PageServerLoad = async ({ url, locals }) => { export const load: PageServerLoad = async ({ url, locals }) => {
const page = url.searchParams.get('page') ?? '1'; const page = url.searchParams.get('page') ?? '1';
@@ -72,20 +25,27 @@ export const load: PageServerLoad = async ({ url, locals }) => {
let novels: NovelListing[] = []; let novels: NovelListing[] = [];
let pageNum = parseInt(page, 10) || 1; let pageNum = parseInt(page, 10) || 1;
let hasNext = false; let hasNext = false;
let total = 0;
// Dynamic facets from Meilisearch — fall back to empty arrays if unavailable.
let genres: string[] = [];
let statuses: string[] = [];
try { try {
const res = await backendFetch(`/api/catalogue?${params.toString()}`); const res = await backendFetch(`/api/catalogue?${params.toString()}`);
if (!res.ok) { if (!res.ok) {
log.error('browse', 'catalogue returned error', { status: res.status }); log.error('catalogue', 'catalogue returned error', { status: res.status });
throw error(502, `Catalogue fetch failed: ${res.status}`); throw error(502, `Catalogue fetch failed: ${res.status}`);
} }
const data: CatalogueResponse = await res.json(); const data: CatalogueResponse = await res.json();
novels = (data.books ?? []).map(bookToListing); novels = (data.books ?? []).map(bookToListing);
pageNum = data.page ?? 1; pageNum = data.page ?? 1;
hasNext = data.has_next ?? false; hasNext = data.has_next ?? false;
total = data.total ?? 0;
genres = data.facets?.genres ?? [];
statuses = data.facets?.statuses ?? [];
} catch (e) { } catch (e) {
if (e instanceof Error && 'status' in e) throw e; if (e instanceof Error && 'status' in e) throw e;
log.error('browse', 'catalogue network error', { err: String(e) }); log.error('catalogue', 'catalogue network error', { err: String(e) });
throw error(502, 'Could not reach catalogue service'); throw error(502, 'Could not reach catalogue service');
} }
@@ -93,9 +53,14 @@ export const load: PageServerLoad = async ({ url, locals }) => {
novels, novels,
page: pageNum, page: pageNum,
hasNext, hasNext,
total,
genre, genre,
sort, sort,
status, status,
// Dynamic filter options from Meilisearch facet distribution.
// Empty arrays when Meilisearch is unavailable — UI falls back to hardcoded lists.
genres,
statuses,
isAdmin: locals.user?.role === 'admin', isAdmin: locals.user?.role === 'admin',
searchQuery: q.trim().length >= 2 ? q.trim() : '', searchQuery: q.trim().length >= 2 ? q.trim() : '',
searchLocalCount: 0, searchLocalCount: 0,

View File

@@ -54,7 +54,7 @@
sort: data.sort, sort: data.sort,
status: data.status status: data.status
}); });
const res = await fetch(`/api/browse-page?${params.toString()}`); const res = await fetch(`/api/catalogue-page?${params.toString()}`);
if (!res.ok) return; if (!res.ok) return;
const body: { novels: NovelListing[]; page: number; hasNext: boolean } = await res.json(); const body: { novels: NovelListing[]; page: number; hasNext: boolean } = await res.json();
novels = [...novels, ...(body.novels ?? [])]; novels = [...novels, ...(body.novels ?? [])];
@@ -82,37 +82,35 @@
return () => observer.disconnect(); return () => observer.disconnect();
}); });
// Filter options // Filter options — built from Meilisearch facet distribution when available,
const genres = [ // with a hardcoded fallback list for when Meilisearch is not yet populated.
{ value: 'all', label: 'All Genres' }, const FALLBACK_GENRES = [
{ value: 'action', label: 'Action' }, 'action', 'adventure', 'comedy', 'drama', 'fantasy', 'harem',
{ value: 'adventure', label: 'Adventure' }, 'historical', 'horror', 'isekai', 'martial-arts', 'mystery',
{ value: 'comedy', label: 'Comedy' }, 'psychological', 'romance', 'sci-fi', 'system', 'xianxia'
{ value: 'drama', label: 'Drama' },
{ value: 'fantasy', label: 'Fantasy' },
{ value: 'harem', label: 'Harem' },
{ value: 'historical', label: 'Historical' },
{ value: 'horror', label: 'Horror' },
{ value: 'isekai', label: 'Isekai' },
{ value: 'martial-arts', label: 'Martial Arts' },
{ value: 'mystery', label: 'Mystery' },
{ value: 'psychological', label: 'Psychological' },
{ value: 'romance', label: 'Romance' },
{ value: 'sci-fi', label: 'Sci-Fi' },
{ value: 'system', label: 'System' },
{ value: 'xianxia', label: 'Xianxia' }
]; ];
const genres = $derived([
{ value: 'all', label: 'All Genres' },
...((data.genres?.length ? data.genres : FALLBACK_GENRES).map((g: string) => ({
value: g,
label: g.charAt(0).toUpperCase() + g.slice(1).replace(/-/g, ' ')
})))
]);
const sorts = [ const sorts = [
{ value: 'popular', label: 'Popular' }, { value: 'popular', label: 'Popular' },
{ value: 'new', label: 'New' }, { value: 'new', label: 'New' },
{ value: 'update', label: 'Updated' }, { value: 'update', label: 'Updated' },
{ value: 'top-rated', label: 'Top Rated' },
{ value: 'rank', label: 'Ranking' } { value: 'rank', label: 'Ranking' }
]; ];
const statuses = [ const FALLBACK_STATUSES = ['ongoing', 'completed'];
const statuses = $derived([
{ value: 'all', label: 'All' }, { value: 'all', label: 'All' },
{ value: 'ongoing', label: 'Ongoing' }, ...((data.statuses?.length ? data.statuses : FALLBACK_STATUSES).map((s: string) => ({
{ value: 'completed', label: 'Completed' } value: s,
]; label: s.charAt(0).toUpperCase() + s.slice(1)
})))
]);
// When sort=rank the ranking API is used — pagination + genre/status filters // When sort=rank the ranking API is used — pagination + genre/status filters
// don't apply to that endpoint. // don't apply to that endpoint.
@@ -173,7 +171,7 @@
let filtersOpen = $state(false); let filtersOpen = $state(false);
// Human-readable summary of active filters shown on the toggle button // Human-readable summary of active filters shown on the toggle button
const filterSummary = $derived(() => { const filterSummary = $derived((() => {
const parts: string[] = []; const parts: string[] = [];
const sortLabel = sorts.find((s) => s.value === data.sort)?.label ?? data.sort; const sortLabel = sorts.find((s) => s.value === data.sort)?.label ?? data.sort;
parts.push(sortLabel); parts.push(sortLabel);
@@ -186,7 +184,7 @@
parts.push(statusLabel); parts.push(statusLabel);
} }
return parts.join(' · '); return parts.join(' · ');
}); })());
// Whether any non-default filter is active (used to show a dot indicator) // Whether any non-default filter is active (used to show a dot indicator)
const hasActiveFilters = $derived( const hasActiveFilters = $derived(
@@ -207,12 +205,12 @@
</script> </script>
<svelte:head> <svelte:head>
<title>Discover — libnovel</title> <title>Catalogue — libnovel</title>
</svelte:head> </svelte:head>
<!-- Header --> <!-- Header -->
<div class="mb-4"> <div class="mb-4">
<h1 class="text-2xl font-bold text-zinc-100">Discover</h1> <h1 class="text-2xl font-bold text-zinc-100">Catalogue</h1>
<p class="text-zinc-400 text-sm mt-1"> <p class="text-zinc-400 text-sm mt-1">
{#if isSearchView} {#if isSearchView}
{novels.length} result{novels.length !== 1 ? 's' : ''} for "<span class="text-zinc-200">{data.searchQuery}</span>" {novels.length} result{novels.length !== 1 ? 's' : ''} for "<span class="text-zinc-200">{data.searchQuery}</span>"
@@ -251,7 +249,7 @@
<!-- Toolbar: search + filter toggle + view toggle + admin refresh --> <!-- Toolbar: search + filter toggle + view toggle + admin refresh -->
<div class="flex gap-2 mb-3"> <div class="flex gap-2 mb-3">
<!-- Search (grows to fill available space) --> <!-- Search (grows to fill available space) -->
<form method="GET" action="/browse" class="flex flex-1 gap-2 min-w-0"> <form method="GET" action="/catalogue" class="flex flex-1 gap-2 min-w-0">
<input <input
type="search" type="search"
name="q" name="q"
@@ -267,7 +265,7 @@
</button> </button>
{#if data.searchQuery} {#if data.searchQuery}
<a <a
href="/browse" href="/catalogue"
class="px-3 py-2 rounded bg-zinc-700 text-zinc-300 text-sm hover:bg-zinc-600 transition-colors whitespace-nowrap" class="px-3 py-2 rounded bg-zinc-700 text-zinc-300 text-sm hover:bg-zinc-600 transition-colors whitespace-nowrap"
> >
Clear Clear
@@ -351,8 +349,8 @@
<!-- Active filter summary (shown when panel is closed and filters are active) --> <!-- Active filter summary (shown when panel is closed and filters are active) -->
{#if !filtersOpen && hasActiveFilters} {#if !filtersOpen && hasActiveFilters}
<p class="text-xs text-zinc-500 mb-3"> <p class="text-xs text-zinc-500 mb-3">
<span class="text-zinc-400">{filterSummary()}</span> <span class="text-zinc-400">{filterSummary}</span>
<a href="/browse" class="ml-2 text-zinc-600 hover:text-zinc-400 underline underline-offset-2">clear</a> <a href="/catalogue" class="ml-2 text-zinc-600 hover:text-zinc-400 underline underline-offset-2">clear</a>
</p> </p>
{/if} {/if}
@@ -382,7 +380,7 @@
</form> </form>
{/if} {/if}
<form method="GET" action="/browse" class="mb-4 p-3 rounded-lg bg-zinc-800/60 border border-zinc-700 flex flex-col gap-3"> <form method="GET" action="/catalogue" class="mb-4 p-3 rounded-lg bg-zinc-800/60 border border-zinc-700 flex flex-col gap-3">
<input type="hidden" name="page" value="1" /> <input type="hidden" name="page" value="1" />
<div class="grid grid-cols-1 sm:grid-cols-3 gap-3"> <div class="grid grid-cols-1 sm:grid-cols-3 gap-3">
@@ -436,7 +434,7 @@
{/if} {/if}
<div class="flex gap-2 justify-end"> <div class="flex gap-2 justify-end">
<a href="/browse" class="px-4 py-2 rounded bg-zinc-700 text-zinc-300 text-sm hover:bg-zinc-600 transition-colors"> <a href="/catalogue" class="px-4 py-2 rounded bg-zinc-700 text-zinc-300 text-sm hover:bg-zinc-600 transition-colors">
Reset Reset
</a> </a>
<button <button

View File

@@ -31,42 +31,18 @@
avatarUploading = true; avatarUploading = true;
avatarError = ''; avatarError = '';
try { try {
// Step 1: get presigned PUT URL // POST raw bytes to the SvelteKit server, which proxies to MinIO internally.
const presignRes = await fetch('/api/profile/avatar', { const res = await fetch('/api/profile/avatar', {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ mime_type: mimeType })
});
if (!presignRes.ok) {
const body = await presignRes.json().catch(() => ({})) as { message?: string };
avatarError = body.message ?? `Failed to prepare upload (${presignRes.status})`;
return;
}
const { upload_url, key } = await presignRes.json() as { upload_url: string; key: string };
// Step 2: PUT blob directly to MinIO
const putRes = await fetch(upload_url, {
method: 'PUT',
headers: { 'Content-Type': mimeType }, headers: { 'Content-Type': mimeType },
body: blob body: blob
}); });
if (!putRes.ok) { if (!res.ok) {
avatarError = `Upload failed (${putRes.status})`; const body = await res.json().catch(() => ({})) as { message?: string };
avatarError = body.message ?? `Upload failed (${res.status})`;
return; return;
} }
const result = await res.json() as { avatar_url: string | null };
// Step 3: record key in PocketBase and get fresh presigned GET URL
const patchRes = await fetch('/api/profile/avatar', {
method: 'PATCH',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ key })
});
if (!patchRes.ok) {
const body = await patchRes.json().catch(() => ({})) as { message?: string };
avatarError = body.message ?? `Failed to save avatar (${patchRes.status})`;
return;
}
const result = await patchRes.json() as { avatar_url: string | null };
avatarUrl = result.avatar_url; avatarUrl = result.avatar_url;
} catch { } catch {
avatarError = 'Network error during upload'; avatarError = 'Network error during upload';