diff --git a/v3/Caddyfile b/v3/Caddyfile index b538844..1dfb9b4 100644 --- a/v3/Caddyfile +++ b/v3/Caddyfile @@ -8,31 +8,53 @@ # Use "localhost" for local dev (no TLS cert attempted). # CADDY_ACME_EMAIL — Let's Encrypt notification email (empty = no email) # -# Routing rules: +# Routing rules (main domain): # /health → backend:8080 (liveness probe) # /scrape* → backend:8080 (Go admin scrape endpoints) -# /api/browse → backend:8080 (MinIO-cached browse pages) # /api/book-preview/* → backend:8080 (live scrape, no store write) # /api/chapter-text/* → backend:8080 (chapter markdown from MinIO) +# /api/chapter-markdown/* → backend:8080 (chapter markdown from MinIO) # /api/reindex/* → backend:8080 (rebuild chapter index) # /api/cover/* → backend:8080 (proxy cover image) # /api/audio-proxy/* → backend:8080 (proxy generated audio) # /avatars/* → minio:9000 (presigned avatar GETs) +# /audio/* → minio:9000 (presigned audio GETs) +# /chapters/* → minio:9000 (presigned chapter GETs) # /* (everything else) → ui:3000 (SvelteKit — handles all # remaining /api/* routes) # +# Subdomain routing: +# feedback.libnovel.cc → fider:3000 (user feedback / feature requests) +# errors.libnovel.cc → glitchtip-web:8000 (error tracking) +# analytics.libnovel.cc → umami:3000 (page analytics) +# logs.libnovel.cc → dozzle:8080 (Docker log viewer) +# uptime.libnovel.cc → uptime-kuma:3001 (uptime monitoring) +# push.libnovel.cc → gotify:80 (push notifications) +# # Routes intentionally removed from direct-to-backend: # /api/scrape/* — SvelteKit has /api/scrape/ counterparts # that enforce auth; routing directly would # bypass SK middleware. # /api/chapter-text-preview/* — Same: SvelteKit owns # /api/chapter-text-preview/[slug]/[n]. +# /api/browse — Endpoint removed; browse snapshot system +# was deleted. { # Email for Let's Encrypt ACME account registration. # When CADDY_ACME_EMAIL is set this expands to e.g. "email you@example.com". # When unset it expands to an empty string and is silently ignored. {$CADDY_ACME_EMAIL:} + + # CrowdSec bouncer — streams decisions from the CrowdSec LAPI every 15s. + # CROWDSEC_API_KEY is injected at runtime via crowdsec/.crowdsec.env. + # The default "disabled" placeholder makes the bouncer fail-open (warn, + # pass traffic) when no key is configured — Caddy still starts cleanly. + crowdsec { + api_url http://crowdsec:8080 + api_key {$CROWDSEC_API_KEY:disabled} + ticker_interval 15s + } } (security_headers) { @@ -57,6 +79,13 @@ {$DOMAIN:localhost} { import security_headers + # ── CrowdSec bouncer ────────────────────────────────────────────────────── + # Checks every incoming request against CrowdSec decisions. + # Banned IPs receive a 403; all others pass through unchanged. + route { + crowdsec + } + # ── Rate limiting ───────────────────────────────────────────────────────── # Auth endpoints: strict — 10 req/min per IP rate_limit { @@ -104,15 +133,15 @@ # ── Backend-only API paths ──────────────────────────────────────────────── # These paths are served exclusively by the Go backend and have no # SvelteKit counterpart. Routing them here skips SK intentionally. - handle /api/browse { - reverse_proxy backend:8080 - } handle /api/book-preview/* { reverse_proxy backend:8080 } handle /api/chapter-text/* { reverse_proxy backend:8080 } + handle /api/chapter-markdown/* { + reverse_proxy backend:8080 + } handle /api/reindex/* { reverse_proxy backend:8080 } @@ -123,10 +152,20 @@ reverse_proxy backend:8080 } - # ── MinIO avatars bucket (presigned GET only) ───────────────────────────── + # ── MinIO bucket paths (presigned URLs) ────────────────────────────────── + # MinIO path-style presigned URLs include the bucket name as the first + # path segment. MINIO_PUBLIC_ENDPOINT points here, so Caddy must proxy + # these paths directly to MinIO — no auth layer needed (the presigned + # signature itself enforces access and expiry). handle /avatars/* { reverse_proxy minio:9000 } + handle /audio/* { + reverse_proxy minio:9000 + } + handle /chapters/* { + reverse_proxy minio:9000 + } # ── SvelteKit UI (catch-all — includes all remaining /api/* routes) ─────── handle { @@ -153,8 +192,49 @@ } # ── Logging ─────────────────────────────────────────────────────────────── + # JSON log file read by CrowdSec for threat detection. log { - output stdout + output file /var/log/caddy/access.log { + roll_size 100MiB + roll_keep 5 + roll_keep_for 720h + } format json } } + +# ── Fider: user feedback & feature requests ─────────────────────────────────── +feedback.libnovel.cc { + import security_headers + reverse_proxy fider:3000 +} + +# ── GlitchTip: error tracking ───────────────────────────────────────────────── +errors.libnovel.cc { + import security_headers + reverse_proxy glitchtip-web:8000 +} + +# ── Umami: page analytics ───────────────────────────────────────────────────── +analytics.libnovel.cc { + import security_headers + reverse_proxy umami:3000 +} + +# ── Dozzle: Docker log viewer ───────────────────────────────────────────────── +logs.libnovel.cc { + import security_headers + reverse_proxy dozzle:8080 +} + +# ── Uptime Kuma: uptime monitoring ──────────────────────────────────────────── +uptime.libnovel.cc { + import security_headers + reverse_proxy uptime-kuma:3001 +} + +# ── Gotify: push notifications ──────────────────────────────────────────────── +push.libnovel.cc { + import security_headers + reverse_proxy gotify:80 +} diff --git a/v3/backend/internal/domain/domain.go b/v3/backend/internal/domain/domain.go index 9597f34..a582e17 100644 --- a/v3/backend/internal/domain/domain.go +++ b/v3/backend/internal/domain/domain.go @@ -28,6 +28,7 @@ type BookMeta struct { // CatalogueEntry is a lightweight book reference returned by catalogue pages. type CatalogueEntry struct { + Slug string `json:"slug"` Title string `json:"title"` URL string `json:"url"` } diff --git a/v3/backend/internal/meili/client.go b/v3/backend/internal/meili/client.go index 6d9904e..8eb24bf 100644 --- a/v3/backend/internal/meili/client.go +++ b/v3/backend/internal/meili/client.go @@ -29,6 +29,9 @@ const indexName = "books" type Client interface { // UpsertBook adds or updates a book document in the search index. UpsertBook(ctx context.Context, book domain.BookMeta) error + // BookExists reports whether a book with the given slug is already in the + // index. Used by the catalogue refresh to skip re-indexing known books. + BookExists(ctx context.Context, slug string) bool // Search returns up to limit books matching query. Search(ctx context.Context, query string, limit int) ([]domain.BookMeta, error) // Catalogue queries books with optional filters, sort, and pagination. @@ -172,6 +175,15 @@ func (c *MeiliClient) UpsertBook(_ context.Context, book domain.BookMeta) error return nil } +// BookExists reports whether the slug is already present in the index. +// It fetches the document by primary key; a 404 or any error is treated as +// "not present" (safe default: re-index rather than silently skip). +func (c *MeiliClient) BookExists(_ context.Context, slug string) bool { + var doc bookDoc + err := c.idx.GetDocument(slug, nil, &doc) + return err == nil && doc.Slug != "" +} + // Search returns books matching query, up to limit results. func (c *MeiliClient) Search(_ context.Context, query string, limit int) ([]domain.BookMeta, error) { if limit <= 0 { @@ -306,6 +318,7 @@ func sortStrings(s []string) { type NoopClient struct{} func (NoopClient) UpsertBook(_ context.Context, _ domain.BookMeta) error { return nil } +func (NoopClient) BookExists(_ context.Context, _ string) bool { return false } func (NoopClient) Search(_ context.Context, _ string, _ int) ([]domain.BookMeta, error) { return nil, nil } diff --git a/v3/backend/internal/novelfire/scraper.go b/v3/backend/internal/novelfire/scraper.go index dad5b1b..b089677 100644 --- a/v3/backend/internal/novelfire/scraper.go +++ b/v3/backend/internal/novelfire/scraper.go @@ -111,7 +111,7 @@ func (s *Scraper) ScrapeCatalogue(ctx context.Context) (<-chan domain.CatalogueE select { case <-ctx.Done(): return - case entries <- domain.CatalogueEntry{Title: title, URL: bookURL}: + case entries <- domain.CatalogueEntry{Slug: slugFromURL(bookURL), Title: title, URL: bookURL}: } } diff --git a/v3/backend/internal/runner/catalogue_refresh.go b/v3/backend/internal/runner/catalogue_refresh.go index 652260a..df388e1 100644 --- a/v3/backend/internal/runner/catalogue_refresh.go +++ b/v3/backend/internal/runner/catalogue_refresh.go @@ -51,6 +51,14 @@ func (r *Runner) runCatalogueRefresh(ctx context.Context) { break } + // Skip books already present in Meilisearch — they were indexed on a + // previous run. Re-indexing only happens when a scrape task is + // explicitly enqueued (e.g. via the admin UI or API). + if r.deps.SearchIndex.BookExists(ctx, entry.Slug) { + skipped++ + continue + } + // Random jitter between books to avoid rate-limiting. jitter := time.Duration(1000+rand.Intn(2000)) * time.Millisecond select { diff --git a/v3/docker-compose.yml b/v3/docker-compose.yml index c08f14d..4a582cb 100644 --- a/v3/docker-compose.yml +++ b/v3/docker-compose.yml @@ -51,10 +51,10 @@ services: entrypoint: > /bin/sh -c " mc alias set local http://minio:9000 $${MINIO_ROOT_USER:-admin} $${MINIO_ROOT_PASSWORD:-changeme123}; - mc mb --ignore-existing local/libnovel-chapters; - mc mb --ignore-existing local/libnovel-audio; + mc mb --ignore-existing local/chapters; + mc mb --ignore-existing local/audio; mc mb --ignore-existing local/avatars; - mc mb --ignore-existing local/libnovel-browse; + mc mb --ignore-existing local/catalogue; echo 'buckets ready'; " environment: @@ -157,6 +157,8 @@ services: <<: *infra-env BACKEND_HTTP_ADDR: ":8080" LOG_LEVEL: "${LOG_LEVEL:-info}" + KOKORO_URL: "${KOKORO_URL:-https://kokoro.kalekber.cc/}" + KOKORO_VOICE: "${KOKORO_VOICE:-af_bella}" healthcheck: test: ["CMD", "/healthcheck", "http://localhost:8080/health"] interval: 15s @@ -200,8 +202,12 @@ services: RUNNER_WORKER_ID: "${RUNNER_WORKER_ID:-runner-1}" RUNNER_TIMEOUT: "${RUNNER_TIMEOUT:-90s}" RUNNER_METRICS_ADDR: "${RUNNER_METRICS_ADDR:-:9091}" + # Suppress the on-startup catalogue walk — catalogue_refresh now skips + # books already in Meilisearch, so a full walk on every restart is wasteful. + # The 24h periodic ticker (CatalogueRefreshInterval) still fires normally. + RUNNER_SKIP_INITIAL_CATALOGUE_REFRESH: "true" # Kokoro-FastAPI TTS endpoint - KOKORO_URL: "${KOKORO_URL:-}" + KOKORO_URL: "${KOKORO_URL:-https://kokoro.kalekber.cc/}" KOKORO_VOICE: "${KOKORO_VOICE:-af_bella}" healthcheck: # The runner writes /tmp/runner.alive on every poll. @@ -253,8 +259,64 @@ services: timeout: 5s retries: 3 + # ─── CrowdSec (threat detection + IP blocking) ─────────────────────────────── + # Reads Caddy JSON access logs from the shared caddy_logs volume and enforces + # decisions via the Caddy bouncer plugin. + crowdsec: + image: crowdsecurity/crowdsec:latest + restart: unless-stopped + environment: + GID: "1000" + COLLECTIONS: "crowdsecurity/caddy crowdsecurity/http-dos crowdsecurity/base-http-scenarios" + volumes: + - crowdsec_data:/var/lib/crowdsec/data + - ./crowdsec/acquis.yaml:/etc/crowdsec/acquis.yaml:ro + - caddy_logs:/var/log/caddy:ro + expose: + - "8080" + healthcheck: + test: ["CMD", "cscli", "version"] + interval: 20s + timeout: 10s + retries: 5 + + # ─── CrowdSec bouncer registration ─────────────────────────────────────────── + # One-shot: registers the Caddy bouncer with the CrowdSec LAPI and writes the + # generated API key to crowdsec/.crowdsec.env, which Caddy reads via env_file. + # Uses the Docker socket to exec cscli inside the running crowdsec container. + crowdsec-init: + image: docker:cli + depends_on: + crowdsec: + condition: service_healthy + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./crowdsec:/crowdsec-out + entrypoint: > + /bin/sh -c " + out=/crowdsec-out/.crowdsec.env; + existing=$$(grep -s '^CROWDSEC_API_KEY=.' \"$$out\" | cut -d= -f2-); + if [ -n \"$$existing\" ]; then + echo 'crowdsec-init: key already present, skipping registration'; + exit 0; + fi; + container=$$(docker ps --filter name=crowdsec --filter status=running --format '{{.Names}}' | grep -v init | head -1); + echo \"crowdsec-init: using container $$container\"; + docker exec $$container cscli bouncers delete caddy-bouncer 2>/dev/null || true; + key=$$(docker exec $$container cscli bouncers add caddy-bouncer -o raw 2>&1); + if [ -z \"$$key\" ]; then + echo 'crowdsec-init: ERROR — failed to obtain bouncer key' >&2; + exit 1; + fi; + printf 'CROWDSEC_API_KEY=%s\n' \"$$key\" > \"$$out\"; + echo \"crowdsec-init: bouncer key written (key length: $${#key})\"; + " + restart: "no" + + # ─── Caddy (reverse proxy + automatic HTTPS) ────────────────────────────────── - # Custom build includes github.com/mholt/caddy-ratelimit. + # Custom build includes github.com/mholt/caddy-ratelimit and + # github.com/hslatman/caddy-crowdsec-bouncer/http. caddy: build: context: ./caddy @@ -265,6 +327,8 @@ services: condition: service_healthy ui: condition: service_healthy + crowdsec-init: + condition: service_completed_successfully ports: - "80:80" - "443:443" @@ -272,11 +336,15 @@ services: environment: DOMAIN: "${DOMAIN:-localhost}" CADDY_ACME_EMAIL: "${CADDY_ACME_EMAIL:-}" + env_file: + - path: ./crowdsec/.crowdsec.env + required: false volumes: - ./Caddyfile:/etc/caddy/Caddyfile:ro - ./caddy/errors:/srv/errors:ro - caddy_data:/data - caddy_config:/config + - caddy_logs:/var/log/caddy # ─── Watchtower (auto-redeploy custom services on new images) ──────────────── # Only watches services labelled com.centurylinklabs.watchtower.enable=true. @@ -292,6 +360,203 @@ services: WATCHTOWER_NOTIFICATION_URL: "${WATCHTOWER_NOTIFICATION_URL:-}" DOCKER_API_VERSION: "1.44" + # ─── Shared PostgreSQL (Fider + GlitchTip + Umami) ─────────────────────────── + # A single Postgres instance hosting three separate databases. + # PocketBase uses its own embedded SQLite; this postgres is only for the + # three new services below. + postgres: + image: postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_USER: "${POSTGRES_USER:-pgadmin}" + POSTGRES_PASSWORD: "${POSTGRES_PASSWORD:-changeme_postgres_123}" + POSTGRES_DB: postgres + expose: + - "5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER:-pgadmin}"] + interval: 10s + timeout: 5s + retries: 5 + + # ─── Postgres database initialisation ──────────────────────────────────────── + # One-shot: creates the fider, glitchtip, and umami databases if missing. + postgres-init: + image: postgres:16-alpine + depends_on: + postgres: + condition: service_healthy + environment: + PGPASSWORD: "${POSTGRES_PASSWORD:-changeme_postgres_123}" + entrypoint: > + /bin/sh -c " + psql -h postgres -U ${POSTGRES_USER:-pgadmin} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='fider'\" | grep -q 1 || + psql -h postgres -U ${POSTGRES_USER:-pgadmin} -d postgres -c \"CREATE DATABASE fider\"; + psql -h postgres -U ${POSTGRES_USER:-pgadmin} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='glitchtip'\" | grep -q 1 || + psql -h postgres -U ${POSTGRES_USER:-pgadmin} -d postgres -c \"CREATE DATABASE glitchtip\"; + psql -h postgres -U ${POSTGRES_USER:-pgadmin} -d postgres -tc \"SELECT 1 FROM pg_database WHERE datname='umami'\" | grep -q 1 || + psql -h postgres -U ${POSTGRES_USER:-pgadmin} -d postgres -c \"CREATE DATABASE umami\"; + echo 'postgres-init: databases ready'; + " + restart: "no" + + # ─── Fider (user feedback & feature requests) ───────────────────────────────── + fider: + image: getfider/fider:stable + restart: unless-stopped + depends_on: + postgres-init: + condition: service_completed_successfully + postgres: + condition: service_healthy + expose: + - "3000" + environment: + BASE_URL: "https://feedback.libnovel.cc" + DATABASE_URL: "postgres://${POSTGRES_USER:-pgadmin}:${POSTGRES_PASSWORD:-changeme_postgres_123}@postgres:5432/fider?sslmode=disable" + JWT_SECRET: "0cd21a84d03b36e730aa6ef5a24c1dd6cb01546187e6ca1161c90e3a167df52d" + # Email: noreply mode — emails are suppressed (logged to stdout). + # Fider still requires SMTP vars to be non-empty even in noreply mode. + EMAIL_NOREPLY: "noreply@libnovel.cc" + EMAIL_SMTP_HOST: "localhost" + EMAIL_SMTP_PORT: "25" + # Disable outbound email — set real SMTP values to enable. + EMAIL_NOREPLY_MODE: "true" + + # ─── GlitchTip DB migration (one-shot) ─────────────────────────────────────── + glitchtip-migrate: + image: glitchtip/glitchtip:latest + depends_on: + postgres-init: + condition: service_completed_successfully + postgres: + condition: service_healthy + environment: + DATABASE_URL: "postgres://${POSTGRES_USER:-pgadmin}:${POSTGRES_PASSWORD:-changeme_postgres_123}@postgres:5432/glitchtip" + SECRET_KEY: "d4936488eb40bafe5cd79f137133b3f642db5537dd3f80ab4c858984443163be" + GLITCHTIP_DOMAIN: "https://errors.libnovel.cc" + EMAIL_URL: "consolemail://" + DEFAULT_FROM_EMAIL: "errors@libnovel.cc" + VALKEY_URL: "redis://valkey:6379/1" + command: "./manage.py migrate" + restart: "no" + + # ─── GlitchTip web (error tracking UI + API) ───────────────────────────────── + glitchtip-web: + image: glitchtip/glitchtip:latest + restart: unless-stopped + depends_on: + glitchtip-migrate: + condition: service_completed_successfully + valkey: + condition: service_healthy + expose: + - "8000" + environment: + DATABASE_URL: "postgres://${POSTGRES_USER:-pgadmin}:${POSTGRES_PASSWORD:-changeme_postgres_123}@postgres:5432/glitchtip" + SECRET_KEY: "d4936488eb40bafe5cd79f137133b3f642db5537dd3f80ab4c858984443163be" + GLITCHTIP_DOMAIN: "https://errors.libnovel.cc" + EMAIL_URL: "consolemail://" + DEFAULT_FROM_EMAIL: "errors@libnovel.cc" + VALKEY_URL: "redis://valkey:6379/1" + PORT: "8000" + ENABLE_USER_REGISTRATION: "false" + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/0/')"] + interval: 15s + timeout: 5s + retries: 5 + + # ─── GlitchTip worker (background task processor) ───────────────────────────── + glitchtip-worker: + image: glitchtip/glitchtip:latest + restart: unless-stopped + depends_on: + glitchtip-migrate: + condition: service_completed_successfully + valkey: + condition: service_healthy + environment: + DATABASE_URL: "postgres://${POSTGRES_USER:-pgadmin}:${POSTGRES_PASSWORD:-changeme_postgres_123}@postgres:5432/glitchtip" + SECRET_KEY: "d4936488eb40bafe5cd79f137133b3f642db5537dd3f80ab4c858984443163be" + GLITCHTIP_DOMAIN: "https://errors.libnovel.cc" + EMAIL_URL: "consolemail://" + DEFAULT_FROM_EMAIL: "errors@libnovel.cc" + VALKEY_URL: "redis://valkey:6379/1" + SERVER_ROLE: "worker" + + # ─── Umami (page analytics) ─────────────────────────────────────────────────── + umami: + image: ghcr.io/umami-software/umami:postgresql-latest + restart: unless-stopped + depends_on: + postgres-init: + condition: service_completed_successfully + postgres: + condition: service_healthy + expose: + - "3000" + environment: + DATABASE_URL: "postgresql://${POSTGRES_USER:-pgadmin}:${POSTGRES_PASSWORD:-changeme_postgres_123}@postgres:5432/umami" + APP_SECRET: "e4a91a11027acfd0270539a67a405115eda3d0450827415900d775c183b83821" + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:3000/api/heartbeat"] + interval: 15s + timeout: 5s + retries: 5 + + # ─── Dozzle (Docker log viewer) ─────────────────────────────────────────────── + dozzle: + image: amir20/dozzle:latest + restart: unless-stopped + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./dozzle/users.yml:/data/users.yml:ro + expose: + - "8080" + environment: + DOZZLE_AUTH_PROVIDER: simple + DOZZLE_HOSTNAME: "logs.libnovel.cc" + healthcheck: + test: ["CMD", "/dozzle", "healthcheck"] + interval: 15s + timeout: 5s + retries: 5 + + # ─── Uptime Kuma (uptime monitoring) ────────────────────────────────────────── + uptime-kuma: + image: louislam/uptime-kuma:1 + restart: unless-stopped + volumes: + - uptime_kuma_data:/app/data + expose: + - "3001" + healthcheck: + test: ["CMD", "extra/healthcheck"] + interval: 15s + timeout: 5s + retries: 5 + + # ─── Gotify (push notifications) ────────────────────────────────────────────── + gotify: + image: gotify/server:latest + restart: unless-stopped + volumes: + - gotify_data:/app/data + expose: + - "80" + environment: + GOTIFY_DEFAULTUSER_NAME: "${GOTIFY_ADMIN_USER:-admin}" + GOTIFY_DEFAULTUSER_PASS: "${GOTIFY_ADMIN_PASS:-changeme_gotify_123}" + GOTIFY_SERVER_PORT: "80" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:80/health"] + interval: 15s + timeout: 5s + retries: 5 + volumes: minio_data: pb_data: @@ -299,3 +564,8 @@ volumes: valkey_data: caddy_data: caddy_config: + caddy_logs: + crowdsec_data: + postgres_data: + uptime_kuma_data: + gotify_data: diff --git a/v3/dozzle/users.yml b/v3/dozzle/users.yml new file mode 100644 index 0000000..c53b951 --- /dev/null +++ b/v3/dozzle/users.yml @@ -0,0 +1,5 @@ +users: + admin: + name: admin + email: admin@libnovel.cc + password: "$2y$10$4jqLza2grpxnQn0EGux2C.UmlSxRmOvH/J1ySzOBxMZgW6cA2TnmK" diff --git a/v3/ui/src/routes/admin/+layout.svelte b/v3/ui/src/routes/admin/+layout.svelte new file mode 100644 index 0000000..74b6f40 --- /dev/null +++ b/v3/ui/src/routes/admin/+layout.svelte @@ -0,0 +1,56 @@ + + + +
+ +
+ {#each adminTabs as tab} + + {tab.label} + + {/each} +
+ + +
+ {#each toolTabs as tool} + + {tool.label} ↗ + + {/each} +
+
+ +{@render children?.()}