From 040072c3f524a0f2acfbea31b2d77965e4944b11 Mon Sep 17 00:00:00 2001 From: Admin Date: Sat, 28 Mar 2026 22:44:31 +0500 Subject: [PATCH] docs(d2): update architecture and api-routing diagrams to current state architecture.d2: - Split app into prod VPS (165.22.70.138) and homelab runner (192.168.0.109) - Add CrowdSec, Dozzle agent, pocket-tts (voice samples) - Valkey now shown as Asynq job queue in addition to presign cache - Add caddy-l4 Redis TCP proxy (:6380) to Caddy label - Add CI/CD node (Gitea Actions) with full job list incl. releases.json bake - Remove runner from prod app group (it runs on homelab only) - Watchtower: note runner is label-disabled on prod api-routing.d2: - Add /api/presign/* routes to backend (presign_be group) - Add /api/audio POST + status GET to both sk and be - Add /api/scrape/book and /api/scrape/book/range to scrape_sk - Catalogue: annotate Meilisearch vs legacy browse - Add Meilisearch filter/sort fields to storage node - Add Asynq queue note to Valkey storage node - Fix presign proxy: sk routes through be.presign_be, not directly to storage --- docs/d2/api-routing.d2 | 47 +++++---- docs/d2/api-routing.svg | 211 ++++++++++++++++++------------------- docs/d2/architecture.d2 | 136 ++++++++++++++---------- docs/d2/architecture.svg | 217 ++++++++++++++++++++------------------- 4 files changed, 324 insertions(+), 287 deletions(-) diff --git a/docs/d2/api-routing.d2 b/docs/d2/api-routing.d2 index db6e9b8..ec45746 100644 --- a/docs/d2/api-routing.d2 +++ b/docs/d2/api-routing.d2 @@ -35,11 +35,11 @@ client: Browser / iOS App { caddy: Caddy :443 { shape: rectangle style.fill: "#f1f5f9" - label: "Caddy :443\ncustom build · caddy-ratelimit\nsecurity headers · rate limiting\nstatic error pages" + label: "Caddy :443\ncustom build · caddy-l4 · caddy-ratelimit\nCrowdSec bouncer · security headers\nrate limiting · static error pages\nRedis TCP proxy :6380" } # ─── SvelteKit UI ───────────────────────────────────────────────────────────── -# Handles: auth enforcement, session, all /api/* routes that have SK counterparts +# All routes here pass through SvelteKit — auth is enforced server-side. sk: SvelteKit UI :3000 { style.fill: "#fef3c7" @@ -53,7 +53,7 @@ sk: SvelteKit UI :3000 { catalogue_sk: Catalogue { style.fill: "#f0fdf4" style.stroke: "#86efac" - label: "GET /api/catalogue-page\nGET /api/search" + label: "GET /api/catalogue-page (infinite scroll)\nGET /api/search" } book_sk: Book { @@ -65,7 +65,7 @@ sk: SvelteKit UI :3000 { scrape_sk: Scrape (admin) { style.fill: "#fff7ed" style.stroke: "#fdba74" - label: "GET /api/scrape/status\nGET /api/scrape/tasks\nPOST /api/scrape\nPOST /api/scrape/range\nPOST /api/scrape/cancel/{id}" + label: "GET /api/scrape/status\nGET /api/scrape/tasks\nPOST /api/scrape\nPOST /api/scrape/book\nPOST /api/scrape/book/range\nPOST /api/scrape/cancel/{id}" } audio_sk: Audio { @@ -74,7 +74,7 @@ sk: SvelteKit UI :3000 { label: "POST /api/audio/{slug}/{n}\nGET /api/audio/status/{slug}/{n}\nGET /api/voices" } - presign_sk: Presigned URLs { + presign_sk: Presigned URLs (public) { style.fill: "#f0fdf4" style.stroke: "#86efac" label: "GET /api/presign/chapter/{slug}/{n}\nGET /api/presign/audio/{slug}/{n}\nGET /api/presign/voice-sample/{voice}" @@ -106,12 +106,12 @@ sk: SvelteKit UI :3000 { } # ─── Go Backend ─────────────────────────────────────────────────────────────── -# Caddy proxies these paths directly — no SvelteKit auth layer +# Caddy proxies these paths directly — bypasses SvelteKit entirely. be: Backend API :8080 { style.fill: "#eef3ff" - health_be: Health { + health_be: Health / Version { style.fill: "#f0fdf4" style.stroke: "#86efac" label: "GET /health\nGET /api/version" @@ -126,7 +126,7 @@ be: Backend API :8080 { catalogue_be: Catalogue { style.fill: "#f0fdf4" style.stroke: "#86efac" - label: "GET /api/browse\nGET /api/catalogue\nGET /api/ranking\nGET /api/cover/{domain}/{slug}" + label: "GET /api/catalogue (Meilisearch)\nGET /api/browse (legacy MinIO cache)\nGET /api/ranking\nGET /api/cover/{domain}/{slug}" } book_be: Book / Chapter { @@ -138,7 +138,13 @@ be: Backend API :8080 { audio_be: Audio { style.fill: "#f0fdf4" style.stroke: "#86efac" - label: "GET /api/audio-proxy/{slug}/{n}\nGET /api/voices" + label: "POST /api/audio/{slug}/{n}\nGET /api/audio/status/{slug}/{n}\nGET /api/audio-proxy/{slug}/{n}\nGET /api/voices" + } + + presign_be: Presigned URLs { + style.fill: "#f0fdf4" + style.stroke: "#86efac" + label: "GET /api/presign/chapter/{slug}/{n}\nGET /api/presign/audio/{slug}/{n}\nGET /api/presign/voice-sample/{voice}\nGET /api/presign/avatar-upload/{userId}\nGET /api/presign/avatar/{userId}" } } @@ -149,19 +155,19 @@ storage: Storage { pb: PocketBase :8090 { shape: cylinder - label: "auth · books · progress\ncomments · library\nscrape_jobs · audio_cache" + label: "auth · books · progress\ncomments · library\nscrape_jobs · audio_cache\nranking" } mn: MinIO :9000 { shape: cylinder - label: "chapters · audio\navatars · browse" + label: "chapters · audio\navatars · catalogue (browse)" } ms: Meilisearch :7700 { shape: cylinder - label: "index: books" + label: "index: books\nfilterable: status · genres\nsortable: rank · rating\n total_chapters · meta_updated" } vk: Valkey :6379 { shape: cylinder - label: "presign URL cache" + label: "presign URL cache (TTL ~55 min)\nAsynq job queue (runner)" } } @@ -169,18 +175,17 @@ storage: Storage { client -> caddy: HTTPS :443 -caddy -> sk: "/* (catch-all)\n→ SvelteKit handles auth" -caddy -> be: "/health /scrape*\n/api/browse /api/book-preview/*\n/api/chapter-text/* /api/chapter-markdown/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/* /api/catalogue /api/ranking" -caddy -> storage.mn: "/avatars/*\n/audio/*\n/chapters/*\n(presigned MinIO GETs)" +caddy -> sk: "/* (catch-all)\n→ SvelteKit enforces auth" +caddy -> be: "/health /scrape*\n/api/browse /api/catalogue /api/ranking\n/api/version /api/book-preview/*\n/api/chapter-text/* /api/chapter-markdown/*\n/api/reindex/* /api/cover/*\n/api/audio* /api/voices /api/presign/*" +caddy -> storage.mn: "/avatars/* /audio/* /chapters/*\n(presigned MinIO GETs)" # ─── SvelteKit → Backend (server-side proxy) ────────────────────────────────── sk.catalogue_sk -> be.catalogue_be: internal proxy sk.book_sk -> be.book_be: internal proxy sk.audio_sk -> be.audio_be: internal proxy -sk.presign_sk -> storage.vk: check cache -sk.presign_sk -> storage.mn: generate presign -sk.presign_user -> storage.mn: generate presign +sk.presign_sk -> be.presign_be: internal proxy +sk.presign_user -> be.presign_be: internal proxy # ─── SvelteKit → Storage (direct) ──────────────────────────────────────────── @@ -192,10 +197,12 @@ sk.comments_sk -> storage.pb # ─── Backend → Storage ──────────────────────────────────────────────────────── -be.catalogue_be -> storage.ms: full-text search +be.catalogue_be -> storage.ms: full-text search + facets be.catalogue_be -> storage.pb: ranking records be.catalogue_be -> storage.mn: cover presign be.book_be -> storage.mn: chapter objects be.book_be -> storage.pb: book metadata be.audio_be -> storage.mn: audio presign be.audio_be -> storage.vk: presign cache +be.presign_be -> storage.vk: check / set presign cache +be.presign_be -> storage.mn: generate presigned URL diff --git a/docs/d2/api-routing.svg b/docs/d2/api-routing.svg index 3f19df2..633b329 100644 --- a/docs/d2/api-routing.svg +++ b/docs/d2/api-routing.svg @@ -1,24 +1,24 @@ -LegendBrowser / iOS AppCaddy :443custom build · caddy-ratelimitsecurity headers · rate limitingstatic error pagesSvelteKit UI :3000Backend API :8080Storagepublicuser authadmin onlyPOST /api/auth/loginPOST /api/auth/registerPOST /api/auth/change-passwordGET /api/auth/sessionGET /api/catalogue-pageGET /api/searchGET /api/book/{slug}GET /api/chapter/{slug}/{n}GET /api/chapter-text-preview/{slug}/{n}GET /api/scrape/statusGET /api/scrape/tasksPOST /api/scrapePOST /api/scrape/rangePOST /api/scrape/cancel/{id}POST /api/audio/{slug}/{n}GET /api/audio/status/{slug}/{n}GET /api/voicesGET /api/presign/chapter/{slug}/{n}GET /api/presign/audio/{slug}/{n}GET /api/presign/voice-sample/{voice}GET /api/presign/avatar-upload/{userId}GET /api/presign/avatar/{userId}GET /api/progressPOST /api/progress/{slug}DELETE /api/progress/{slug}GET /api/libraryPOST /api/library/{slug}DELETE /api/library/{slug}GET /api/comments/{slug}POST /api/comments/{slug}GET /healthGET /api/versionPOST /scrapePOST /scrape/bookPOST /scrape/book/rangeGET /api/browseGET /api/catalogueGET /api/rankingGET /api/cover/{domain}/{slug}GET /api/book-preview/{slug}GET /api/chapter-text/{slug}/{n}GET /api/chapter-markdown/{slug}/{n}POST /api/reindex/{slug} ⚠ adminGET /api/audio-proxy/{slug}/{n}GET /api/voicesauth · books · progresscomments · libraryscrape_jobs · audio_cachechapters · audioavatars · browseindex: bookspresign URL cache HTTPS :443/* (catch-all)→ SvelteKit handles auth/health /scrape*/api/browse /api/book-preview/*/api/chapter-text/* /api/chapter-markdown/*/api/reindex/* /api/cover/*/api/audio-proxy/* /api/catalogue /api/ranking/avatars/*/libnovel-audio/*/libnovel-chapters/*(presigned MinIO GETs)internal proxyinternal proxyinternal proxycheck cachegenerate presigngenerate presignsessions / usersscrape job recordsfull-text searchranking recordscover presignchapter objectsbook metadataaudio presignpresign cache - - - - - - - - - - - - - - - - - - - - + .d2-4043408742 .fill-N1{fill:#0A0F25;} + .d2-4043408742 .fill-N2{fill:#676C7E;} + .d2-4043408742 .fill-N3{fill:#9499AB;} + .d2-4043408742 .fill-N4{fill:#CFD2DD;} + .d2-4043408742 .fill-N5{fill:#DEE1EB;} + .d2-4043408742 .fill-N6{fill:#EEF1F8;} + .d2-4043408742 .fill-N7{fill:#FFFFFF;} + .d2-4043408742 .fill-B1{fill:#0D32B2;} + .d2-4043408742 .fill-B2{fill:#0D32B2;} + .d2-4043408742 .fill-B3{fill:#E3E9FD;} + .d2-4043408742 .fill-B4{fill:#E3E9FD;} + .d2-4043408742 .fill-B5{fill:#EDF0FD;} + .d2-4043408742 .fill-B6{fill:#F7F8FE;} + .d2-4043408742 .fill-AA2{fill:#4A6FF3;} + .d2-4043408742 .fill-AA4{fill:#EDF0FD;} + .d2-4043408742 .fill-AA5{fill:#F7F8FE;} + .d2-4043408742 .fill-AB4{fill:#EDF0FD;} + .d2-4043408742 .fill-AB5{fill:#F7F8FE;} + .d2-4043408742 .stroke-N1{stroke:#0A0F25;} + .d2-4043408742 .stroke-N2{stroke:#676C7E;} + .d2-4043408742 .stroke-N3{stroke:#9499AB;} + .d2-4043408742 .stroke-N4{stroke:#CFD2DD;} + .d2-4043408742 .stroke-N5{stroke:#DEE1EB;} + .d2-4043408742 .stroke-N6{stroke:#EEF1F8;} + .d2-4043408742 .stroke-N7{stroke:#FFFFFF;} + .d2-4043408742 .stroke-B1{stroke:#0D32B2;} + .d2-4043408742 .stroke-B2{stroke:#0D32B2;} + .d2-4043408742 .stroke-B3{stroke:#E3E9FD;} + .d2-4043408742 .stroke-B4{stroke:#E3E9FD;} + .d2-4043408742 .stroke-B5{stroke:#EDF0FD;} + .d2-4043408742 .stroke-B6{stroke:#F7F8FE;} + .d2-4043408742 .stroke-AA2{stroke:#4A6FF3;} + .d2-4043408742 .stroke-AA4{stroke:#EDF0FD;} + .d2-4043408742 .stroke-AA5{stroke:#F7F8FE;} + .d2-4043408742 .stroke-AB4{stroke:#EDF0FD;} + .d2-4043408742 .stroke-AB5{stroke:#F7F8FE;} + .d2-4043408742 .background-color-N1{background-color:#0A0F25;} + .d2-4043408742 .background-color-N2{background-color:#676C7E;} + .d2-4043408742 .background-color-N3{background-color:#9499AB;} + .d2-4043408742 .background-color-N4{background-color:#CFD2DD;} + .d2-4043408742 .background-color-N5{background-color:#DEE1EB;} + .d2-4043408742 .background-color-N6{background-color:#EEF1F8;} + .d2-4043408742 .background-color-N7{background-color:#FFFFFF;} + .d2-4043408742 .background-color-B1{background-color:#0D32B2;} + .d2-4043408742 .background-color-B2{background-color:#0D32B2;} + .d2-4043408742 .background-color-B3{background-color:#E3E9FD;} + .d2-4043408742 .background-color-B4{background-color:#E3E9FD;} + .d2-4043408742 .background-color-B5{background-color:#EDF0FD;} + .d2-4043408742 .background-color-B6{background-color:#F7F8FE;} + .d2-4043408742 .background-color-AA2{background-color:#4A6FF3;} + .d2-4043408742 .background-color-AA4{background-color:#EDF0FD;} + .d2-4043408742 .background-color-AA5{background-color:#F7F8FE;} + .d2-4043408742 .background-color-AB4{background-color:#EDF0FD;} + .d2-4043408742 .background-color-AB5{background-color:#F7F8FE;} + .d2-4043408742 .color-N1{color:#0A0F25;} + .d2-4043408742 .color-N2{color:#676C7E;} + .d2-4043408742 .color-N3{color:#9499AB;} + .d2-4043408742 .color-N4{color:#CFD2DD;} + .d2-4043408742 .color-N5{color:#DEE1EB;} + .d2-4043408742 .color-N6{color:#EEF1F8;} + .d2-4043408742 .color-N7{color:#FFFFFF;} + .d2-4043408742 .color-B1{color:#0D32B2;} + .d2-4043408742 .color-B2{color:#0D32B2;} + .d2-4043408742 .color-B3{color:#E3E9FD;} + .d2-4043408742 .color-B4{color:#E3E9FD;} + .d2-4043408742 .color-B5{color:#EDF0FD;} + .d2-4043408742 .color-B6{color:#F7F8FE;} + .d2-4043408742 .color-AA2{color:#4A6FF3;} + .d2-4043408742 .color-AA4{color:#EDF0FD;} + .d2-4043408742 .color-AA5{color:#F7F8FE;} + .d2-4043408742 .color-AB4{color:#EDF0FD;} + .d2-4043408742 .color-AB5{color:#F7F8FE;}.appendix text.text{fill:#0A0F25}.md{--color-fg-default:#0A0F25;--color-fg-muted:#676C7E;--color-fg-subtle:#9499AB;--color-canvas-default:#FFFFFF;--color-canvas-subtle:#EEF1F8;--color-border-default:#0D32B2;--color-border-muted:#0D32B2;--color-neutral-muted:#EEF1F8;--color-accent-fg:#0D32B2;--color-accent-emphasis:#0D32B2;--color-attention-subtle:#676C7E;--color-danger-fg:red;}.sketch-overlay-B1{fill:url(#streaks-darker-d2-4043408742);mix-blend-mode:lighten}.sketch-overlay-B2{fill:url(#streaks-darker-d2-4043408742);mix-blend-mode:lighten}.sketch-overlay-B3{fill:url(#streaks-bright-d2-4043408742);mix-blend-mode:darken}.sketch-overlay-B4{fill:url(#streaks-bright-d2-4043408742);mix-blend-mode:darken}.sketch-overlay-B5{fill:url(#streaks-bright-d2-4043408742);mix-blend-mode:darken}.sketch-overlay-B6{fill:url(#streaks-bright-d2-4043408742);mix-blend-mode:darken}.sketch-overlay-AA2{fill:url(#streaks-dark-d2-4043408742);mix-blend-mode:overlay}.sketch-overlay-AA4{fill:url(#streaks-bright-d2-4043408742);mix-blend-mode:darken}.sketch-overlay-AA5{fill:url(#streaks-bright-d2-4043408742);mix-blend-mode:darken}.sketch-overlay-AB4{fill:url(#streaks-bright-d2-4043408742);mix-blend-mode:darken}.sketch-overlay-AB5{fill:url(#streaks-bright-d2-4043408742);mix-blend-mode:darken}.sketch-overlay-N1{fill:url(#streaks-darker-d2-4043408742);mix-blend-mode:lighten}.sketch-overlay-N2{fill:url(#streaks-dark-d2-4043408742);mix-blend-mode:overlay}.sketch-overlay-N3{fill:url(#streaks-normal-d2-4043408742);mix-blend-mode:color-burn}.sketch-overlay-N4{fill:url(#streaks-normal-d2-4043408742);mix-blend-mode:color-burn}.sketch-overlay-N5{fill:url(#streaks-bright-d2-4043408742);mix-blend-mode:darken}.sketch-overlay-N6{fill:url(#streaks-bright-d2-4043408742);mix-blend-mode:darken}.sketch-overlay-N7{fill:url(#streaks-bright-d2-4043408742);mix-blend-mode:darken}.light-code{display: block}.dark-code{display: none}]]>LegendBrowser / iOS AppCaddy :443custom build · caddy-l4 · caddy-ratelimitCrowdSec bouncer · security headersrate limiting · static error pagesRedis TCP proxy :6380SvelteKit UI :3000Backend API :8080Storagepublicuser authadmin onlyPOST /api/auth/loginPOST /api/auth/registerPOST /api/auth/change-passwordGET /api/auth/sessionGET /api/catalogue-page (infinite scroll)GET /api/searchGET /api/book/{slug}GET /api/chapter/{slug}/{n}GET /api/chapter-text-preview/{slug}/{n}GET /api/scrape/statusGET /api/scrape/tasksPOST /api/scrapePOST /api/scrape/bookPOST /api/scrape/book/rangePOST /api/scrape/cancel/{id}POST /api/audio/{slug}/{n}GET /api/audio/status/{slug}/{n}GET /api/voicesGET /api/presign/chapter/{slug}/{n}GET /api/presign/audio/{slug}/{n}GET /api/presign/voice-sample/{voice}GET /api/presign/avatar-upload/{userId}GET /api/presign/avatar/{userId}GET /api/progressPOST /api/progress/{slug}DELETE /api/progress/{slug}GET /api/libraryPOST /api/library/{slug}DELETE /api/library/{slug}GET /api/comments/{slug}POST /api/comments/{slug}GET /healthGET /api/versionPOST /scrapePOST /scrape/bookPOST /scrape/book/rangeGET /api/catalogue (Meilisearch)GET /api/browse (legacy MinIO cache)GET /api/rankingGET /api/cover/{domain}/{slug}GET /api/book-preview/{slug}GET /api/chapter-text/{slug}/{n}GET /api/chapter-markdown/{slug}/{n}POST /api/reindex/{slug} ⚠ adminPOST /api/audio/{slug}/{n}GET /api/audio/status/{slug}/{n}GET /api/audio-proxy/{slug}/{n}GET /api/voicesGET /api/presign/chapter/{slug}/{n}GET /api/presign/audio/{slug}/{n}GET /api/presign/voice-sample/{voice}GET /api/presign/avatar-upload/{userId}GET /api/presign/avatar/{userId}auth · books · progresscomments · libraryscrape_jobs · audio_cacherankingchapters · audioavatars · catalogue (browse)index: booksfilterable: status · genressortable: rank · rating total_chapters · meta_updatedpresign URL cache (TTL ~55 min)Asynq job queue (runner) HTTPS :443/* (catch-all)→ SvelteKit enforces auth/health /scrape*/api/browse /api/catalogue /api/ranking/api/version /api/book-preview/*/api/chapter-text/* /api/chapter-markdown/*/api/reindex/* /api/cover/*/api/audio* /api/voices /api/presign/*/avatars/* /audio/* /chapters/*(presigned MinIO GETs)internal proxyinternal proxyinternal proxyinternal proxyinternal proxysessions / usersscrape job recordsfull-text search + facetsranking recordscover presignchapter objectsbook metadataaudio presignpresign cachecheck / set presign cachegenerate presigned URL + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/d2/architecture.d2 b/docs/d2/architecture.d2 index 470145b..cb726d0 100644 --- a/docs/d2/architecture.d2 +++ b/docs/d2/architecture.d2 @@ -5,16 +5,25 @@ direction: right novelfire: novelfire.net { shape: cloud style.fill: "#f0f4ff" + label: "novelfire.net\n(scrape source)" } kokoro: Kokoro-FastAPI TTS { shape: cloud style.fill: "#f0f4ff" + label: "Kokoro-FastAPI TTS\n(self-hosted · homelab)\nchapter audio" +} + +pockettts: pocket-tts { + shape: cloud + style.fill: "#f0f4ff" + label: "pocket-tts\n(self-hosted · homelab)\nvoice sample MP3s" } letsencrypt: Let's Encrypt { shape: cloud style.fill: "#f0f4ff" + label: "Let's Encrypt\n(ACME TLS-ALPN-01)" } browser: Browser / iOS App { @@ -30,12 +39,12 @@ init: Init containers { minio-init: minio-init { shape: rectangle - label: "minio-init\n(mc: create buckets)" + label: "minio-init\n(mc: create buckets\n chapters · audio\n avatars · catalogue)" } pb-init: pb-init { shape: rectangle - label: "pb-init\n(bootstrap collections)" + label: "pb-init\n(bootstrap PocketBase\n collections + schema)" } } @@ -46,109 +55,126 @@ storage: Storage { minio: MinIO { shape: cylinder - label: "MinIO :9000\n\nbuckets:\n chapters\n audio\n avatars\n catalogue" + label: "MinIO :9000\nbuckets:\n chapters · audio\n avatars · catalogue" } pocketbase: PocketBase { shape: cylinder - label: "PocketBase :8090\n\ncollections:\n books chapters_idx\n audio_cache progress\n scrape_jobs app_users\n ranking" + label: "PocketBase :8090\ncollections:\n books · chapters_idx\n audio_cache · progress\n scrape_jobs · app_users\n ranking · library\n comments" } valkey: Valkey { shape: cylinder - label: "Valkey :6379\n\n(presign URL cache\nTTL-based, shared)" + label: "Valkey :6379\npresign URL cache (TTL ~55 min)\nAsynq job queue (runner tasks)" } meilisearch: Meilisearch { shape: cylinder - label: "Meilisearch :7700\n\nindices:\n books" + label: "Meilisearch :7700\nindex: books\n(filterable: status · genres\n sortable: rank · rating\n total_chapters · meta_updated)" } } -# ─── Application ────────────────────────────────────────────────────────────── +# ─── Application — prod VPS (165.22.70.138) ─────────────────────────────────── -app: Application { +app: Application — prod (165.22.70.138) { style.fill: "#eef3ff" caddy: caddy { shape: rectangle - label: "Caddy :443 / :80\ncustom build + caddy-ratelimit\n\nfeatures:\n auto-HTTPS (Let's Encrypt)\n security headers\n rate limiting (per-IP)\n static error pages (502/503/504)" + label: "Caddy :443 / :80 / :6380\ncustom build\n+ caddy-l4 (Redis TCP proxy)\n+ caddy-ratelimit\nauto-HTTPS · security headers\nrate limiting (per-IP)\nstatic error pages (404/502/503/504)\nCrowdSec bouncer" } backend: backend { shape: rectangle - label: "Backend API :8080\n(Go — HTTP API server)" - } - - runner: runner { - shape: rectangle - label: "Runner :9091\n(Go — background worker\nscraping + TTS jobs\n/metrics endpoint)" + label: "Backend API :8080\n(Go)\nHTTP API server\nffmpeg (audio sample conv.)\nOpenTelemetry tracing\nSentry / GlitchTip errors" } ui: ui { shape: rectangle - label: "SvelteKit UI :3000\n(adapter-node)" + label: "SvelteKit UI :3000\n(adapter-node)\nSSR · session auth\nserver-side API proxy" + } + + crowdsec: CrowdSec { + shape: rectangle + label: "CrowdSec :8080\nsecurity engine\nreads Caddy JSON logs\nbouncer integrated in Caddy" + } + + dozzle: Dozzle agent { + shape: rectangle + label: "Dozzle agent\n127.0.0.1:7007\nlog relay → homelab dashboard" + } +} + +# ─── Runner — homelab (192.168.0.109) ──────────────────────────────────────── + +homelab: Runner — homelab (192.168.0.109) { + style.fill: "#fef9ec" + + runner: runner { + shape: rectangle + label: "Runner :9091\n(Go background worker)\nscrape pipeline\nTTS audio job queue\nPrometheus /metrics\ncron: catalogue refresh\nAsynq worker → Valkey" } } # ─── Ops ────────────────────────────────────────────────────────────────────── ops: Ops { - style.fill: "#fef9ec" + style.fill: "#f5f5f5" watchtower: Watchtower { shape: rectangle - label: "Watchtower\n(containrrr/watchtower)\n\npolls every 5 min\nautopulls + redeploys:\n backend · runner · ui" + label: "Watchtower\n(containrrr/watchtower)\npolls Docker Hub every 5 min\nautopulls + redeploys:\n backend · ui\n(runner: label-disabled on prod)" } } -# ─── Init → Storage deps ────────────────────────────────────────────────────── +# ─── CI / CD ────────────────────────────────────────────────────────────────── + +cicd: CI / CD { + style.fill: "#f0f9ff" + + gitea: Gitea Actions { + shape: rectangle + label: "Gitea Actions\n(homelab runner)\ntag v* trigger:\n test-backend\n check-ui (type-check + build)\n docker-backend\n docker-runner\n docker-ui (bakes releases.json)\n docker-caddy\n → push Docker Hub\n → Gitea Release" + } +} + +# ─── Init → Storage ─────────────────────────────────────────────────────────── init.minio-init -> storage.minio: create buckets {style.stroke-dash: 4} init.pb-init -> storage.pocketbase: bootstrap schema {style.stroke-dash: 4} -# ─── App → Storage ──────────────────────────────────────────────────────────── - -app.backend -> storage.minio: blobs (chapters, audio,\navatars, browse) -app.backend -> storage.pocketbase: structured records\n(books, progress, jobs…) -app.backend -> storage.valkey: cache presigned URLs\n(SET/GET with TTL) - -app.runner -> storage.minio: write chapter markdown\n& audio MP3s -app.runner -> storage.pocketbase: read/update scrape jobs\nwrite book records -app.runner -> storage.meilisearch: index books on\nscrape completion - -app.ui -> storage.valkey: read presigned URL cache -app.ui -> storage.pocketbase: auth, progress,\ncomments, settings - # ─── App internal ───────────────────────────────────────────────────────────── -app.ui -> app.backend: REST API calls (server-side)\n/api/catalogue /api/book-preview\n/api/chapter-text /api/audio etc. +app.caddy -> app.ui: "/* (catch-all)\nSvelteKit — auth enforced" +app.caddy -> app.backend: "/health /scrape*\n/api/browse /api/catalogue\n/api/ranking /api/version\n/api/book-preview/*\n/api/chapter-text/*\n/api/chapter-markdown/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/* /api/voices\n/api/audio* /api/presign/*" +app.caddy -> storage.minio: "/avatars/* /audio/*\n/chapters/*\n(presigned GETs)" +app.caddy -> app.crowdsec: bouncer check (15 s poll) +app.caddy -> letsencrypt: ACME cert (TLS-ALPN-01) -# ─── Caddy routing ──────────────────────────────────────────────────────────── -# Routes sent directly to backend (no SvelteKit counterpart): -# /health /scrape* -# /api/browse /api/book-preview/* /api/chapter-text/* -# /api/reindex/* /api/cover/* /api/audio-proxy/* -# Routes sent to MinIO: -# /avatars/* -# Everything else → SvelteKit UI (including /api/scrape/*, /api/chapter-text-preview/*) +app.ui -> app.backend: "internal REST proxy\n(server-side only)" +app.ui -> storage.pocketbase: "auth · sessions\nprogress · library\ncomments" -app.caddy -> app.ui: "/* (catch-all)\n/api/scrape/*\n/api/chapter-text-preview/*\n→ SvelteKit (auth enforced)" -app.caddy -> app.backend: "/health /scrape*\n/api/browse /api/book-preview/*\n/api/chapter-text/*\n/api/reindex/* /api/cover/*\n/api/audio-proxy/*" -app.caddy -> storage.minio: "/avatars/*\n/audio/*\n/chapters/*\n(presigned MinIO GETs)" +app.backend -> storage.minio: "chapter objs · audio MP3s\navatars · browse cache" +app.backend -> storage.pocketbase: "books · scrape_jobs\naudio_cache · ranking" +app.backend -> storage.valkey: "presign URL cache\n(SET/GET TTL ~55 min)" +app.backend -> storage.meilisearch: "catalogue search\nfacets: genres · status" +app.backend -> pockettts: "voice sample gen.\n(on-demand · ffmpeg conv.)" -# ─── External → App ─────────────────────────────────────────────────────────── +# ─── Runner → deps ──────────────────────────────────────────────────────────── -app.runner -> novelfire: scrape\n(HTTP GET) -app.runner -> kokoro: TTS generation\n(HTTP POST) -app.caddy -> letsencrypt: ACME certificate\n(TLS-ALPN-01) +homelab.runner -> novelfire: "HTTP scrape\nHTML → Markdown" +homelab.runner -> kokoro: "TTS generation\ntext → MP3" +homelab.runner -> storage.minio: "write chapters\n& audio MP3s" +homelab.runner -> storage.pocketbase: "read/update scrape_jobs\nwrite book records" +homelab.runner -> storage.meilisearch: "index books\n(on scrape completion)" +homelab.runner -> storage.valkey: "Asynq job queue\n(task consume)" -# ─── Ops → Docker socket ────────────────────────────────────────────────────── - -ops.watchtower -> app.backend: watch (label-enabled) -ops.watchtower -> app.runner: watch (label-enabled) -ops.watchtower -> app.ui: watch (label-enabled) - -# ─── Browser ────────────────────────────────────────────────────────────────── +# ─── Client ─────────────────────────────────────────────────────────────────── browser -> app.caddy: HTTPS :443\n(single entry point) + +# ─── Ops / CI ───────────────────────────────────────────────────────────────── + +ops.watchtower -> app.backend: watch (label-enabled) +ops.watchtower -> app.ui: watch (label-enabled) +cicd.gitea -> ops.watchtower: push to Docker Hub\n→ Watchtower detects new tag diff --git a/docs/d2/architecture.svg b/docs/d2/architecture.svg index c1d9fcb..6a7941b 100644 --- a/docs/d2/architecture.svg +++ b/docs/d2/architecture.svg @@ -1,24 +1,24 @@ -novelfire.netKokoro-FastAPI TTSLet's EncryptBrowser / iOS AppInit containersStorageApplicationOpsminio-init(mc: create buckets)pb-init(bootstrap collections)MinIO :9000 buckets: libnovel-chapters libnovel-audio avatars libnovel-browsePocketBase :8090 collections: books chapters_idx audio_cache progress scrape_jobs app_users rankingValkey :6379 (presign URL cacheTTL-based, shared)Meilisearch :7700 indices: booksCaddy :443 / :80custom build + caddy-ratelimit features: auto-HTTPS (Let's Encrypt) security headers rate limiting (per-IP) static error pages (502/503/504)Backend API :8080(Go — HTTP API server)Runner :9091(Go — background workerscraping + TTS jobs/metrics endpoint)SvelteKit UI :3000(adapter-node)Watchtower(containrrr/watchtower) polls every 5 minautopulls + redeploys: backend · runner · ui create bucketsbootstrap schema blobs (chapters, audio,avatars, browse)structured records(books, progress, jobs…)cache presigned URLs(SET/GET with TTL)write chapter markdown& audio MP3sread/update scrape jobswrite book recordsindex books onscrape completionread presigned URL cacheauth, progress,comments, settingsREST API calls (server-side)/api/catalogue /api/book-preview/api/chapter-text /api/audio etc./* (catch-all)/api/scrape/*/api/chapter-text-preview/*→ SvelteKit (auth enforced)/health /scrape*/api/browse /api/book-preview/*/api/chapter-text/*/api/reindex/* /api/cover/*/api/audio-proxy/*/avatars/*/libnovel-audio/*/libnovel-chapters/*(presigned MinIO GETs)scrape(HTTP GET)TTS generation(HTTP POST)ACME certificate(TLS-ALPN-01)watch (label-enabled)watch (label-enabled)watch (label-enabled)HTTPS :443(single entry point) - - - - - - - - - - - - - - - - - - - - - - + .d2-286243964 .fill-N1{fill:#0A0F25;} + .d2-286243964 .fill-N2{fill:#676C7E;} + .d2-286243964 .fill-N3{fill:#9499AB;} + .d2-286243964 .fill-N4{fill:#CFD2DD;} + .d2-286243964 .fill-N5{fill:#DEE1EB;} + .d2-286243964 .fill-N6{fill:#EEF1F8;} + .d2-286243964 .fill-N7{fill:#FFFFFF;} + .d2-286243964 .fill-B1{fill:#0D32B2;} + .d2-286243964 .fill-B2{fill:#0D32B2;} + .d2-286243964 .fill-B3{fill:#E3E9FD;} + .d2-286243964 .fill-B4{fill:#E3E9FD;} + .d2-286243964 .fill-B5{fill:#EDF0FD;} + .d2-286243964 .fill-B6{fill:#F7F8FE;} + .d2-286243964 .fill-AA2{fill:#4A6FF3;} + .d2-286243964 .fill-AA4{fill:#EDF0FD;} + .d2-286243964 .fill-AA5{fill:#F7F8FE;} + .d2-286243964 .fill-AB4{fill:#EDF0FD;} + .d2-286243964 .fill-AB5{fill:#F7F8FE;} + .d2-286243964 .stroke-N1{stroke:#0A0F25;} + .d2-286243964 .stroke-N2{stroke:#676C7E;} + .d2-286243964 .stroke-N3{stroke:#9499AB;} + .d2-286243964 .stroke-N4{stroke:#CFD2DD;} + .d2-286243964 .stroke-N5{stroke:#DEE1EB;} + .d2-286243964 .stroke-N6{stroke:#EEF1F8;} + .d2-286243964 .stroke-N7{stroke:#FFFFFF;} + .d2-286243964 .stroke-B1{stroke:#0D32B2;} + .d2-286243964 .stroke-B2{stroke:#0D32B2;} + .d2-286243964 .stroke-B3{stroke:#E3E9FD;} + .d2-286243964 .stroke-B4{stroke:#E3E9FD;} + .d2-286243964 .stroke-B5{stroke:#EDF0FD;} + .d2-286243964 .stroke-B6{stroke:#F7F8FE;} + .d2-286243964 .stroke-AA2{stroke:#4A6FF3;} + .d2-286243964 .stroke-AA4{stroke:#EDF0FD;} + .d2-286243964 .stroke-AA5{stroke:#F7F8FE;} + .d2-286243964 .stroke-AB4{stroke:#EDF0FD;} + .d2-286243964 .stroke-AB5{stroke:#F7F8FE;} + .d2-286243964 .background-color-N1{background-color:#0A0F25;} + .d2-286243964 .background-color-N2{background-color:#676C7E;} + .d2-286243964 .background-color-N3{background-color:#9499AB;} + .d2-286243964 .background-color-N4{background-color:#CFD2DD;} + .d2-286243964 .background-color-N5{background-color:#DEE1EB;} + .d2-286243964 .background-color-N6{background-color:#EEF1F8;} + .d2-286243964 .background-color-N7{background-color:#FFFFFF;} + .d2-286243964 .background-color-B1{background-color:#0D32B2;} + .d2-286243964 .background-color-B2{background-color:#0D32B2;} + .d2-286243964 .background-color-B3{background-color:#E3E9FD;} + .d2-286243964 .background-color-B4{background-color:#E3E9FD;} + .d2-286243964 .background-color-B5{background-color:#EDF0FD;} + .d2-286243964 .background-color-B6{background-color:#F7F8FE;} + .d2-286243964 .background-color-AA2{background-color:#4A6FF3;} + .d2-286243964 .background-color-AA4{background-color:#EDF0FD;} + .d2-286243964 .background-color-AA5{background-color:#F7F8FE;} + .d2-286243964 .background-color-AB4{background-color:#EDF0FD;} + .d2-286243964 .background-color-AB5{background-color:#F7F8FE;} + .d2-286243964 .color-N1{color:#0A0F25;} + .d2-286243964 .color-N2{color:#676C7E;} + .d2-286243964 .color-N3{color:#9499AB;} + .d2-286243964 .color-N4{color:#CFD2DD;} + .d2-286243964 .color-N5{color:#DEE1EB;} + .d2-286243964 .color-N6{color:#EEF1F8;} + .d2-286243964 .color-N7{color:#FFFFFF;} + .d2-286243964 .color-B1{color:#0D32B2;} + .d2-286243964 .color-B2{color:#0D32B2;} + .d2-286243964 .color-B3{color:#E3E9FD;} + .d2-286243964 .color-B4{color:#E3E9FD;} + .d2-286243964 .color-B5{color:#EDF0FD;} + .d2-286243964 .color-B6{color:#F7F8FE;} + .d2-286243964 .color-AA2{color:#4A6FF3;} + .d2-286243964 .color-AA4{color:#EDF0FD;} + .d2-286243964 .color-AA5{color:#F7F8FE;} + .d2-286243964 .color-AB4{color:#EDF0FD;} + .d2-286243964 .color-AB5{color:#F7F8FE;}.appendix text.text{fill:#0A0F25}.md{--color-fg-default:#0A0F25;--color-fg-muted:#676C7E;--color-fg-subtle:#9499AB;--color-canvas-default:#FFFFFF;--color-canvas-subtle:#EEF1F8;--color-border-default:#0D32B2;--color-border-muted:#0D32B2;--color-neutral-muted:#EEF1F8;--color-accent-fg:#0D32B2;--color-accent-emphasis:#0D32B2;--color-attention-subtle:#676C7E;--color-danger-fg:red;}.sketch-overlay-B1{fill:url(#streaks-darker-d2-286243964);mix-blend-mode:lighten}.sketch-overlay-B2{fill:url(#streaks-darker-d2-286243964);mix-blend-mode:lighten}.sketch-overlay-B3{fill:url(#streaks-bright-d2-286243964);mix-blend-mode:darken}.sketch-overlay-B4{fill:url(#streaks-bright-d2-286243964);mix-blend-mode:darken}.sketch-overlay-B5{fill:url(#streaks-bright-d2-286243964);mix-blend-mode:darken}.sketch-overlay-B6{fill:url(#streaks-bright-d2-286243964);mix-blend-mode:darken}.sketch-overlay-AA2{fill:url(#streaks-dark-d2-286243964);mix-blend-mode:overlay}.sketch-overlay-AA4{fill:url(#streaks-bright-d2-286243964);mix-blend-mode:darken}.sketch-overlay-AA5{fill:url(#streaks-bright-d2-286243964);mix-blend-mode:darken}.sketch-overlay-AB4{fill:url(#streaks-bright-d2-286243964);mix-blend-mode:darken}.sketch-overlay-AB5{fill:url(#streaks-bright-d2-286243964);mix-blend-mode:darken}.sketch-overlay-N1{fill:url(#streaks-darker-d2-286243964);mix-blend-mode:lighten}.sketch-overlay-N2{fill:url(#streaks-dark-d2-286243964);mix-blend-mode:overlay}.sketch-overlay-N3{fill:url(#streaks-normal-d2-286243964);mix-blend-mode:color-burn}.sketch-overlay-N4{fill:url(#streaks-normal-d2-286243964);mix-blend-mode:color-burn}.sketch-overlay-N5{fill:url(#streaks-bright-d2-286243964);mix-blend-mode:darken}.sketch-overlay-N6{fill:url(#streaks-bright-d2-286243964);mix-blend-mode:darken}.sketch-overlay-N7{fill:url(#streaks-bright-d2-286243964);mix-blend-mode:darken}.light-code{display: block}.dark-code{display: none}]]>novelfire.net(scrape source)Kokoro-FastAPI TTS(self-hosted · homelab)chapter audiopocket-tts(self-hosted · homelab)voice sample MP3sLet's Encrypt(ACME TLS-ALPN-01)Browser / iOS AppInit containersStorageApplication — prod (165.22.70.138)Runner — homelab (192.168.0.109)OpsCI / CDminio-init(mc: create buckets chapters · audio avatars · catalogue)pb-init(bootstrap PocketBase collections + schema)MinIO :9000buckets: chapters · audio avatars · cataloguePocketBase :8090collections: books · chapters_idx audio_cache · progress scrape_jobs · app_users ranking · library commentsValkey :6379presign URL cache (TTL ~55 min)Asynq job queue (runner tasks)Meilisearch :7700index: books(filterable: status · genres sortable: rank · rating total_chapters · meta_updated)Caddy :443 / :80 / :6380custom build+ caddy-l4 (Redis TCP proxy)+ caddy-ratelimitauto-HTTPS · security headersrate limiting (per-IP)static error pages (404/502/503/504)CrowdSec bouncerBackend API :8080(Go)HTTP API serverffmpeg (audio sample conv.)OpenTelemetry tracingSentry / GlitchTip errorsSvelteKit UI :3000(adapter-node)SSR · session authserver-side API proxyCrowdSec :8080security enginereads Caddy JSON logsbouncer integrated in CaddyDozzle agent127.0.0.1:7007log relay → homelab dashboardRunner :9091(Go background worker)scrape pipelineTTS audio job queuePrometheus /metricscron: catalogue refreshAsynq worker → ValkeyWatchtower(containrrr/watchtower)polls Docker Hub every 5 minautopulls + redeploys: backend · ui(runner: label-disabled on prod)Gitea Actions(homelab runner)tag v* trigger: test-backend check-ui (type-check + build) docker-backend docker-runner docker-ui (bakes releases.json) docker-caddy → push Docker Hub → Gitea Release create bucketsbootstrap schema /* (catch-all)SvelteKit — auth enforced/health /scrape*/api/browse /api/catalogue/api/ranking /api/version/api/book-preview/*/api/chapter-text/*/api/chapter-markdown/*/api/reindex/* /api/cover/*/api/audio-proxy/* /api/voices/api/audio* /api/presign/*/avatars/* /audio/*/chapters/*(presigned GETs)bouncer check (15 s poll)ACME cert (TLS-ALPN-01)internal REST proxy(server-side only)auth · sessionsprogress · librarycommentschapter objs · audio MP3savatars · browse cachebooks · scrape_jobsaudio_cache · rankingpresign URL cache(SET/GET TTL ~55 min)catalogue searchfacets: genres · statusvoice sample gen.(on-demand · ffmpeg conv.)HTTP scrapeHTML → MarkdownTTS generationtext → MP3write chapters& audio MP3sread/update scrape_jobswrite book recordsindex books(on scrape completion)Asynq job queue(task consume)HTTPS :443(single entry point)watch (label-enabled)watch (label-enabled)push to Docker Hub→ Watchtower detects new tag + + + + + + + + + + + + + + + + + + + + + + + + +