diff --git a/scraper/internal/orchestrator/orchestrator.go b/scraper/internal/orchestrator/orchestrator.go index 3440e3f..e9fba33 100644 --- a/scraper/internal/orchestrator/orchestrator.go +++ b/scraper/internal/orchestrator/orchestrator.go @@ -45,6 +45,14 @@ type Config struct { // that one book instead of walking the full catalogue. SingleBookURL string + // FromChapter, when > 0, skips chapters with number < FromChapter. + // Only effective in single-book mode. + FromChapter int + + // ToChapter, when > 0, skips chapters with number > ToChapter. + // Only effective in single-book mode. 0 means "no upper limit". + ToChapter int + // OnProgress is called periodically with the current progress counters. // It is always called on completion (success or failure). May be nil. OnProgress func(p Progress) @@ -204,6 +212,15 @@ func (o *Orchestrator) Run(ctx context.Context) error { // Enqueue chapter jobs. for _, ref := range refs { + // Apply chapter range filter (only in single-book mode when set). + if o.cfg.FromChapter > 0 && ref.Number < o.cfg.FromChapter { + chaptersSkipped.Add(1) + continue + } + if o.cfg.ToChapter > 0 && ref.Number > o.cfg.ToChapter { + chaptersSkipped.Add(1) + continue + } select { case <-ctx.Done(): return diff --git a/scraper/internal/server/handlers_scrape.go b/scraper/internal/server/handlers_scrape.go index d0544bc..d99dcb2 100644 --- a/scraper/internal/server/handlers_scrape.go +++ b/scraper/internal/server/handlers_scrape.go @@ -33,6 +33,29 @@ func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) { s.runAsync(w, cfg) } +// handleScrapeBookRange handles POST /api/scrape/book/range. +// Body: {"url": "...", "from": N, "to": M} +// Scrapes only chapters in the range [from, to] (inclusive). +// from=0 means "start from chapter 1"; to=0 means "no upper limit". +func (s *Server) handleScrapeBookRange(w http.ResponseWriter, r *http.Request) { + var body struct { + URL string `json:"url"` + From int `json:"from"` + To int `json:"to"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" { + http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest) + return + } + + cfg := s.oCfg + cfg.SingleBookURL = body.URL + cfg.FromChapter = body.From + cfg.ToChapter = body.To + + s.runAsync(w, cfg) +} + // runAsync launches an orchestrator in the background and returns 202 Accepted. // Only one scrape job runs at a time; concurrent requests receive 409 Conflict. func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) { diff --git a/ui/src/routes/api/scrape/range/+server.ts b/ui/src/routes/api/scrape/range/+server.ts new file mode 100644 index 0000000..c827dfb --- /dev/null +++ b/ui/src/routes/api/scrape/range/+server.ts @@ -0,0 +1,62 @@ +/** + * POST /api/scrape/range + * + * Proxies range-scrape requests to the Go scraper backend at POST /scrape/book/range. + * Admin-only. + * + * Request body (JSON): + * { "url": "https://novelfire.net/book/...", "from": 50, "to": 100 } + * "to" is optional — omit to scrape from "from" to the end. + * + * Responses mirror the Go scraper: + * 202 Accepted — job enqueued + * 409 Conflict — a scrape job is already running + * 400 Bad Request + * 403 Forbidden — not an admin + */ + +import { json, error } from '@sveltejs/kit'; +import type { RequestHandler } from './$types'; +import { env } from '$env/dynamic/private'; +import { log } from '$lib/server/logger'; + +const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080'; + +export const POST: RequestHandler = async ({ request, locals }) => { + // Admin guard + if (!locals.user || locals.user.role !== 'admin') { + throw error(403, 'Forbidden'); + } + + let body: { url?: string; from?: number; to?: number } = {}; + try { + body = await request.json(); + } catch { + throw error(400, 'Invalid JSON body'); + } + + if (!body.url || typeof body.from !== 'number') { + throw error(400, 'url and from are required'); + } + + const upstream = `${SCRAPER_URL}/scrape/book/range`; + let res: Response; + try { + res = await fetch(upstream, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ url: body.url, from: body.from, to: body.to }) + }); + } catch (e) { + log.error('scrape/range', 'scraper proxy network error', { err: String(e) }); + throw error(502, 'Could not reach scraper'); + } + + if (!res.ok && res.status >= 500) { + const text = await res.text().catch(() => ''); + log.error('scrape/range', 'scraper returned error', { status: res.status, body: text }); + } + + const data = await res.json().catch(() => ({})); + return json(data, { status: res.status }); +}; diff --git a/ui/src/routes/books/[slug]/+page.svelte b/ui/src/routes/books/[slug]/+page.svelte index 0c73655..752f059 100644 --- a/ui/src/routes/books/[slug]/+page.svelte +++ b/ui/src/routes/books/[slug]/+page.svelte @@ -17,9 +17,16 @@ // Paginate chapter list — show 100 at a time let page = $state(0); const PAGE_SIZE = 100; - const totalPages = $derived(Math.ceil(data.chapters.length / PAGE_SIZE)); + + // Use preview chapters if the book is not in the library + const chapterList = $derived( + data.inLib + ? data.chapters + : (data.previewChapters ?? []) + ); + const totalPages = $derived(Math.ceil(chapterList.length / PAGE_SIZE)); const visibleChapters = $derived( - data.chapters.slice(page * PAGE_SIZE, (page + 1) * PAGE_SIZE) + chapterList.slice(page * PAGE_SIZE, (page + 1) * PAGE_SIZE) ); // ── Admin: rescrape ─────────────────────────────────────────────────────── @@ -45,6 +52,55 @@ scraping = false; } } + + // ── Admin: scrape range ─────────────────────────────────────────────────── + let rangeFrom = $state(''); + let rangeTo = $state(''); + let rangeScraping = $state(false); + let rangeResult = $state<'queued' | 'busy' | 'error' | ''>(''); + + async function scrapeRange() { + if (rangeScraping || !data.book.source_url) return; + const from = parseInt(rangeFrom, 10); + const to = parseInt(rangeTo, 10); + if (!from || from < 1) return; + rangeScraping = true; + rangeResult = ''; + try { + const res = await fetch('/api/scrape/range', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ url: data.book.source_url, from, to: to || undefined }) + }); + if (res.ok) rangeResult = 'queued'; + else if (res.status === 409) rangeResult = 'busy'; + else rangeResult = 'error'; + } catch { + rangeResult = 'error'; + } finally { + rangeScraping = false; + } + } + + async function scrapeFromChapter(n: number) { + if (rangeScraping || !data.book.source_url) return; + rangeScraping = true; + rangeResult = ''; + try { + const res = await fetch('/api/scrape/range', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ url: data.book.source_url, from: n }) + }); + if (res.ok) rangeResult = 'queued'; + else if (res.status === 409) rangeResult = 'busy'; + else rangeResult = 'error'; + } catch { + rangeResult = 'error'; + } finally { + rangeScraping = false; + } + } @@ -62,7 +118,14 @@ {/if}
-

{data.book.title}

+
+

{data.book.title}

+ {#if !data.inLib} + + not in library + + {/if} +
{#if data.book.author}

{data.book.author}

@@ -90,12 +153,12 @@ Continue ch.{data.lastChapter} {/if} - {#if data.chapters.length > 0} + {#if chapterList.length > 0} - Start from ch.1 + {data.inLib ? 'Start from ch.1' : 'Preview ch.1'} {/if}
@@ -104,13 +167,13 @@
-
+

Chapters - ({data.chapters.length}) + ({chapterList.length})

-
+
{#if data.isAdmin && data.book.source_url} + + {#if rangeResult} + + {rangeResult === 'queued' ? 'Range scrape queued.' : rangeResult === 'busy' ? 'Scraper busy.' : 'Error queuing.'} + + {/if} +
+ {/if} + + {#if chapterList.length === 0}

No chapters available yet.

{:else}
{#each visibleChapters as chapter} {@const isCurrent = data.lastChapter === chapter.number} - - + - {chapter.number} - - - {chapter.title || `Chapter ${chapter.number}`} - - {#if isCurrent} - reading + + {chapter.number} + + + {chapter.title || `Chapter ${chapter.number}`} + + {#if isCurrent} + reading + {/if} + {#if (chapter as { date_label?: string }).date_label} + + {/if} + + + {#if data.isAdmin && data.book.source_url && data.inLib} + {/if} - {#if chapter.date_label} - - {/if} - +
{/each}
{/if}