feat(scrape-range): add chapter range scraping for admin users
Adds FromChapter/ToChapter fields to orchestrator.Config and skips out-of-range chapters in processBook. Exposes POST /scrape/book/range Go endpoint and a matching UI proxy at /api/scrape/range. The book detail page now shows admins a range input (from/to chapter) and a per-chapter 'scrape from here up' button.
This commit is contained in:
Admin
@@ -45,6 +45,14 @@ type Config struct {
|
||||
// that one book instead of walking the full catalogue.
|
||||
SingleBookURL string
|
||||
|
||||
// FromChapter, when > 0, skips chapters with number < FromChapter.
|
||||
// Only effective in single-book mode.
|
||||
FromChapter int
|
||||
|
||||
// ToChapter, when > 0, skips chapters with number > ToChapter.
|
||||
// Only effective in single-book mode. 0 means "no upper limit".
|
||||
ToChapter int
|
||||
|
||||
// OnProgress is called periodically with the current progress counters.
|
||||
// It is always called on completion (success or failure). May be nil.
|
||||
OnProgress func(p Progress)
|
||||
@@ -204,6 +212,15 @@ func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
|
||||
// Enqueue chapter jobs.
|
||||
for _, ref := range refs {
|
||||
// Apply chapter range filter (only in single-book mode when set).
|
||||
if o.cfg.FromChapter > 0 && ref.Number < o.cfg.FromChapter {
|
||||
chaptersSkipped.Add(1)
|
||||
continue
|
||||
}
|
||||
if o.cfg.ToChapter > 0 && ref.Number > o.cfg.ToChapter {
|
||||
chaptersSkipped.Add(1)
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
|
||||
@@ -33,6 +33,29 @@ func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) {
|
||||
s.runAsync(w, cfg)
|
||||
}
|
||||
|
||||
// handleScrapeBookRange handles POST /api/scrape/book/range.
|
||||
// Body: {"url": "...", "from": N, "to": M}
|
||||
// Scrapes only chapters in the range [from, to] (inclusive).
|
||||
// from=0 means "start from chapter 1"; to=0 means "no upper limit".
|
||||
func (s *Server) handleScrapeBookRange(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
URL string `json:"url"`
|
||||
From int `json:"from"`
|
||||
To int `json:"to"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
|
||||
http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
cfg := s.oCfg
|
||||
cfg.SingleBookURL = body.URL
|
||||
cfg.FromChapter = body.From
|
||||
cfg.ToChapter = body.To
|
||||
|
||||
s.runAsync(w, cfg)
|
||||
}
|
||||
|
||||
// runAsync launches an orchestrator in the background and returns 202 Accepted.
|
||||
// Only one scrape job runs at a time; concurrent requests receive 409 Conflict.
|
||||
func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
|
||||
|
||||
62
ui/src/routes/api/scrape/range/+server.ts
Normal file
62
ui/src/routes/api/scrape/range/+server.ts
Normal file
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* POST /api/scrape/range
|
||||
*
|
||||
* Proxies range-scrape requests to the Go scraper backend at POST /scrape/book/range.
|
||||
* Admin-only.
|
||||
*
|
||||
* Request body (JSON):
|
||||
* { "url": "https://novelfire.net/book/...", "from": 50, "to": 100 }
|
||||
* "to" is optional — omit to scrape from "from" to the end.
|
||||
*
|
||||
* Responses mirror the Go scraper:
|
||||
* 202 Accepted — job enqueued
|
||||
* 409 Conflict — a scrape job is already running
|
||||
* 400 Bad Request
|
||||
* 403 Forbidden — not an admin
|
||||
*/
|
||||
|
||||
import { json, error } from '@sveltejs/kit';
|
||||
import type { RequestHandler } from './$types';
|
||||
import { env } from '$env/dynamic/private';
|
||||
import { log } from '$lib/server/logger';
|
||||
|
||||
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
|
||||
|
||||
export const POST: RequestHandler = async ({ request, locals }) => {
|
||||
// Admin guard
|
||||
if (!locals.user || locals.user.role !== 'admin') {
|
||||
throw error(403, 'Forbidden');
|
||||
}
|
||||
|
||||
let body: { url?: string; from?: number; to?: number } = {};
|
||||
try {
|
||||
body = await request.json();
|
||||
} catch {
|
||||
throw error(400, 'Invalid JSON body');
|
||||
}
|
||||
|
||||
if (!body.url || typeof body.from !== 'number') {
|
||||
throw error(400, 'url and from are required');
|
||||
}
|
||||
|
||||
const upstream = `${SCRAPER_URL}/scrape/book/range`;
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(upstream, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ url: body.url, from: body.from, to: body.to })
|
||||
});
|
||||
} catch (e) {
|
||||
log.error('scrape/range', 'scraper proxy network error', { err: String(e) });
|
||||
throw error(502, 'Could not reach scraper');
|
||||
}
|
||||
|
||||
if (!res.ok && res.status >= 500) {
|
||||
const text = await res.text().catch(() => '');
|
||||
log.error('scrape/range', 'scraper returned error', { status: res.status, body: text });
|
||||
}
|
||||
|
||||
const data = await res.json().catch(() => ({}));
|
||||
return json(data, { status: res.status });
|
||||
};
|
||||
@@ -17,9 +17,16 @@
|
||||
// Paginate chapter list — show 100 at a time
|
||||
let page = $state(0);
|
||||
const PAGE_SIZE = 100;
|
||||
const totalPages = $derived(Math.ceil(data.chapters.length / PAGE_SIZE));
|
||||
|
||||
// Use preview chapters if the book is not in the library
|
||||
const chapterList = $derived(
|
||||
data.inLib
|
||||
? data.chapters
|
||||
: (data.previewChapters ?? [])
|
||||
);
|
||||
const totalPages = $derived(Math.ceil(chapterList.length / PAGE_SIZE));
|
||||
const visibleChapters = $derived(
|
||||
data.chapters.slice(page * PAGE_SIZE, (page + 1) * PAGE_SIZE)
|
||||
chapterList.slice(page * PAGE_SIZE, (page + 1) * PAGE_SIZE)
|
||||
);
|
||||
|
||||
// ── Admin: rescrape ───────────────────────────────────────────────────────
|
||||
@@ -45,6 +52,55 @@
|
||||
scraping = false;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Admin: scrape range ───────────────────────────────────────────────────
|
||||
let rangeFrom = $state('');
|
||||
let rangeTo = $state('');
|
||||
let rangeScraping = $state(false);
|
||||
let rangeResult = $state<'queued' | 'busy' | 'error' | ''>('');
|
||||
|
||||
async function scrapeRange() {
|
||||
if (rangeScraping || !data.book.source_url) return;
|
||||
const from = parseInt(rangeFrom, 10);
|
||||
const to = parseInt(rangeTo, 10);
|
||||
if (!from || from < 1) return;
|
||||
rangeScraping = true;
|
||||
rangeResult = '';
|
||||
try {
|
||||
const res = await fetch('/api/scrape/range', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ url: data.book.source_url, from, to: to || undefined })
|
||||
});
|
||||
if (res.ok) rangeResult = 'queued';
|
||||
else if (res.status === 409) rangeResult = 'busy';
|
||||
else rangeResult = 'error';
|
||||
} catch {
|
||||
rangeResult = 'error';
|
||||
} finally {
|
||||
rangeScraping = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function scrapeFromChapter(n: number) {
|
||||
if (rangeScraping || !data.book.source_url) return;
|
||||
rangeScraping = true;
|
||||
rangeResult = '';
|
||||
try {
|
||||
const res = await fetch('/api/scrape/range', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ url: data.book.source_url, from: n })
|
||||
});
|
||||
if (res.ok) rangeResult = 'queued';
|
||||
else if (res.status === 409) rangeResult = 'busy';
|
||||
else rangeResult = 'error';
|
||||
} catch {
|
||||
rangeResult = 'error';
|
||||
} finally {
|
||||
rangeScraping = false;
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<svelte:head>
|
||||
@@ -62,7 +118,14 @@
|
||||
{/if}
|
||||
|
||||
<div class="flex flex-col gap-2 min-w-0">
|
||||
<div class="flex items-center gap-2 flex-wrap">
|
||||
<h1 class="text-2xl font-bold text-zinc-100 leading-tight">{data.book.title}</h1>
|
||||
{#if !data.inLib}
|
||||
<span class="text-xs px-2 py-0.5 rounded-full bg-zinc-700 text-zinc-400 border border-zinc-600 shrink-0">
|
||||
not in library
|
||||
</span>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
{#if data.book.author}
|
||||
<p class="text-zinc-400 text-sm">{data.book.author}</p>
|
||||
@@ -90,12 +153,12 @@
|
||||
Continue ch.{data.lastChapter}
|
||||
</a>
|
||||
{/if}
|
||||
{#if data.chapters.length > 0}
|
||||
{#if chapterList.length > 0}
|
||||
<a
|
||||
href="/books/{data.book.slug}/chapters/1"
|
||||
class="px-4 py-2 bg-zinc-700 text-zinc-100 font-semibold rounded text-sm hover:bg-zinc-600 transition-colors"
|
||||
>
|
||||
Start from ch.1
|
||||
{data.inLib ? 'Start from ch.1' : 'Preview ch.1'}
|
||||
</a>
|
||||
{/if}
|
||||
</div>
|
||||
@@ -104,13 +167,13 @@
|
||||
|
||||
<!-- Chapter list -->
|
||||
<div class="mt-4">
|
||||
<div class="flex items-center justify-between mb-3">
|
||||
<div class="flex items-center justify-between mb-3 flex-wrap gap-2">
|
||||
<h2 class="text-lg font-semibold text-zinc-100">
|
||||
Chapters
|
||||
<span class="text-zinc-500 font-normal text-sm ml-1">({data.chapters.length})</span>
|
||||
<span class="text-zinc-500 font-normal text-sm ml-1">({chapterList.length})</span>
|
||||
</h2>
|
||||
|
||||
<div class="flex items-center gap-3">
|
||||
<div class="flex items-center gap-3 flex-wrap">
|
||||
{#if data.isAdmin && data.book.source_url}
|
||||
<button
|
||||
onclick={rescrape}
|
||||
@@ -169,17 +232,61 @@
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
{#if data.chapters.length === 0}
|
||||
<!-- Admin: range scrape controls -->
|
||||
{#if data.isAdmin && data.book.source_url}
|
||||
<div class="mb-4 p-3 rounded bg-zinc-800/60 border border-zinc-700 flex flex-wrap items-end gap-3">
|
||||
<div class="flex flex-col gap-1">
|
||||
<label class="text-xs text-zinc-500">From chapter</label>
|
||||
<input
|
||||
type="number"
|
||||
min="1"
|
||||
bind:value={rangeFrom}
|
||||
placeholder="1"
|
||||
class="w-24 px-2 py-1 rounded bg-zinc-700 border border-zinc-600 text-zinc-200 text-xs focus:outline-none focus:border-amber-400"
|
||||
/>
|
||||
</div>
|
||||
<div class="flex flex-col gap-1">
|
||||
<label class="text-xs text-zinc-500">To chapter (optional)</label>
|
||||
<input
|
||||
type="number"
|
||||
min="1"
|
||||
bind:value={rangeTo}
|
||||
placeholder="end"
|
||||
class="w-24 px-2 py-1 rounded bg-zinc-700 border border-zinc-600 text-zinc-200 text-xs focus:outline-none focus:border-amber-400"
|
||||
/>
|
||||
</div>
|
||||
<button
|
||||
onclick={scrapeRange}
|
||||
disabled={rangeScraping || !rangeFrom}
|
||||
class="px-3 py-1.5 rounded text-xs font-medium transition-colors
|
||||
{rangeScraping || !rangeFrom
|
||||
? 'bg-zinc-700 text-zinc-500 cursor-not-allowed'
|
||||
: 'bg-amber-500/20 text-amber-300 hover:bg-amber-500/40 border border-amber-500/30'}"
|
||||
>
|
||||
{rangeScraping ? 'Queuing…' : 'Scrape range'}
|
||||
</button>
|
||||
|
||||
{#if rangeResult}
|
||||
<span class="text-xs {rangeResult === 'queued' ? 'text-green-400' : rangeResult === 'busy' ? 'text-amber-400' : 'text-red-400'}">
|
||||
{rangeResult === 'queued' ? 'Range scrape queued.' : rangeResult === 'busy' ? 'Scraper busy.' : 'Error queuing.'}
|
||||
</span>
|
||||
{/if}
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
{#if chapterList.length === 0}
|
||||
<p class="text-zinc-500 text-sm">No chapters available yet.</p>
|
||||
{:else}
|
||||
<div class="grid grid-cols-1 sm:grid-cols-2 gap-1">
|
||||
{#each visibleChapters as chapter}
|
||||
{@const isCurrent = data.lastChapter === chapter.number}
|
||||
{@const chapterUrl = data.inLib
|
||||
? `/books/${data.book.slug}/chapters/${chapter.number}`
|
||||
: `/books/${data.book.slug}/chapters/${chapter.number}?preview=1&chapter_url=${encodeURIComponent((chapter as { url?: string }).url ?? '')}&title=${encodeURIComponent(chapter.title ?? '')}`}
|
||||
<div class="flex items-center gap-3 px-3 py-2 rounded hover:bg-zinc-800 transition-colors group {isCurrent ? 'bg-zinc-800' : ''}">
|
||||
<a
|
||||
href="/books/{data.book.slug}/chapters/{chapter.number}"
|
||||
class="flex items-center gap-3 px-3 py-2 rounded hover:bg-zinc-800 transition-colors group {isCurrent
|
||||
? 'bg-zinc-800'
|
||||
: ''}"
|
||||
href={chapterUrl}
|
||||
class="flex items-center gap-3 flex-1 min-w-0"
|
||||
>
|
||||
<span
|
||||
class="text-xs font-mono w-10 text-right flex-shrink-0 {isCurrent
|
||||
@@ -194,10 +301,22 @@
|
||||
{#if isCurrent}
|
||||
<span class="text-xs text-amber-400 flex-shrink-0">reading</span>
|
||||
{/if}
|
||||
{#if chapter.date_label}
|
||||
<span class="text-xs text-zinc-600 flex-shrink-0 hidden sm:block">{chapter.date_label}</span>
|
||||
{#if (chapter as { date_label?: string }).date_label}
|
||||
<span class="text-xs text-zinc-600 flex-shrink-0 hidden sm:block">{(chapter as { date_label?: string }).date_label}</span>
|
||||
{/if}
|
||||
</a>
|
||||
<!-- Admin: scrape from this chapter up -->
|
||||
{#if data.isAdmin && data.book.source_url && data.inLib}
|
||||
<button
|
||||
onclick={() => scrapeFromChapter(chapter.number)}
|
||||
disabled={rangeScraping}
|
||||
class="opacity-0 group-hover:opacity-100 shrink-0 text-xs px-1.5 py-0.5 rounded bg-amber-500/10 text-amber-500 hover:bg-amber-500/30 transition-all border border-amber-500/20 disabled:opacity-30"
|
||||
title="Scrape from chapter {chapter.number} up"
|
||||
>
|
||||
↑ here
|
||||
</button>
|
||||
{/if}
|
||||
</div>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
Reference in New Issue
Block a user