Compare commits
3 Commits
fb6b364382
...
1d00fd4e2e
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1d00fd4e2e | ||
|
|
a54d8d43aa | ||
|
|
97e7a8dc02 |
@@ -45,6 +45,14 @@ type Config struct {
|
||||
// that one book instead of walking the full catalogue.
|
||||
SingleBookURL string
|
||||
|
||||
// FromChapter, when > 0, skips chapters with number < FromChapter.
|
||||
// Only effective in single-book mode.
|
||||
FromChapter int
|
||||
|
||||
// ToChapter, when > 0, skips chapters with number > ToChapter.
|
||||
// Only effective in single-book mode. 0 means "no upper limit".
|
||||
ToChapter int
|
||||
|
||||
// OnProgress is called periodically with the current progress counters.
|
||||
// It is always called on completion (success or failure). May be nil.
|
||||
OnProgress func(p Progress)
|
||||
@@ -204,6 +212,15 @@ func (o *Orchestrator) Run(ctx context.Context) error {
|
||||
|
||||
// Enqueue chapter jobs.
|
||||
for _, ref := range refs {
|
||||
// Apply chapter range filter (only in single-book mode when set).
|
||||
if o.cfg.FromChapter > 0 && ref.Number < o.cfg.FromChapter {
|
||||
chaptersSkipped.Add(1)
|
||||
continue
|
||||
}
|
||||
if o.cfg.ToChapter > 0 && ref.Number > o.cfg.ToChapter {
|
||||
chaptersSkipped.Add(1)
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -474,3 +475,101 @@ func attrVal(n *html.Node, key string) string { return htmlutil.AttrVal(n, key)
|
||||
// textContent returns the concatenated text content of a node and its descendants.
|
||||
// Delegates to htmlutil.TextContent.
|
||||
func textContent(n *html.Node) string { return htmlutil.TextContent(n) }
|
||||
|
||||
// ─── Search API ───────────────────────────────────────────────────────────────
|
||||
|
||||
// handleSearch handles GET /api/search.
|
||||
//
|
||||
// Query params:
|
||||
//
|
||||
// q — search query string (required, min 2 chars)
|
||||
// source — "local" | "remote" | "all" (default: "all")
|
||||
//
|
||||
// When source includes "local", it searches books already in the local store
|
||||
// by title substring match. When source includes "remote", it fetches the
|
||||
// novelfire.net search page and parses results. Results from both sources
|
||||
// are merged with local results first (de-duplicated by slug).
|
||||
//
|
||||
// Returns JSON: {"results": [...NovelListing], "local_count": N, "remote_count": N}
|
||||
func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query().Get("q")
|
||||
if len([]rune(q)) < 2 {
|
||||
http.Error(w, `{"error":"query must be at least 2 characters"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
source := r.URL.Query().Get("source")
|
||||
if source == "" {
|
||||
source = "all"
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second)
|
||||
defer cancel()
|
||||
|
||||
var localResults []NovelListing
|
||||
var remoteResults []NovelListing
|
||||
|
||||
// ── Local search (PocketBase books) ──────────────────────────────────
|
||||
if source == "local" || source == "all" {
|
||||
books, err := s.store.ListBooks(ctx)
|
||||
if err != nil {
|
||||
s.log.Warn("search: ListBooks failed", "err", err)
|
||||
} else {
|
||||
qLower := strings.ToLower(q)
|
||||
for _, b := range books {
|
||||
if strings.Contains(strings.ToLower(b.Title), qLower) ||
|
||||
strings.Contains(strings.ToLower(b.Author), qLower) {
|
||||
listing := NovelListing{
|
||||
Slug: b.Slug,
|
||||
Title: b.Title,
|
||||
Cover: b.Cover,
|
||||
URL: b.SourceURL,
|
||||
}
|
||||
localResults = append(localResults, listing)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Remote search (novelfire.net /search?keyword=...) ─────────────────
|
||||
if source == "remote" || source == "all" {
|
||||
searchURL := novelFireBase + "/search?keyword=" + url.QueryEscape(q)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil)
|
||||
if err == nil {
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
if resp, fetchErr := http.DefaultClient.Do(req); fetchErr == nil {
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
parsed, _ := parseBrowsePage(resp.Body)
|
||||
remoteResults = parsed
|
||||
} else {
|
||||
s.log.Warn("search: remote returned non-200", "status", resp.StatusCode, "url", searchURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Merge: de-duplicate remote results already in local ───────────────
|
||||
localSlugs := make(map[string]bool, len(localResults))
|
||||
for _, item := range localResults {
|
||||
localSlugs[item.Slug] = true
|
||||
}
|
||||
|
||||
combined := make([]NovelListing, 0, len(localResults)+len(remoteResults))
|
||||
combined = append(combined, localResults...)
|
||||
for _, item := range remoteResults {
|
||||
if !localSlugs[item.Slug] {
|
||||
combined = append(combined, item)
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||
"results": combined,
|
||||
"local_count": len(localResults),
|
||||
"remote_count": len(remoteResults),
|
||||
})
|
||||
}
|
||||
|
||||
147
scraper/internal/server/handlers_preview.go
Normal file
147
scraper/internal/server/handlers_preview.go
Normal file
@@ -0,0 +1,147 @@
|
||||
package server
|
||||
|
||||
// handlers_preview.go — on-demand preview endpoints for books not yet in PocketBase.
|
||||
//
|
||||
// These endpoints allow the UI to display a book's metadata and chapter list
|
||||
// (scraped live from novelfire.net) without requiring a full scrape to have
|
||||
// been run first. They are read-only: nothing is persisted to PocketBase or
|
||||
// MinIO.
|
||||
//
|
||||
// Endpoints:
|
||||
//
|
||||
// GET /api/book-preview/{slug} — scrape book metadata + chapter list live
|
||||
// GET /api/chapter-text-preview/{slug}/{n} — scrape a single chapter text live
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/libnovel/scraper/internal/scraper"
|
||||
)
|
||||
|
||||
// BookPreviewResponse is the JSON response for /api/book-preview/{slug}.
|
||||
type BookPreviewResponse struct {
|
||||
InLib bool `json:"in_lib"`
|
||||
Meta scraper.BookMeta `json:"meta"`
|
||||
Chapters []scraper.ChapterRef `json:"chapters"`
|
||||
}
|
||||
|
||||
// handleBookPreview handles GET /api/book-preview/{slug}.
|
||||
//
|
||||
// It scrapes book metadata and the full chapter list live from novelfire.net.
|
||||
// It also checks whether the book exists in the local store (PocketBase) and
|
||||
// sets the InLib flag accordingly. Nothing is written to any store.
|
||||
//
|
||||
// Query param: source_url (optional) — if provided, uses that URL instead of
|
||||
// constructing one from the slug.
|
||||
func (s *Server) handleBookPreview(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
if slug == "" {
|
||||
http.Error(w, `{"error":"missing slug"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Determine the book URL: prefer explicit source_url query param.
|
||||
bookURL := r.URL.Query().Get("source_url")
|
||||
if bookURL == "" {
|
||||
bookURL = fmt.Sprintf("%s/book/%s", novelFireBase, slug)
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
|
||||
// Check whether the book is already in the local library.
|
||||
_, inLib, err := s.store.ReadMetadata(ctx, slug)
|
||||
if err != nil {
|
||||
// Non-fatal: we can still serve the preview.
|
||||
s.log.Warn("book-preview: ReadMetadata failed", "slug", slug, "err", err)
|
||||
inLib = false
|
||||
}
|
||||
|
||||
// Scrape live metadata.
|
||||
meta, err := s.novel.ScrapeMetadata(ctx, bookURL)
|
||||
if err != nil {
|
||||
s.log.Error("book-preview: ScrapeMetadata failed", "slug", slug, "url", bookURL, "err", err)
|
||||
http.Error(w, fmt.Sprintf(`{"error":"metadata scrape failed: %s"}`, err.Error()), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
// Scrape live chapter list.
|
||||
chapters, err := s.novel.ScrapeChapterList(ctx, bookURL)
|
||||
if err != nil {
|
||||
s.log.Error("book-preview: ScrapeChapterList failed", "slug", slug, "url", bookURL, "err", err)
|
||||
// Return partial response with metadata only — chapters are non-critical.
|
||||
chapters = []scraper.ChapterRef{}
|
||||
}
|
||||
|
||||
resp := BookPreviewResponse{
|
||||
InLib: inLib,
|
||||
Meta: meta,
|
||||
Chapters: chapters,
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
|
||||
// ChapterPreviewResponse is the JSON response for /api/chapter-text-preview/{slug}/{n}.
|
||||
type ChapterPreviewResponse struct {
|
||||
Slug string `json:"slug"`
|
||||
Number int `json:"number"`
|
||||
Title string `json:"title"`
|
||||
Text string `json:"text"` // plain text (markdown stripped)
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// handleChapterTextPreview handles GET /api/chapter-text-preview/{slug}/{n}.
|
||||
//
|
||||
// It scrapes a single chapter from novelfire.net live without storing anything.
|
||||
// The chapter URL is determined from either:
|
||||
// - the "chapter_url" query param (preferred — used when the UI knows it from
|
||||
// a prior book-preview call), or
|
||||
// - a best-effort construction: {novelFireBase}/book/{slug}/chapter-{n}
|
||||
//
|
||||
// Returns plain text (markdown stripped) suitable for TTS or display.
|
||||
func (s *Server) handleChapterTextPreview(w http.ResponseWriter, r *http.Request) {
|
||||
slug := r.PathValue("slug")
|
||||
nStr := r.PathValue("n")
|
||||
n, err := strconv.Atoi(nStr)
|
||||
if err != nil || n < 1 || slug == "" {
|
||||
http.Error(w, `{"error":"invalid params"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Chapter URL: prefer explicit query param.
|
||||
chapterURL := r.URL.Query().Get("chapter_url")
|
||||
if chapterURL == "" {
|
||||
chapterURL = fmt.Sprintf("%s/book/%s/chapter-%d", novelFireBase, slug, n)
|
||||
}
|
||||
|
||||
title := r.URL.Query().Get("title")
|
||||
|
||||
ref := scraper.ChapterRef{
|
||||
Number: n,
|
||||
Title: title,
|
||||
URL: chapterURL,
|
||||
}
|
||||
|
||||
chapter, err := s.novel.ScrapeChapterText(r.Context(), ref)
|
||||
if err != nil {
|
||||
s.log.Error("chapter-text-preview: ScrapeChapterText failed",
|
||||
"slug", slug, "n", n, "url", chapterURL, "err", err)
|
||||
http.Error(w, fmt.Sprintf(`{"error":"chapter scrape failed: %s"}`, err.Error()), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
|
||||
resp := ChapterPreviewResponse{
|
||||
Slug: slug,
|
||||
Number: n,
|
||||
Title: chapter.Ref.Title,
|
||||
Text: stripMarkdown(chapter.Text),
|
||||
URL: chapterURL,
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
@@ -33,6 +33,29 @@ func (s *Server) handleScrapeBook(w http.ResponseWriter, r *http.Request) {
|
||||
s.runAsync(w, cfg)
|
||||
}
|
||||
|
||||
// handleScrapeBookRange handles POST /api/scrape/book/range.
|
||||
// Body: {"url": "...", "from": N, "to": M}
|
||||
// Scrapes only chapters in the range [from, to] (inclusive).
|
||||
// from=0 means "start from chapter 1"; to=0 means "no upper limit".
|
||||
func (s *Server) handleScrapeBookRange(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
URL string `json:"url"`
|
||||
From int `json:"from"`
|
||||
To int `json:"to"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.URL == "" {
|
||||
http.Error(w, `{"error":"request body must be JSON with \"url\" field"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
cfg := s.oCfg
|
||||
cfg.SingleBookURL = body.URL
|
||||
cfg.FromChapter = body.From
|
||||
cfg.ToChapter = body.To
|
||||
|
||||
s.runAsync(w, cfg)
|
||||
}
|
||||
|
||||
// runAsync launches an orchestrator in the background and returns 202 Accepted.
|
||||
// Only one scrape job runs at a time; concurrent requests receive 409 Conflict.
|
||||
func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
|
||||
|
||||
@@ -139,6 +139,7 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
mux.HandleFunc("GET /health", s.handleHealth)
|
||||
mux.HandleFunc("POST /scrape", s.handleScrapeCatalogue)
|
||||
mux.HandleFunc("POST /scrape/book", s.handleScrapeBook)
|
||||
mux.HandleFunc("POST /scrape/book/range", s.handleScrapeBookRange)
|
||||
// Browse API — fetches and parses novelfire catalogue page
|
||||
mux.HandleFunc("GET /api/browse", s.handleBrowse)
|
||||
// Ranking API
|
||||
@@ -150,6 +151,11 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
|
||||
mux.HandleFunc("GET /api/scrape/tasks", s.handleScrapeTasks)
|
||||
// Re-index chapters for a book from MinIO into PocketBase chapters_idx
|
||||
mux.HandleFunc("POST /api/reindex/{slug}", s.handleReindex)
|
||||
// On-demand preview (no store writes) — for books not yet in the library
|
||||
mux.HandleFunc("GET /api/book-preview/{slug}", s.handleBookPreview)
|
||||
mux.HandleFunc("GET /api/chapter-text-preview/{slug}/{n}", s.handleChapterTextPreview)
|
||||
// Search: local PocketBase + remote novelfire.net
|
||||
mux.HandleFunc("GET /api/search", s.handleSearch)
|
||||
// Progress API
|
||||
mux.HandleFunc("GET /api/progress", s.handleGetProgress)
|
||||
mux.HandleFunc("POST /api/progress/{slug}", s.handleSetProgress)
|
||||
|
||||
62
ui/src/routes/api/scrape/range/+server.ts
Normal file
62
ui/src/routes/api/scrape/range/+server.ts
Normal file
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* POST /api/scrape/range
|
||||
*
|
||||
* Proxies range-scrape requests to the Go scraper backend at POST /scrape/book/range.
|
||||
* Admin-only.
|
||||
*
|
||||
* Request body (JSON):
|
||||
* { "url": "https://novelfire.net/book/...", "from": 50, "to": 100 }
|
||||
* "to" is optional — omit to scrape from "from" to the end.
|
||||
*
|
||||
* Responses mirror the Go scraper:
|
||||
* 202 Accepted — job enqueued
|
||||
* 409 Conflict — a scrape job is already running
|
||||
* 400 Bad Request
|
||||
* 403 Forbidden — not an admin
|
||||
*/
|
||||
|
||||
import { json, error } from '@sveltejs/kit';
|
||||
import type { RequestHandler } from './$types';
|
||||
import { env } from '$env/dynamic/private';
|
||||
import { log } from '$lib/server/logger';
|
||||
|
||||
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
|
||||
|
||||
export const POST: RequestHandler = async ({ request, locals }) => {
|
||||
// Admin guard
|
||||
if (!locals.user || locals.user.role !== 'admin') {
|
||||
throw error(403, 'Forbidden');
|
||||
}
|
||||
|
||||
let body: { url?: string; from?: number; to?: number } = {};
|
||||
try {
|
||||
body = await request.json();
|
||||
} catch {
|
||||
throw error(400, 'Invalid JSON body');
|
||||
}
|
||||
|
||||
if (!body.url || typeof body.from !== 'number') {
|
||||
throw error(400, 'url and from are required');
|
||||
}
|
||||
|
||||
const upstream = `${SCRAPER_URL}/scrape/book/range`;
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(upstream, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ url: body.url, from: body.from, to: body.to })
|
||||
});
|
||||
} catch (e) {
|
||||
log.error('scrape/range', 'scraper proxy network error', { err: String(e) });
|
||||
throw error(502, 'Could not reach scraper');
|
||||
}
|
||||
|
||||
if (!res.ok && res.status >= 500) {
|
||||
const text = await res.text().catch(() => '');
|
||||
log.error('scrape/range', 'scraper returned error', { status: res.status, body: text });
|
||||
}
|
||||
|
||||
const data = await res.json().catch(() => ({}));
|
||||
return json(data, { status: res.status });
|
||||
};
|
||||
@@ -2,34 +2,99 @@ import { error } from '@sveltejs/kit';
|
||||
import type { PageServerLoad } from './$types';
|
||||
import { getBook, listChapterIdx, getProgress } from '$lib/server/pocketbase';
|
||||
import { log } from '$lib/server/logger';
|
||||
import { env } from '$env/dynamic/private';
|
||||
|
||||
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
|
||||
|
||||
// Minimal chapter shape returned by /api/book-preview
|
||||
export interface PreviewChapter {
|
||||
number: number;
|
||||
title: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export const load: PageServerLoad = async ({ params, locals }) => {
|
||||
const { slug } = params;
|
||||
|
||||
let book: Awaited<ReturnType<typeof getBook>>;
|
||||
let chapters: Awaited<ReturnType<typeof listChapterIdx>>;
|
||||
let progress: Awaited<ReturnType<typeof getProgress>>;
|
||||
// Try fetching from PocketBase first
|
||||
let book = await getBook(slug).catch((e) => {
|
||||
log.error('books', 'getBook failed', { slug, err: String(e) });
|
||||
return null;
|
||||
});
|
||||
|
||||
try {
|
||||
[book, chapters, progress] = await Promise.all([
|
||||
getBook(slug),
|
||||
listChapterIdx(slug),
|
||||
getProgress(locals.sessionId, slug, locals.user?.id)
|
||||
]);
|
||||
} catch (e) {
|
||||
log.error('books', 'failed to load book page', { slug, err: String(e) });
|
||||
throw error(500, 'Failed to load book');
|
||||
if (book) {
|
||||
// Book is in the library — normal path
|
||||
let chapters, progress;
|
||||
try {
|
||||
[chapters, progress] = await Promise.all([
|
||||
listChapterIdx(slug),
|
||||
getProgress(locals.sessionId, slug, locals.user?.id)
|
||||
]);
|
||||
} catch (e) {
|
||||
log.error('books', 'failed to load book page data', { slug, err: String(e) });
|
||||
throw error(500, 'Failed to load book');
|
||||
}
|
||||
|
||||
return {
|
||||
book,
|
||||
chapters,
|
||||
previewChapters: null as PreviewChapter[] | null,
|
||||
inLib: true,
|
||||
lastChapter: progress?.chapter ?? null,
|
||||
isAdmin: locals.user?.role === 'admin'
|
||||
};
|
||||
}
|
||||
|
||||
if (!book) {
|
||||
log.warn('books', 'book not found', { slug });
|
||||
// Book not in PocketBase — try live preview from scraper
|
||||
try {
|
||||
const res = await fetch(`${SCRAPER_URL}/api/book-preview/${encodeURIComponent(slug)}`);
|
||||
if (!res.ok) {
|
||||
log.warn('books', 'book-preview returned error', { slug, status: res.status });
|
||||
error(404, `Book "${slug}" not found`);
|
||||
}
|
||||
const preview: {
|
||||
in_lib: boolean;
|
||||
meta: {
|
||||
slug: string;
|
||||
title: string;
|
||||
author: string;
|
||||
cover: string;
|
||||
status: string;
|
||||
genres: string[];
|
||||
summary: string;
|
||||
total_chapters: number;
|
||||
source_url: string;
|
||||
};
|
||||
chapters: PreviewChapter[];
|
||||
} = await res.json();
|
||||
|
||||
// Shape the meta into a Book-like object (no PocketBase id fields)
|
||||
const previewBook = {
|
||||
id: '',
|
||||
slug: preview.meta.slug || slug,
|
||||
title: preview.meta.title,
|
||||
author: preview.meta.author,
|
||||
cover: preview.meta.cover,
|
||||
status: preview.meta.status,
|
||||
genres: preview.meta.genres ?? [],
|
||||
summary: preview.meta.summary,
|
||||
total_chapters: preview.meta.total_chapters,
|
||||
source_url: preview.meta.source_url,
|
||||
ranking: 0,
|
||||
meta_updated: ''
|
||||
};
|
||||
|
||||
return {
|
||||
book: previewBook,
|
||||
chapters: [],
|
||||
previewChapters: preview.chapters,
|
||||
inLib: preview.in_lib,
|
||||
lastChapter: null,
|
||||
isAdmin: locals.user?.role === 'admin'
|
||||
};
|
||||
} catch (e) {
|
||||
if (e instanceof Error && 'status' in e) throw e;
|
||||
log.error('books', 'book-preview fetch failed', { slug, err: String(e) });
|
||||
error(404, `Book "${slug}" not found`);
|
||||
}
|
||||
|
||||
return {
|
||||
book,
|
||||
chapters,
|
||||
lastChapter: progress?.chapter ?? null,
|
||||
isAdmin: locals.user?.role === 'admin'
|
||||
};
|
||||
};
|
||||
|
||||
@@ -17,9 +17,16 @@
|
||||
// Paginate chapter list — show 100 at a time
|
||||
let page = $state(0);
|
||||
const PAGE_SIZE = 100;
|
||||
const totalPages = $derived(Math.ceil(data.chapters.length / PAGE_SIZE));
|
||||
|
||||
// Use preview chapters if the book is not in the library
|
||||
const chapterList = $derived(
|
||||
data.inLib
|
||||
? data.chapters
|
||||
: (data.previewChapters ?? [])
|
||||
);
|
||||
const totalPages = $derived(Math.ceil(chapterList.length / PAGE_SIZE));
|
||||
const visibleChapters = $derived(
|
||||
data.chapters.slice(page * PAGE_SIZE, (page + 1) * PAGE_SIZE)
|
||||
chapterList.slice(page * PAGE_SIZE, (page + 1) * PAGE_SIZE)
|
||||
);
|
||||
|
||||
// ── Admin: rescrape ───────────────────────────────────────────────────────
|
||||
@@ -45,6 +52,55 @@
|
||||
scraping = false;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Admin: scrape range ───────────────────────────────────────────────────
|
||||
let rangeFrom = $state('');
|
||||
let rangeTo = $state('');
|
||||
let rangeScraping = $state(false);
|
||||
let rangeResult = $state<'queued' | 'busy' | 'error' | ''>('');
|
||||
|
||||
async function scrapeRange() {
|
||||
if (rangeScraping || !data.book.source_url) return;
|
||||
const from = parseInt(rangeFrom, 10);
|
||||
const to = parseInt(rangeTo, 10);
|
||||
if (!from || from < 1) return;
|
||||
rangeScraping = true;
|
||||
rangeResult = '';
|
||||
try {
|
||||
const res = await fetch('/api/scrape/range', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ url: data.book.source_url, from, to: to || undefined })
|
||||
});
|
||||
if (res.ok) rangeResult = 'queued';
|
||||
else if (res.status === 409) rangeResult = 'busy';
|
||||
else rangeResult = 'error';
|
||||
} catch {
|
||||
rangeResult = 'error';
|
||||
} finally {
|
||||
rangeScraping = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function scrapeFromChapter(n: number) {
|
||||
if (rangeScraping || !data.book.source_url) return;
|
||||
rangeScraping = true;
|
||||
rangeResult = '';
|
||||
try {
|
||||
const res = await fetch('/api/scrape/range', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ url: data.book.source_url, from: n })
|
||||
});
|
||||
if (res.ok) rangeResult = 'queued';
|
||||
else if (res.status === 409) rangeResult = 'busy';
|
||||
else rangeResult = 'error';
|
||||
} catch {
|
||||
rangeResult = 'error';
|
||||
} finally {
|
||||
rangeScraping = false;
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<svelte:head>
|
||||
@@ -62,7 +118,14 @@
|
||||
{/if}
|
||||
|
||||
<div class="flex flex-col gap-2 min-w-0">
|
||||
<h1 class="text-2xl font-bold text-zinc-100 leading-tight">{data.book.title}</h1>
|
||||
<div class="flex items-center gap-2 flex-wrap">
|
||||
<h1 class="text-2xl font-bold text-zinc-100 leading-tight">{data.book.title}</h1>
|
||||
{#if !data.inLib}
|
||||
<span class="text-xs px-2 py-0.5 rounded-full bg-zinc-700 text-zinc-400 border border-zinc-600 shrink-0">
|
||||
not in library
|
||||
</span>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
{#if data.book.author}
|
||||
<p class="text-zinc-400 text-sm">{data.book.author}</p>
|
||||
@@ -90,12 +153,12 @@
|
||||
Continue ch.{data.lastChapter}
|
||||
</a>
|
||||
{/if}
|
||||
{#if data.chapters.length > 0}
|
||||
{#if chapterList.length > 0}
|
||||
<a
|
||||
href="/books/{data.book.slug}/chapters/1"
|
||||
class="px-4 py-2 bg-zinc-700 text-zinc-100 font-semibold rounded text-sm hover:bg-zinc-600 transition-colors"
|
||||
>
|
||||
Start from ch.1
|
||||
{data.inLib ? 'Start from ch.1' : 'Preview ch.1'}
|
||||
</a>
|
||||
{/if}
|
||||
</div>
|
||||
@@ -104,13 +167,13 @@
|
||||
|
||||
<!-- Chapter list -->
|
||||
<div class="mt-4">
|
||||
<div class="flex items-center justify-between mb-3">
|
||||
<div class="flex items-center justify-between mb-3 flex-wrap gap-2">
|
||||
<h2 class="text-lg font-semibold text-zinc-100">
|
||||
Chapters
|
||||
<span class="text-zinc-500 font-normal text-sm ml-1">({data.chapters.length})</span>
|
||||
<span class="text-zinc-500 font-normal text-sm ml-1">({chapterList.length})</span>
|
||||
</h2>
|
||||
|
||||
<div class="flex items-center gap-3">
|
||||
<div class="flex items-center gap-3 flex-wrap">
|
||||
{#if data.isAdmin && data.book.source_url}
|
||||
<button
|
||||
onclick={rescrape}
|
||||
@@ -169,35 +232,91 @@
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
{#if data.chapters.length === 0}
|
||||
<!-- Admin: range scrape controls -->
|
||||
{#if data.isAdmin && data.book.source_url}
|
||||
<div class="mb-4 p-3 rounded bg-zinc-800/60 border border-zinc-700 flex flex-wrap items-end gap-3">
|
||||
<div class="flex flex-col gap-1">
|
||||
<label class="text-xs text-zinc-500">From chapter</label>
|
||||
<input
|
||||
type="number"
|
||||
min="1"
|
||||
bind:value={rangeFrom}
|
||||
placeholder="1"
|
||||
class="w-24 px-2 py-1 rounded bg-zinc-700 border border-zinc-600 text-zinc-200 text-xs focus:outline-none focus:border-amber-400"
|
||||
/>
|
||||
</div>
|
||||
<div class="flex flex-col gap-1">
|
||||
<label class="text-xs text-zinc-500">To chapter (optional)</label>
|
||||
<input
|
||||
type="number"
|
||||
min="1"
|
||||
bind:value={rangeTo}
|
||||
placeholder="end"
|
||||
class="w-24 px-2 py-1 rounded bg-zinc-700 border border-zinc-600 text-zinc-200 text-xs focus:outline-none focus:border-amber-400"
|
||||
/>
|
||||
</div>
|
||||
<button
|
||||
onclick={scrapeRange}
|
||||
disabled={rangeScraping || !rangeFrom}
|
||||
class="px-3 py-1.5 rounded text-xs font-medium transition-colors
|
||||
{rangeScraping || !rangeFrom
|
||||
? 'bg-zinc-700 text-zinc-500 cursor-not-allowed'
|
||||
: 'bg-amber-500/20 text-amber-300 hover:bg-amber-500/40 border border-amber-500/30'}"
|
||||
>
|
||||
{rangeScraping ? 'Queuing…' : 'Scrape range'}
|
||||
</button>
|
||||
|
||||
{#if rangeResult}
|
||||
<span class="text-xs {rangeResult === 'queued' ? 'text-green-400' : rangeResult === 'busy' ? 'text-amber-400' : 'text-red-400'}">
|
||||
{rangeResult === 'queued' ? 'Range scrape queued.' : rangeResult === 'busy' ? 'Scraper busy.' : 'Error queuing.'}
|
||||
</span>
|
||||
{/if}
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
{#if chapterList.length === 0}
|
||||
<p class="text-zinc-500 text-sm">No chapters available yet.</p>
|
||||
{:else}
|
||||
<div class="grid grid-cols-1 sm:grid-cols-2 gap-1">
|
||||
{#each visibleChapters as chapter}
|
||||
{@const isCurrent = data.lastChapter === chapter.number}
|
||||
<a
|
||||
href="/books/{data.book.slug}/chapters/{chapter.number}"
|
||||
class="flex items-center gap-3 px-3 py-2 rounded hover:bg-zinc-800 transition-colors group {isCurrent
|
||||
? 'bg-zinc-800'
|
||||
: ''}"
|
||||
>
|
||||
<span
|
||||
class="text-xs font-mono w-10 text-right flex-shrink-0 {isCurrent
|
||||
? 'text-amber-400'
|
||||
: 'text-zinc-500'}"
|
||||
{@const chapterUrl = data.inLib
|
||||
? `/books/${data.book.slug}/chapters/${chapter.number}`
|
||||
: `/books/${data.book.slug}/chapters/${chapter.number}?preview=1&chapter_url=${encodeURIComponent((chapter as { url?: string }).url ?? '')}&title=${encodeURIComponent(chapter.title ?? '')}`}
|
||||
<div class="flex items-center gap-3 px-3 py-2 rounded hover:bg-zinc-800 transition-colors group {isCurrent ? 'bg-zinc-800' : ''}">
|
||||
<a
|
||||
href={chapterUrl}
|
||||
class="flex items-center gap-3 flex-1 min-w-0"
|
||||
>
|
||||
{chapter.number}
|
||||
</span>
|
||||
<span class="text-sm text-zinc-300 group-hover:text-zinc-100 truncate flex-1">
|
||||
{chapter.title || `Chapter ${chapter.number}`}
|
||||
</span>
|
||||
{#if isCurrent}
|
||||
<span class="text-xs text-amber-400 flex-shrink-0">reading</span>
|
||||
<span
|
||||
class="text-xs font-mono w-10 text-right flex-shrink-0 {isCurrent
|
||||
? 'text-amber-400'
|
||||
: 'text-zinc-500'}"
|
||||
>
|
||||
{chapter.number}
|
||||
</span>
|
||||
<span class="text-sm text-zinc-300 group-hover:text-zinc-100 truncate flex-1">
|
||||
{chapter.title || `Chapter ${chapter.number}`}
|
||||
</span>
|
||||
{#if isCurrent}
|
||||
<span class="text-xs text-amber-400 flex-shrink-0">reading</span>
|
||||
{/if}
|
||||
{#if (chapter as { date_label?: string }).date_label}
|
||||
<span class="text-xs text-zinc-600 flex-shrink-0 hidden sm:block">{(chapter as { date_label?: string }).date_label}</span>
|
||||
{/if}
|
||||
</a>
|
||||
<!-- Admin: scrape from this chapter up -->
|
||||
{#if data.isAdmin && data.book.source_url && data.inLib}
|
||||
<button
|
||||
onclick={() => scrapeFromChapter(chapter.number)}
|
||||
disabled={rangeScraping}
|
||||
class="opacity-0 group-hover:opacity-100 shrink-0 text-xs px-1.5 py-0.5 rounded bg-amber-500/10 text-amber-500 hover:bg-amber-500/30 transition-all border border-amber-500/20 disabled:opacity-30"
|
||||
title="Scrape from chapter {chapter.number} up"
|
||||
>
|
||||
↑ here
|
||||
</button>
|
||||
{/if}
|
||||
{#if chapter.date_label}
|
||||
<span class="text-xs text-zinc-600 flex-shrink-0 hidden sm:block">{chapter.date_label}</span>
|
||||
{/if}
|
||||
</a>
|
||||
</div>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
@@ -8,12 +8,81 @@ import { env } from '$env/dynamic/private';
|
||||
|
||||
const SCRAPER_URL = env.SCRAPER_API_URL ?? 'http://localhost:8080';
|
||||
|
||||
export const load: PageServerLoad = async ({ params, locals }) => {
|
||||
export const load: PageServerLoad = async ({ params, url, locals }) => {
|
||||
const { slug } = params;
|
||||
const n = parseInt(params.n, 10);
|
||||
|
||||
if (!n || n < 1) error(400, 'Invalid chapter number');
|
||||
|
||||
const isPreview = url.searchParams.get('preview') === '1';
|
||||
const chapterUrl = url.searchParams.get('chapter_url') ?? '';
|
||||
const chapterTitle = url.searchParams.get('title') ?? '';
|
||||
|
||||
if (isPreview) {
|
||||
// ── Preview path: scrape chapter live, nothing from PocketBase/MinIO ──
|
||||
const previewParams = new URLSearchParams();
|
||||
if (chapterUrl) previewParams.set('chapter_url', chapterUrl);
|
||||
if (chapterTitle) previewParams.set('title', chapterTitle);
|
||||
|
||||
let chapterData: { slug: string; number: number; title: string; text: string; url: string };
|
||||
try {
|
||||
const res = await fetch(
|
||||
`${SCRAPER_URL}/api/chapter-text-preview/${encodeURIComponent(slug)}/${n}?${previewParams.toString()}`
|
||||
);
|
||||
if (!res.ok) {
|
||||
log.error('chapter', 'chapter-text-preview returned error', { slug, n, status: res.status });
|
||||
error(404, `Chapter ${n} not found`);
|
||||
}
|
||||
chapterData = await res.json();
|
||||
} catch (e) {
|
||||
if (e instanceof Error && 'status' in e) throw e;
|
||||
log.error('chapter', 'chapter-text-preview fetch failed', { slug, n, err: String(e) });
|
||||
error(502, 'Could not fetch chapter preview');
|
||||
}
|
||||
|
||||
// Wrap plain text in minimal HTML paragraphs for display
|
||||
const html = chapterData.text
|
||||
? '<p>' + chapterData.text.replace(/\n{2,}/g, '</p><p>').replace(/\n/g, '<br>') + '</p>'
|
||||
: '';
|
||||
|
||||
// Fetch voices (non-critical for preview)
|
||||
let voices: string[] = [];
|
||||
try {
|
||||
const vRes = await fetch(`${SCRAPER_URL}/api/voices`);
|
||||
if (vRes.ok) {
|
||||
const d = (await vRes.json()) as { voices: string[] };
|
||||
voices = d.voices ?? [];
|
||||
}
|
||||
} catch {
|
||||
// Non-critical
|
||||
}
|
||||
|
||||
// Try to get book title/cover from PocketBase for breadcrumbs; fall back to slug
|
||||
const pb = await getBook(slug).catch(() => null);
|
||||
|
||||
return {
|
||||
book: {
|
||||
slug,
|
||||
title: pb?.title ?? slug,
|
||||
cover: pb?.cover ?? ''
|
||||
},
|
||||
chapter: {
|
||||
id: '',
|
||||
slug,
|
||||
number: n,
|
||||
title: chapterData.title || `Chapter ${n}`,
|
||||
date_label: ''
|
||||
},
|
||||
html,
|
||||
voices,
|
||||
prev: null as number | null,
|
||||
next: null as number | null,
|
||||
sessionId: locals.sessionId,
|
||||
isPreview: true
|
||||
};
|
||||
}
|
||||
|
||||
// ── Normal path: fetch from PocketBase + MinIO ─────────────────────────
|
||||
// Fetch book metadata, chapter index, and voice list in parallel
|
||||
const [book, chapters, voicesRes] = await Promise.all([
|
||||
getBook(slug),
|
||||
@@ -40,8 +109,8 @@ export const load: PageServerLoad = async ({ params, locals }) => {
|
||||
// Get presigned URL and fetch chapter markdown server-side
|
||||
let html = '';
|
||||
try {
|
||||
const url = await presignChapter(slug, n);
|
||||
const res = await fetch(url);
|
||||
const presignUrl = await presignChapter(slug, n);
|
||||
const res = await fetch(presignUrl);
|
||||
if (!res.ok) throw new Error(`MinIO returned ${res.status}`);
|
||||
const markdown = await res.text();
|
||||
html = await marked(markdown, { async: true });
|
||||
@@ -60,6 +129,7 @@ export const load: PageServerLoad = async ({ params, locals }) => {
|
||||
voices,
|
||||
prev: prevChapter ? prevChapter.number : null,
|
||||
next: nextChapter ? nextChapter.number : null,
|
||||
sessionId: locals.sessionId
|
||||
sessionId: locals.sessionId,
|
||||
isPreview: false
|
||||
};
|
||||
};
|
||||
|
||||
@@ -5,8 +5,9 @@
|
||||
|
||||
let { data }: { data: PageData } = $props();
|
||||
|
||||
// Record reading progress when the chapter is opened
|
||||
// Record reading progress when the chapter is opened (skip for preview chapters)
|
||||
onMount(async () => {
|
||||
if (data.isPreview) return;
|
||||
try {
|
||||
await fetch('/api/progress', {
|
||||
method: 'POST',
|
||||
@@ -67,6 +68,7 @@
|
||||
</div>
|
||||
|
||||
<!-- Audio player -->
|
||||
{#if !data.isPreview}
|
||||
<AudioPlayer
|
||||
slug={data.book.slug}
|
||||
chapter={data.chapter.number}
|
||||
@@ -76,6 +78,11 @@
|
||||
nextChapter={data.next}
|
||||
voices={data.voices}
|
||||
/>
|
||||
{:else}
|
||||
<div class="mb-6 px-4 py-3 rounded bg-zinc-800/60 border border-zinc-700 text-zinc-500 text-sm">
|
||||
Preview chapter — audio not available for books outside the library.
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<!-- Chapter content -->
|
||||
{#if !data.html}
|
||||
|
||||
@@ -25,10 +25,52 @@ export const load: PageServerLoad = async ({ url, locals }) => {
|
||||
const genre = url.searchParams.get('genre') ?? 'all';
|
||||
const sort = url.searchParams.get('sort') ?? 'popular';
|
||||
const status = url.searchParams.get('status') ?? 'all';
|
||||
const q = url.searchParams.get('q') ?? '';
|
||||
|
||||
let novels: NovelListing[] = [];
|
||||
let pageNum = parseInt(page, 10) || 1;
|
||||
let hasNext = false;
|
||||
let searchQuery = '';
|
||||
let searchLocalCount = 0;
|
||||
let searchRemoteCount = 0;
|
||||
|
||||
// ── Search mode: ?q= overrides browse/ranking ─────────────────────────
|
||||
if (q.trim().length >= 2) {
|
||||
searchQuery = q.trim();
|
||||
const apiURL = `${SCRAPER_URL}/api/search?q=${encodeURIComponent(searchQuery)}`;
|
||||
try {
|
||||
const res = await fetch(apiURL);
|
||||
if (!res.ok) {
|
||||
log.error('browse', 'search returned error', { status: res.status });
|
||||
throw error(502, `Search failed: ${res.status}`);
|
||||
}
|
||||
const data: {
|
||||
results: NovelListing[];
|
||||
local_count: number;
|
||||
remote_count: number;
|
||||
} = await res.json();
|
||||
novels = data.results ?? [];
|
||||
searchLocalCount = data.local_count ?? 0;
|
||||
searchRemoteCount = data.remote_count ?? 0;
|
||||
} catch (e) {
|
||||
if (e instanceof Error && 'status' in e) throw e;
|
||||
log.error('browse', 'search network error', { q: searchQuery, err: String(e) });
|
||||
throw error(502, 'Could not reach search service');
|
||||
}
|
||||
|
||||
return {
|
||||
novels,
|
||||
page: 1,
|
||||
hasNext: false,
|
||||
genre,
|
||||
sort,
|
||||
status,
|
||||
isAdmin: locals.user?.role === 'admin',
|
||||
searchQuery,
|
||||
searchLocalCount,
|
||||
searchRemoteCount
|
||||
};
|
||||
}
|
||||
|
||||
if (sort === 'rank') {
|
||||
// Ranking view: fetch from /api/ranking which returns richer metadata.
|
||||
@@ -99,7 +141,10 @@ export const load: PageServerLoad = async ({ url, locals }) => {
|
||||
genre,
|
||||
sort,
|
||||
status,
|
||||
isAdmin: locals.user?.role === 'admin'
|
||||
isAdmin: locals.user?.role === 'admin',
|
||||
searchQuery: '',
|
||||
searchLocalCount: 0,
|
||||
searchRemoteCount: 0
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@
|
||||
// When sort=rank the ranking API is used — pagination + genre/status filters
|
||||
// don't apply to that endpoint.
|
||||
const isRankView = $derived(data.sort === 'rank');
|
||||
const isSearchView = $derived(!!data.searchQuery);
|
||||
|
||||
function buildURL(overrides: Record<string, string | number>) {
|
||||
const params = new URLSearchParams({
|
||||
@@ -96,7 +97,12 @@
|
||||
<div>
|
||||
<h1 class="text-2xl font-bold text-zinc-100">Discover</h1>
|
||||
<p class="text-zinc-400 text-sm mt-1">
|
||||
{#if isRankView}
|
||||
{#if isSearchView}
|
||||
{data.novels.length} result{data.novels.length !== 1 ? 's' : ''} for "<span class="text-zinc-200">{data.searchQuery}</span>"
|
||||
{#if data.searchLocalCount > 0 || data.searchRemoteCount > 0}
|
||||
<span class="text-zinc-500 text-xs ml-1">({data.searchLocalCount} local, {data.searchRemoteCount} from novelfire)</span>
|
||||
{/if}
|
||||
{:else if isRankView}
|
||||
{#if data.novels.length > 0}
|
||||
{data.novels.length} novels ranked from last catalogue scrape
|
||||
{:else}
|
||||
@@ -105,8 +111,7 @@
|
||||
{:else}
|
||||
Browse novels from novelfire.net
|
||||
{/if}
|
||||
</p>
|
||||
</div>
|
||||
</p> </div>
|
||||
|
||||
<div class="flex items-center gap-3 flex-wrap">
|
||||
<!-- View toggle -->
|
||||
@@ -181,6 +186,31 @@
|
||||
{/if}
|
||||
{/if}
|
||||
|
||||
<!-- Search -->
|
||||
<form method="GET" action="/browse" class="flex gap-2 mb-4">
|
||||
<input
|
||||
type="search"
|
||||
name="q"
|
||||
value={data.searchQuery}
|
||||
placeholder="Search novels by title or author…"
|
||||
class="flex-1 bg-zinc-800 border border-zinc-700 text-zinc-200 text-sm rounded px-3 py-1.5 focus:outline-none focus:border-amber-400 placeholder-zinc-500"
|
||||
/>
|
||||
<button
|
||||
type="submit"
|
||||
class="px-4 py-1.5 rounded bg-amber-400 text-zinc-900 text-sm font-semibold hover:bg-amber-300 transition-colors whitespace-nowrap"
|
||||
>
|
||||
Search
|
||||
</button>
|
||||
{#if data.searchQuery}
|
||||
<a
|
||||
href="/browse"
|
||||
class="px-3 py-1.5 rounded bg-zinc-700 text-zinc-300 text-sm hover:bg-zinc-600 transition-colors whitespace-nowrap"
|
||||
>
|
||||
Clear
|
||||
</a>
|
||||
{/if}
|
||||
</form>
|
||||
|
||||
<!-- Filters -->
|
||||
<form method="GET" action="/browse" class="flex flex-wrap gap-3 mb-6">
|
||||
<input type="hidden" name="page" value="1" />
|
||||
@@ -232,9 +262,11 @@
|
||||
<!-- Content -->
|
||||
{#if data.novels.length === 0}
|
||||
<div class="text-center py-20 text-zinc-500">
|
||||
<p class="text-lg">{isRankView ? 'No ranking data.' : 'No novels found.'}</p>
|
||||
<p class="text-lg">{isSearchView ? 'No results found.' : isRankView ? 'No ranking data.' : 'No novels found.'}</p>
|
||||
<p class="text-sm mt-2">
|
||||
{#if isRankView}
|
||||
{#if isSearchView}
|
||||
Try a different search term.
|
||||
{:else if isRankView}
|
||||
{#if data.isAdmin}
|
||||
Click <span class="text-amber-400">Refresh catalogue</span> above to trigger a full catalogue scrape.
|
||||
{:else}
|
||||
@@ -417,7 +449,7 @@
|
||||
{/if}
|
||||
|
||||
<!-- Pagination (browse mode only) -->
|
||||
{#if !isRankView && data.novels.length > 0}
|
||||
{#if !isRankView && !isSearchView && data.novels.length > 0}
|
||||
<div class="flex items-center justify-center gap-3 mt-8">
|
||||
{#if data.page > 1}
|
||||
<a
|
||||
|
||||
Reference in New Issue
Block a user