diff --git a/scraper/go.mod b/scraper/go.mod index 1ad5393..61b5e73 100644 --- a/scraper/go.mod +++ b/scraper/go.mod @@ -5,7 +5,6 @@ go 1.25.0 require ( github.com/gorilla/websocket v1.5.3 github.com/minio/minio-go/v7 v7.0.98 - github.com/yuin/goldmark v1.7.16 golang.org/x/net v0.51.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/scraper/go.sum b/scraper/go.sum index ebd83af..43ef929 100644 --- a/scraper/go.sum +++ b/scraper/go.sum @@ -31,8 +31,6 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= -github.com/yuin/goldmark v1.7.16 h1:n+CJdUxaFMiDUNnWC3dMWCIQJSkxH4uz3ZwQBkAlVNE= -github.com/yuin/goldmark v1.7.16/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= diff --git a/scraper/internal/server/helpers.go b/scraper/internal/server/helpers.go new file mode 100644 index 0000000..b8cda24 --- /dev/null +++ b/scraper/internal/server/helpers.go @@ -0,0 +1,62 @@ +package server + +import ( + "regexp" + "strings" +) + +// kokoroVoices is the built-in fallback list of voices shipped with Kokoro-FastAPI. +// Used when the live GET /v1/audio/voices request to Kokoro fails. +// Grouped by language prefix: +// +// af_ / am_ American English female / male +// bf_ / bm_ British English female / male +// ef_ / em_ Spanish female / male +// ff_ French female +// hf_ / hm_ Hindi female / male +// if_ / im_ Italian female / male +// jf_ / jm_ Japanese female / male +// pf_ / pm_ Portuguese female / male +// zf_ / zm_ Chinese female / male +var kokoroVoices = []string{ + // American English + "af_alloy", "af_aoede", "af_bella", "af_heart", "af_jadzia", + "af_jessica", "af_kore", "af_nicole", "af_nova", "af_river", + "af_sarah", "af_sky", + "am_adam", "am_echo", "am_eric", "am_fenrir", "am_liam", + "am_michael", "am_onyx", "am_puck", + // British English + "bf_alice", "bf_emma", "bf_lily", + "bm_daniel", "bm_fable", "bm_george", "bm_lewis", + // Spanish + "ef_dora", "em_alex", + // French + "ff_siwis", + // Hindi + "hf_alpha", "hf_beta", "hm_omega", "hm_psi", + // Italian + "if_sara", "im_nicola", + // Japanese + "jf_alpha", "jf_gongitsune", "jf_nezumi", "jf_tebukuro", "jm_kumo", + // Portuguese + "pf_dora", "pm_alex", + // Chinese + "zf_xiaobei", "zf_xiaoni", "zf_xiaoxiao", "zf_xiaoyi", + "zm_yunjian", "zm_yunxi", "zm_yunxia", "zm_yunyang", +} + +// stripMarkdown removes common markdown syntax from src, returning plain text +// suitable for TTS or display. Not a full markdown parser — handles the most +// common constructs (headings, bold/italic, code blocks, links, blockquotes). +func stripMarkdown(src string) string { + src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "") + src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "") + src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "") + src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "") + src = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(src, "$1") + src = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`).ReplaceAllString(src, "") + src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "") + src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "") + src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n") + return strings.TrimSpace(src) +} diff --git a/scraper/internal/server/server.go b/scraper/internal/server/server.go index 19c9006..9c8fd94 100644 --- a/scraper/internal/server/server.go +++ b/scraper/internal/server/server.go @@ -1,10 +1,18 @@ -// Package server exposes the scraper as an HTTP service. +// Package server exposes the scraper as an HTTP API service. // // Endpoints: // -// POST /scrape — enqueue a full catalogue scrape -// POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."}) -// GET /health — liveness probe +// POST /scrape — enqueue a full catalogue scrape +// POST /scrape/book — enqueue a single-book scrape (JSON body: {"url":"..."}) +// GET /health — liveness probe +// GET /api/progress — get reading progress map (session-scoped) +// POST /api/progress/{slug} — set reading progress +// DELETE /api/progress/{slug} — delete reading progress +// GET /api/presign/chapter/{slug}/{n} — presigned MinIO URL for chapter markdown +// GET /api/presign/audio/{slug}/{n} — presigned MinIO URL for chapter audio +// GET /api/chapter-text/{slug}/{n} — plain text of chapter (markdown stripped) +// POST /api/audio/{slug}/{n} — trigger Kokoro audio generation +// GET /api/audio-proxy/{slug}/{n} — proxy generated audio from Kokoro package server import ( @@ -118,18 +126,6 @@ func (s *Server) ListenAndServe(ctx context.Context) error { // Presigned URL API (for SvelteKit UI) mux.HandleFunc("GET /api/presign/chapter/{slug}/{n}", s.handlePresignChapter) mux.HandleFunc("GET /api/presign/audio/{slug}/{n}", s.handlePresignAudio) - // UI routes - mux.HandleFunc("GET /", s.handleHome) - mux.HandleFunc("GET /scrape", s.handleScrape) - mux.HandleFunc("GET /ranking", s.handleRanking) - mux.HandleFunc("POST /ranking/refresh", s.handleRankingRefresh) - mux.HandleFunc("GET /ranking/view", s.handleRankingView) - mux.HandleFunc("GET /books/{slug}", s.handleBook) - mux.HandleFunc("GET /books/{slug}/chapters/{n}", s.handleChapter) - mux.HandleFunc("GET /books/{slug}/chapters-page", s.handleBookChaptersPage) - mux.HandleFunc("POST /ui/scrape/book", s.handleUIScrapeBook) - mux.HandleFunc("GET /ui/scrape/status", s.handleUIScrapeStatus) - mux.HandleFunc("GET /ui/ranking/status", s.handleRankingStatus) // Plain-text chapter content (used server-side for audio generation) mux.HandleFunc("GET /api/chapter-text/{slug}/{n}", s.handleChapterText) // Server-side audio generation via Kokoro /v1/audio/speech. diff --git a/scraper/internal/server/ui.go b/scraper/internal/server/ui.go deleted file mode 100644 index f4e471c..0000000 --- a/scraper/internal/server/ui.go +++ /dev/null @@ -1,3289 +0,0 @@ -package server - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "html/template" - "net/http" - "regexp" - "sort" - "strconv" - "strings" - "time" - - "github.com/libnovel/scraper/internal/orchestrator" - "github.com/libnovel/scraper/internal/scraper" - "github.com/libnovel/scraper/internal/storage" - "github.com/yuin/goldmark" - "github.com/yuin/goldmark/extension" - goldhtml "github.com/yuin/goldmark/renderer/html" -) - -// md is the shared goldmark instance used for all markdown→HTML conversions. -var md = goldmark.New( - goldmark.WithExtensions(extension.Typographer, extension.Table), - goldmark.WithRendererOptions(goldhtml.WithUnsafe()), -) - -// kokoroVoices is the full list of voices shipped with Kokoro-FastAPI, -// grouped loosely by language prefix: -// -// af_ / am_ American English female / male -// bf_ / bm_ British English female / male -// ef_ / em_ Spanish female / male -// ff_ French female -// hf_ / hm_ Hindi female / male -// if_ / im_ Italian female / male -// jf_ / jm_ Japanese female / male -// pf_ / pm_ Portuguese female / male -// zf_ / zm_ Chinese female / male -var kokoroVoices = []string{ - // American English - "af_alloy", "af_aoede", "af_bella", "af_heart", "af_jadzia", - "af_jessica", "af_kore", "af_nicole", "af_nova", "af_river", - "af_sarah", "af_sky", - "am_adam", "am_echo", "am_eric", "am_fenrir", "am_liam", - "am_michael", "am_onyx", "am_puck", - // British English - "bf_alice", "bf_emma", "bf_lily", - "bm_daniel", "bm_fable", "bm_george", "bm_lewis", - // Spanish - "ef_dora", "em_alex", - // French - "ff_siwis", - // Hindi - "hf_alpha", "hf_beta", "hm_omega", "hm_psi", - // Italian - "if_sara", "im_nicola", - // Japanese - "jf_alpha", "jf_gongitsune", "jf_nezumi", "jf_tebukuro", "jm_kumo", - // Portuguese - "pf_dora", "pm_alex", - // Chinese - "zf_xiaobei", "zf_xiaoni", "zf_xiaoxiao", "zf_xiaoyi", - "zm_yunjian", "zm_yunxi", "zm_yunxia", "zm_yunyang", -} - -// voiceInfo holds the parsed display metadata for a single Kokoro voice. -type voiceInfo struct { - ID string // raw voice ID, e.g. "af_bella" - Name string // display name, e.g. "Bella" - Lang string // language label, e.g. "EN-US" - Gender string // "F" or "M" -} - -// langLabel maps the two-letter prefix to a human-readable language tag. -var langLabel = map[string]string{ - "a": "EN-US", - "b": "EN-GB", - "e": "ES", - "f": "FR", - "h": "HI", - "i": "IT", - "j": "JA", - "p": "PT", - "z": "ZH", -} - -// parseVoice decodes a Kokoro voice ID into display metadata. -// IDs follow the pattern {lang}{gender}_{name} e.g. "af_bella". -func parseVoice(id string) voiceInfo { - v := voiceInfo{ID: id, Name: id, Lang: "?", Gender: "?"} - if len(id) < 3 || id[2] != '_' { - return v - } - lc := string(id[0]) - gc := string(id[1]) - name := id[3:] - if l, ok := langLabel[lc]; ok { - v.Lang = l - } - switch gc { - case "f": - v.Gender = "F" - case "m": - v.Gender = "M" - } - // Capitalise name, replace underscores with spaces. - if len(name) > 0 { - runes := []rune(name) - runes[0] -= 'a' - 'A' - v.Name = strings.ReplaceAll(string(runes), "_", " ") - } - return v -} - -// parseVoices converts a slice of raw voice IDs to voiceInfo structs. -func parseVoices(ids []string) []voiceInfo { - out := make([]voiceInfo, len(ids)) - for i, id := range ids { - out[i] = parseVoice(id) - } - return out -} - -// ─── shared layout ──────────────────────────────────────────────────────────── - -const layoutHead = ` - - - - - {{.Title}} — libnovel - - - - -` - -const layoutFoot = `` - -func renderPage(w http.ResponseWriter, title, body string) { - t := template.Must(template.New("layout").Parse(layoutHead + body + layoutFoot)) - w.Header().Set("Content-Type", "text/html; charset=utf-8") - _ = t.Execute(w, struct{ Title string }{Title: title}) -} - -func renderFragment(w http.ResponseWriter, body string) { - w.Header().Set("Content-Type", "text/html; charset=utf-8") - fmt.Fprint(w, body) -} - -func isHTMX(r *http.Request) bool { - return r.Header.Get("HX-Request") == "true" -} - -// respond writes either a full page or an HTMX fragment depending on the request. -func (s *Server) respond(w http.ResponseWriter, r *http.Request, title, fragment string) { - if isHTMX(r) { - renderFragment(w, fragment) - return - } - renderPage(w, title, - `
`+fragment+`
`) -} - -// ─── GET / — book catalogue ─────────────────────────────────────────────────── - -const homeTmpl = ` - - -
- - -
-

libnovel

-
- + Add - Browse Rankings -
-
- - - -

{{len .Books}} book{{if ne (len .Books) 1}}s{{end}} on disk

- - - - - - - - - - - -
-
- -
- -
- - - - -
- - - -
- - -
-

All books

- -
-
- -` - -// homeBookItem wraps BookMeta with the count of chapters already on disk -// and the Unix timestamp of when the book was added (metadata.yaml mtime). -type homeBookItem struct { - scraper.BookMeta - Downloaded int - AddedAt int64 -} - -func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/" { - http.NotFound(w, r) - return - } - - books, err := s.store.ListBooks(r.Context()) - if err != nil { - http.Error(w, "failed to list books: "+err.Error(), http.StatusInternalServerError) - return - } - - items := make([]homeBookItem, len(books)) - for i, b := range books { - items[i] = homeBookItem{ - BookMeta: b, - Downloaded: s.store.CountChapters(r.Context(), b.Slug), - AddedAt: s.store.MetadataMtime(r.Context(), b.Slug), - } - } - - t := template.Must(template.New("home").Parse(homeTmpl)) - var buf bytes.Buffer - _ = t.Execute(&buf, struct { - Books interface{} - }{ - Books: items, - }) - - s.respond(w, r, "Home", buf.String()) -} - -// ─── GET /scrape — add a new book ───────────────────────────────────────────── - -const scrapeTmpl = ` -
- - ← All books - - -

Add a book

-

Search the rankings or paste a novelfire.net URL to scrape a new book.

- -
-
- - - - - -
- -
-
-
- -` - -func (s *Server) handleScrape(w http.ResponseWriter, r *http.Request) { - rankingItems, _ := s.store.ReadRankingItems(r.Context()) - rankingJSON, _ := json.Marshal(rankingItems) - - t := template.Must(template.New("scrape").Parse(scrapeTmpl)) - var buf bytes.Buffer - _ = t.Execute(&buf, struct { - RankingJSON template.JS - }{ - RankingJSON: template.JS(rankingJSON), - }) - - s.respond(w, r, "Add a book", buf.String()) -} - -// ─── GET /ranking — ranking page ─────────────────────────────────────────────── - -const rankingTmpl = ` - - -
- - - - - -
- - ← All books - -
-
-

Novel Rankings

-
- {{if .TotalItems}}{{.TotalItems}} novels{{end}} - {{if .CachedAt}}· cached {{.CachedAt}}{{end}} - -
-
-
- - -
-
-
-
- - -
- -
- - - - -
- - -
- Filter: - - {{range .AllStatuses}} - - {{end}} - {{range .TopGenres}} - - {{end}} - -
-
- - -
- {{range .Books}} - {{if .Local}} - - -
- {{if .Rank}}#{{.Rank}}{{end}} -
- -
- {{if .Cover}} - cover - {{else}} -
N/A
- {{end}} -
- -
-
-

{{.Title}}

- In library -
- {{if .Author}}

{{.Author}}

{{end}} -
- {{if .Status}}{{.Status}}{{end}} - {{range .Genres}}{{.}}{{end}} -
-
- - {{else}} -
- -
- {{if .Rank}}#{{.Rank}}{{end}} -
- -
- {{if .Cover}} - cover - {{else}} -
N/A
- {{end}} -
- -
-

{{.Title}}

- {{if .Author}}

{{.Author}}

{{end}} -
- {{if .Status}}{{.Status}}{{end}} - {{range .Genres}}{{.}}{{end}} -
- {{if .SourceURL}} -
- - -
- {{end}} -
-
- {{end}} - {{else}} -
-

No ranking data cached yet.

-
- - -
-
- {{end}} -
- - - -
- -` - -// rankingViewItem enriches a RankingItem with whether it is present in the -// local book library, so the template can highlight it differently. -type rankingViewItem struct { - storage.RankingItem - Local bool -} - -// toRankingViewItems annotates items with Local=true for slugs found in localSlugs. -func toRankingViewItems(items []storage.RankingItem, localSlugs map[string]bool) []rankingViewItem { - out := make([]rankingViewItem, len(items)) - for i, it := range items { - out[i] = rankingViewItem{ - RankingItem: it, - Local: localSlugs[it.Slug], - } - } - return out -} - -// pageNum is one entry in the ranking pagination bar. -// Num == 0 is a sentinel that renders as an ellipsis gap. -type pageNum struct { - Num int -} - -// rankingPageNums builds a pagination list with smart ellipsis. -// It always shows: first 2, last 2, and a ±2 window around current. -// Gaps between non-consecutive runs are filled with a sentinel (Num==0) for "…". -// Pass current=0 when there is no concept of a current page (e.g. fetch bar). -func rankingPageNums(total, current int) []pageNum { - if total <= 0 { - return nil - } - show := make(map[int]bool) - // First 2 and last 2. - for i := 1; i <= 2 && i <= total; i++ { - show[i] = true - } - for i := total - 1; i <= total; i++ { - if i >= 1 { - show[i] = true - } - } - // ±2 window around current page. - if current > 0 { - for i := current - 2; i <= current+2; i++ { - if i >= 1 && i <= total { - show[i] = true - } - } - } - - // Collect and sort. - pages := make([]int, 0, len(show)) - for p := range show { - pages = append(pages, p) - } - for i := 0; i < len(pages); i++ { - for j := i + 1; j < len(pages); j++ { - if pages[j] < pages[i] { - pages[i], pages[j] = pages[j], pages[i] - } - } - } - - // Build output with ellipsis sentinels between non-consecutive pages. - out := make([]pageNum, 0, len(pages)*2) - for i, p := range pages { - if i > 0 && p > pages[i-1]+1 { - out = append(out, pageNum{0}) - } - out = append(out, pageNum{p}) - } - return out -} - -const rankingPageSize = 20 - -// handleRanking serves the ranking page from the cached ranking.json file. -// It does NOT trigger a live scrape; use POST /ranking/refresh for that. -// Supports ?page=N for browsing through cached items (20 per page). -func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) { - rankingItems, err := s.store.ReadRankingItems(r.Context()) - if err != nil { - s.log.Error("failed to read cached ranking", "err", err) - } - - cachedAt := "" - if info, statErr := s.store.RankingFileInfo(r.Context()); statErr == nil && info != nil { - cachedAt = info.ModTime().Format("Jan 2, 2006 at 15:04") - } - - // Parse requested display page (1-indexed). - currentPage := 1 - if p := r.URL.Query().Get("page"); p != "" { - if n, err2 := strconv.Atoi(p); err2 == nil && n > 0 { - currentPage = n - } - } - - totalItems := len(rankingItems) - totalPages := 1 - if totalItems > 0 { - totalPages = (totalItems + rankingPageSize - 1) / rankingPageSize - } - if currentPage > totalPages { - currentPage = totalPages - } - - // Slice items for the current display page. - start := (currentPage - 1) * rankingPageSize - end := start + rankingPageSize - if end > totalItems { - end = totalItems - } - pageItems := rankingItems - if totalItems > 0 { - pageItems = rankingItems[start:end] - } - - t := template.Must(template.New("ranking").Parse(rankingTmpl)) - var buf bytes.Buffer - - // Collect distinct genres and statuses across ALL items for facet filters. - genreFreq := map[string]int{} - statusSet := map[string]bool{} - for _, it := range rankingItems { - if it.Status != "" { - statusSet[it.Status] = true - } - for _, g := range it.Genres { - genreFreq[g]++ - } - } - allStatuses := sortedKeys(statusSet) - - // Build top-10 genres sorted by frequency descending. - type genreCount struct { - name string - count int - } - gcSlice := make([]genreCount, 0, len(genreFreq)) - for g, c := range genreFreq { - gcSlice = append(gcSlice, genreCount{g, c}) - } - sort.Slice(gcSlice, func(i, j int) bool { - if gcSlice[i].count != gcSlice[j].count { - return gcSlice[i].count > gcSlice[j].count - } - return gcSlice[i].name < gcSlice[j].name - }) - topN := 10 - if len(gcSlice) < topN { - topN = len(gcSlice) - } - topGenres := make([]string, topN) - for i := 0; i < topN; i++ { - topGenres[i] = gcSlice[i].name - } - - // Encode full dataset + local slugs for client-side cross-page filtering. - localSlugs, _ := s.store.LocalSlugs(r.Context()) - type rankingJSONItem struct { - Rank int `json:"rank"` - Slug string `json:"slug"` - Title string `json:"title"` - Author string `json:"author,omitempty"` - Cover string `json:"cover,omitempty"` - Status string `json:"status,omitempty"` - Genres []string `json:"genres,omitempty"` - SourceURL string `json:"source_url,omitempty"` - Local bool `json:"local"` - } - allItemsForJS := make([]rankingJSONItem, len(rankingItems)) - for i, it := range rankingItems { - allItemsForJS[i] = rankingJSONItem{ - Rank: it.Rank, - Slug: it.Slug, - Title: it.Title, - Author: it.Author, - Cover: it.Cover, - Status: it.Status, - Genres: it.Genres, - SourceURL: it.SourceURL, - Local: localSlugs[it.Slug], - } - } - allItemsJSON, _ := json.Marshal(allItemsForJS) - - _ = t.Execute(&buf, struct { - Books interface{} - CachedAt string - FetchNums []pageNum - DisplayNums []pageNum - CurrentPage int - TotalPages int - TotalItems int - TopGenres []string - AllStatuses []string - AllItemsJSON template.JS - }{ - Books: toRankingViewItems(pageItems, localSlugs), - CachedAt: cachedAt, - FetchNums: rankingPageNums(100, 0), - DisplayNums: rankingPageNums(totalPages, currentPage), - CurrentPage: currentPage, - TotalPages: totalPages, - TotalItems: totalItems, - TopGenres: topGenres, - AllStatuses: allStatuses, - AllItemsJSON: template.JS(allItemsJSON), - }) - s.respond(w, r, "Rankings", buf.String()) -} - -// handleRankingRefresh starts an async scrape of novelfire.net/ranking and -// immediately returns a polling badge. The browser polls /ui/ranking/status -// until the job finishes, then follows an HX-Redirect back to /ranking. -// -// Accepts an optional form field "pages" (integer ≥ 1). 0 or absent means -// fetch all pages; otherwise at most that many pages are scraped. -func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) { - _ = r.ParseForm() - maxPages := 0 - if p := strings.TrimSpace(r.FormValue("pages")); p != "" { - if n, err := strconv.Atoi(p); err == nil && n > 0 { - maxPages = n - } - } - - s.mu.Lock() - if s.rankingRunning { - s.mu.Unlock() - renderFragment(w, rankingStatusHTML("running", "Ranking refresh already in progress…")) - return - } - s.rankingRunning = true - s.mu.Unlock() - - go func() { - defer func() { - s.mu.Lock() - s.rankingRunning = false - s.mu.Unlock() - }() - - // Allow ~90 s per page; minimum 120 s for a single page. - timeout := 120 * time.Second - if maxPages > 1 { - timeout = time.Duration(maxPages) * 90 * time.Second - } - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - rankingCh, errCh := s.novel.ScrapeRanking(ctx, maxPages) - - var rankingItems []storage.RankingItem - for rankingCh != nil || errCh != nil { - select { - case meta, ok := <-rankingCh: - if !ok { - rankingCh = nil - } else { - rankingItems = append(rankingItems, storage.RankingItem{ - Rank: meta.Ranking, - Slug: meta.Slug, - Title: meta.Title, - Author: meta.Author, - Cover: meta.Cover, - Status: meta.Status, - Genres: meta.Genres, - SourceURL: meta.SourceURL, - }) - } - case err, ok := <-errCh: - if !ok { - errCh = nil - } else if err != nil { - s.log.Error("ranking scrape error", "err", err) - } - } - } - - if len(rankingItems) > 0 { - if err := s.store.WriteRanking(ctx, rankingItems); err != nil { - s.log.Error("failed to save ranking", "err", err) - } - } - }() - - renderFragment(w, rankingStatusHTML("running", "Fetching rankings…")) -} - -// handleRankingStatus is the HTMX polling endpoint for ranking refresh jobs. -// While running it returns a self-replacing badge; when done it issues an -// HX-Redirect so the browser navigates to /ranking. -func (s *Server) handleRankingStatus(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - running := s.rankingRunning - s.mu.Unlock() - - if running { - renderFragment(w, rankingStatusHTML("running", "Fetching rankings…")) - return - } - // Job done — redirect the HTMX request to the ranking page. - w.Header().Set("HX-Redirect", "/ranking") - w.WriteHeader(http.StatusOK) -} - -// rankingStatusHTML returns a self-replacing polling badge for the ranking -// refresh job. state is "running" or "done". -func rankingStatusHTML(state, msg string) string { - var colour, dot, poll string - switch state { - case "running": - colour = "text-amber-300 bg-amber-950 border-amber-800" - dot = `` - poll = `hx-get="/ui/ranking/status" hx-trigger="every 3s" hx-target="this" hx-swap="outerHTML"` - default: - colour = "text-green-300 bg-green-950 border-green-800" - dot = `` - } - return fmt.Sprintf( - `
%s%s
`, - colour, poll, dot, template.HTMLEscapeString(msg), - ) -} - -// ─── GET /ranking/view — view ranking markdown ───────────────────────────────── - -const rankingViewTmpl = ` -
- - ← Back to Rankings - - -

Ranking Data

- - 
{{.JSON}}
-
` - -func (s *Server) handleRankingView(w http.ResponseWriter, r *http.Request) { - items, err := s.store.ReadRankingItems(r.Context()) - if err != nil { - http.Error(w, "failed to read ranking: "+err.Error(), http.StatusInternalServerError) - return - } - if len(items) == 0 { - http.NotFound(w, r) - return - } - - pretty, err := json.MarshalIndent(items, "", " ") - if err != nil { - http.Error(w, "json marshal error: "+err.Error(), http.StatusInternalServerError) - return - } - - t := template.Must(template.New("rankingView").Parse(rankingViewTmpl)) - var buf bytes.Buffer - _ = t.Execute(&buf, struct{ JSON string }{JSON: string(pretty)}) - - s.respond(w, r, "Ranking Data", buf.String()) -} - -// ─── GET /books/{slug} — chapter list ──────────────────────────────────────── - -const chapterPageSize = 50 - -const bookTmpl = ` -
- - ← All books - - - - - - - - -
- {{if .Meta.Cover}} - cover - {{end}} -
-
-

{{.Meta.Title}}

- {{if .Meta.SourceURL}} - - - - - Source - -
- - -
- {{end}} -
- {{if .Meta.Author}}

{{.Meta.Author}}

{{end}} -
- {{if .Meta.Status}}{{.Meta.Status}}{{end}} - {{if .Meta.TotalChapters}}{{.Meta.TotalChapters}} ch total{{end}} - {{.TotalDownloaded}} downloaded -
- {{if .Meta.Summary}} -

{{.Meta.Summary}}

- {{end}} -
-
- - - - - - {{if .LastChapter}} - - {{end}} - -

Chapters

- - - {{if gt .TotalPages 1}} -
- {{if gt .CurrentPage 1}} - « - {{end}} - {{range pages .TotalPages}} - {{if eq . $.CurrentPage}} - {{.}} - {{else}} - {{.}} - {{end}} - {{end}} - {{if lt .CurrentPage .TotalPages}} - » - {{end}} -
- {{end}} -
- -` - -func (s *Server) handleBook(w http.ResponseWriter, r *http.Request) { - slug := r.PathValue("slug") - - meta, ok, err := s.store.ReadMetadata(r.Context(), slug) - if err != nil { - http.Error(w, "failed to read metadata: "+err.Error(), http.StatusInternalServerError) - return - } - if !ok { - http.NotFound(w, r) - return - } - - chapters, err := s.store.ListChapters(r.Context(), slug) - if err != nil { - http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError) - return - } - - total := len(chapters) - totalPages := (total + chapterPageSize - 1) / chapterPageSize - if totalPages < 1 { - totalPages = 1 - } - - currentPage := 1 - page := chapters - if total > chapterPageSize { - page = chapters[:chapterPageSize] - } - - funcMap := template.FuncMap{ - "pages": func(n int) []int { - out := make([]int, n) - for i := range out { - out[i] = i + 1 - } - return out - }, - "prev": func(n int) int { return n - 1 }, - "next": func(n int) int { return n + 1 }, - } - - t := template.Must(template.New("book").Funcs(funcMap).Parse(bookTmpl)) - var buf bytes.Buffer - lastChapter := 0 - if total > 0 { - lastChapter = chapters[total-1].Number - } - _ = t.Execute(&buf, struct { - Slug string - Meta interface{} - Chapters interface{} - TotalDownloaded int - TotalPages int - CurrentPage int - LastChapter int - }{ - Slug: slug, - Meta: meta, - Chapters: page, - TotalDownloaded: total, - TotalPages: totalPages, - CurrentPage: currentPage, - LastChapter: lastChapter, - }) - - s.respond(w, r, meta.Title, buf.String()) -} - -// ─── GET /books/{slug}/chapters-page — paginated chapter list fragment ──────── - -const chapterPageTmpl = `{{range .Chapters}} -
  • - - {{.Number}} -
    - {{.Title}} - {{if .Date}}{{.Date}}{{end}} -
    - -     -
    • -{{end}} -
    - {{if gt .CurrentPage 1}} - « - {{end}} - {{range pages .TotalPages}} - {{if eq . $.CurrentPage}} - {{.}} - {{else}} - {{.}} - {{end}} - {{end}} - {{if lt .CurrentPage .TotalPages}} - » - {{end}} -
    ` - -func (s *Server) handleBookChaptersPage(w http.ResponseWriter, r *http.Request) { - slug := r.PathValue("slug") - currentPage := 1 - if p := r.URL.Query().Get("page"); p != "" { - if n, err := strconv.Atoi(p); err == nil && n > 0 { - currentPage = n - } - } - - chapters, err := s.store.ListChapters(r.Context(), slug) - if err != nil { - http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError) - return - } - - total := len(chapters) - totalPages := (total + chapterPageSize - 1) / chapterPageSize - if totalPages < 1 { - totalPages = 1 - } - - start := (currentPage - 1) * chapterPageSize - if start >= total { - w.WriteHeader(http.StatusNoContent) - return - } - end := start + chapterPageSize - if end > total { - end = total - } - - funcMap := template.FuncMap{ - "pages": func(n int) []int { - out := make([]int, n) - for i := range out { - out[i] = i + 1 - } - return out - }, - "prev": func(n int) int { return n - 1 }, - "next": func(n int) int { return n + 1 }, - } - - t := template.Must(template.New("chapterPage").Funcs(funcMap).Parse(chapterPageTmpl)) - var buf bytes.Buffer - _ = t.Execute(&buf, struct { - Slug string - Chapters interface{} - TotalPages int - CurrentPage int - }{ - Slug: slug, - Chapters: chapters[start:end], - TotalPages: totalPages, - CurrentPage: currentPage, - }) - - w.Header().Set("Content-Type", "text/html; charset=utf-8") - _, _ = buf.WriteTo(w) -} - -// ─── GET /books/{slug}/chapters/{n} — chapter reader ───────────────────────── - -const chapterTmpl = ` - - - - - - -
    - - -
    -

    Chapter {{.ChapterN}}

    -

    {{.Title}}

    - {{if .ChapterDate}}

    {{.ChapterDate}}

    {{end}} -
    - -
    - {{.HTML}} -
    - -
    - - - - -` - -func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) { - slug := r.PathValue("slug") - n, err := strconv.Atoi(r.PathValue("n")) - if err != nil || n < 1 { - http.NotFound(w, r) - return - } - - raw, err := s.store.ReadChapter(r.Context(), slug, n) - if err != nil { - http.NotFound(w, r) - return - } - - // Strip the first heading line so it isn't rendered as a duplicate

    - // inside the article (the template already renders an explicit

    ). - rawForHTML := stripFirstHeadingLine(raw) - - var htmlBuf bytes.Buffer - if err := md.Convert([]byte(rawForHTML), &htmlBuf); err != nil { - http.Error(w, "markdown render error: "+err.Error(), http.StatusInternalServerError) - return - } - - chapters, _ := s.store.ListChapters(r.Context(), slug) - prevN, nextN := adjacentChapters(chapters, n) - - title := firstHeading(raw, fmt.Sprintf("Chapter %d", n)) - chapterTitle, chapterDate := splitChapterTitle(title) - - // Load cover URL for Media Session artwork (best-effort; ignore errors). - var coverURL string - if meta, ok, err := s.store.ReadMetadata(r.Context(), slug); err == nil && ok { - coverURL = meta.Cover - } - - t := template.Must(template.New("chapter").Parse(chapterTmpl)) - var buf bytes.Buffer - _ = t.Execute(&buf, struct { - Slug string - HTML template.HTML - PrevN int - NextN int - ChapterN int - Title string - ChapterDate string - AllChapters interface{} - Voices []voiceInfo - DefaultVoice string - Cover string - }{ - Slug: slug, - HTML: template.HTML(htmlBuf.String()), - PrevN: prevN, - NextN: nextN, - ChapterN: n, - Title: chapterTitle, - ChapterDate: chapterDate, - AllChapters: chapters, - Voices: parseVoices(s.voices()), - DefaultVoice: s.kokoroVoice, - Cover: coverURL, - }) - - s.respond(w, r, chapterTitle, buf.String()) -} - -// ─── helpers ────────────────────────────────────────────────────────────────── - -// splitChapterTitle splits a raw chapter heading into a human-readable title -// and a trailing relative-date string (e.g. "1 year ago"). It mirrors the -// same logic in internal/writer and internal/storage/hybrid. -func splitChapterTitle(raw string) (title, date string) { - raw = strings.TrimSpace(raw) - // Strip leading numeric index. - if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 { - prefix := raw[:idx] - allDigit := true - for _, c := range prefix { - if c < '0' || c > '9' { - allDigit = false - break - } - } - if allDigit { - raw = strings.TrimSpace(raw[idx:]) - } - } - // Strip "Chapter N - N: " prefix. - chNumRe := regexp.MustCompile(`(?i)^chapter\s+\d+(?:\s*-\s*\d+)?\s*:\s*`) - raw = strings.TrimSpace(chNumRe.ReplaceAllString(raw, "")) - // Detect trailing relative date. - dateRe := regexp.MustCompile(`\s*(\d+\s+(?:second|minute|hour|day|week|month|year)s?\s+ago)\s*$`) - if m := dateRe.FindStringSubmatchIndex(raw); m != nil { - return strings.TrimSpace(raw[:m[0]]), strings.TrimSpace(raw[m[2]:m[3]]) - } - return raw, "" -} - -// sortedKeys returns the keys of a string-bool map in sorted order. -func sortedKeys(m map[string]bool) []string { - out := make([]string, 0, len(m)) - for k := range m { - out = append(out, k) - } - // Simple insertion sort — sets are small (< 100 items). - for i := 1; i < len(out); i++ { - for j := i; j > 0 && out[j] < out[j-1]; j-- { - out[j], out[j-1] = out[j-1], out[j] - } - } - return out -} - -// stripMarkdown removes Markdown syntax and returns clean plain text. -func stripMarkdown(src string) string { - src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "") - src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "") - src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "") - src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "") - src = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(src, "$1") - src = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`).ReplaceAllString(src, "") - src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "") - src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "") - src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n") - return strings.TrimSpace(src) -} - -// adjacentChapters returns the chapter numbers immediately before and after n -// in the sorted chapters list. 0 means "does not exist". -func adjacentChapters(chapters []storage.ChapterInfo, n int) (prev, next int) { - for i, ch := range chapters { - if ch.Number == n { - if i > 0 { - prev = chapters[i-1].Number - } - if i < len(chapters)-1 { - next = chapters[i+1].Number - } - return - } - } - return -} - -// stripFirstHeadingLine removes the first non-empty line if it is a markdown -// heading (starts with one or more "#"). This prevents the heading from being -// rendered as a duplicate

    inside the article when the template already -// renders an explicit title above the article. -func stripFirstHeadingLine(src string) string { - lines := strings.SplitN(src, "\n", -1) - for i, line := range lines { - trimmed := strings.TrimSpace(line) - if trimmed == "" { - continue - } - if strings.HasPrefix(trimmed, "#") { - // Remove this line and return the rest. - rest := strings.Join(append(lines[:i], lines[i+1:]...), "\n") - return strings.TrimLeft(rest, "\n") - } - // First non-empty line is not a heading — nothing to strip. - break - } - return src -} - -// firstHeading returns the text of the first non-empty line, stripping a -// leading "# " markdown heading marker. Falls back to fallback. -func firstHeading(md, fallback string) string { - for _, line := range strings.SplitN(md, "\n", 20) { - line = strings.TrimSpace(line) - if line == "" { - continue - } - return strings.TrimPrefix(line, "# ") - } - return fallback -} - -// ─── POST /ui/scrape/book — form submission ─────────────────────────────────── - -func (s *Server) handleUIScrapeBook(w http.ResponseWriter, r *http.Request) { - bookURL := strings.TrimSpace(r.FormValue("url")) - if bookURL == "" { - renderFragment(w, scrapeStatusHTML("error", "Please enter a book URL.")) - return - } - - s.mu.Lock() - already := s.running - if !already { - s.running = true - } - s.mu.Unlock() - - if already { - renderFragment(w, scrapeStatusHTML("busy", "A scrape job is already running. Please wait.")) - return - } - - cfg := s.oCfg - cfg.SingleBookURL = bookURL - - go func() { - defer func() { - s.mu.Lock() - s.running = false - s.mu.Unlock() - }() - - ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) - defer cancel() - - o := orchestrator.New(cfg, s.novel, s.log, s.store) - if err := o.Run(ctx); err != nil { - s.log.Error("UI scrape job failed", "url", bookURL, "err", err) - } - }() - - // Return a status badge that polls until the job finishes. - renderFragment(w, scrapeStatusHTML("running", "Scraping "+bookURL+"…")) -} - -// ─── GET /ui/scrape/status — polling endpoint ───────────────────────────────── - -func (s *Server) handleUIScrapeStatus(w http.ResponseWriter, r *http.Request) { - s.mu.Lock() - running := s.running - s.mu.Unlock() - - if running { - // Keep polling every 3 s while the job is in progress. - renderFragment(w, scrapeStatusHTML("running", "Scraping in progress…")) - return - } - // Job finished — show a done badge and stop polling. - renderFragment(w, scrapeStatusHTML("done", "Done! Refresh the page to see new books.")) -} - -// scrapeStatusHTML returns a self-contained status badge fragment. -// state is one of: "running" | "done" | "busy" | "error". -func scrapeStatusHTML(state, msg string) string { - var colour, dot, poll string - switch state { - case "running": - colour = "text-amber-300 bg-amber-950 border-amber-800" - dot = `` - poll = `hx-get="/ui/scrape/status" hx-trigger="every 3s" hx-target="this" hx-swap="outerHTML"` - case "done": - colour = "text-green-300 bg-green-950 border-green-800" - dot = `` - case "busy": - colour = "text-yellow-300 bg-yellow-950 border-yellow-800" - dot = `` - default: // error - colour = "text-red-300 bg-red-950 border-red-800" - dot = `` - } - return fmt.Sprintf( - `
    %s%s
    `, - colour, poll, dot, template.HTMLEscapeString(msg), - ) -}