package server import ( "bytes" "context" "encoding/json" "fmt" "html/template" "net/http" "regexp" "strconv" "strings" "time" "github.com/libnovel/scraper/internal/orchestrator" "github.com/libnovel/scraper/internal/writer" "github.com/yuin/goldmark" "github.com/yuin/goldmark/extension" goldhtml "github.com/yuin/goldmark/renderer/html" ) // md is the shared goldmark instance used for all markdown→HTML conversions. var md = goldmark.New( goldmark.WithExtensions(extension.Typographer, extension.Table), goldmark.WithRendererOptions(goldhtml.WithUnsafe()), ) // kokoroVoices is the full list of voices shipped with Kokoro-FastAPI, // grouped loosely by language prefix: // // af_ / am_ American English female / male // bf_ / bm_ British English female / male // ef_ / em_ Spanish female / male // ff_ French female // hf_ / hm_ Hindi female / male // if_ / im_ Italian female / male // jf_ / jm_ Japanese female / male // pf_ / pm_ Portuguese female / male // zf_ / zm_ Chinese female / male var kokoroVoices = []string{ // American English "af_alloy", "af_aoede", "af_bella", "af_heart", "af_jadzia", "af_jessica", "af_kore", "af_nicole", "af_nova", "af_river", "af_sarah", "af_sky", "am_adam", "am_echo", "am_eric", "am_fenrir", "am_liam", "am_michael", "am_onyx", "am_puck", // British English "bf_alice", "bf_emma", "bf_lily", "bm_daniel", "bm_fable", "bm_george", "bm_lewis", // Spanish "ef_dora", "em_alex", // French "ff_siwis", // Hindi "hf_alpha", "hf_beta", "hm_omega", "hm_psi", // Italian "if_sara", "im_nicola", // Japanese "jf_alpha", "jf_gongitsune", "jf_nezumi", "jf_tebukuro", "jm_kumo", // Portuguese "pf_dora", "pm_alex", // Chinese "zf_xiaobei", "zf_xiaoni", "zf_xiaoxiao", "zf_xiaoyi", "zm_yunjian", "zm_yunxi", "zm_yunxia", "zm_yunyang", } // ─── shared layout ──────────────────────────────────────────────────────────── const layoutHead = ` {{.Title}} — libnovel ` const layoutFoot = `` func renderPage(w http.ResponseWriter, title, body string) { t := template.Must(template.New("layout").Parse(layoutHead + body + layoutFoot)) w.Header().Set("Content-Type", "text/html; charset=utf-8") _ = t.Execute(w, struct{ Title string }{Title: title}) } func renderFragment(w http.ResponseWriter, body string) { w.Header().Set("Content-Type", "text/html; charset=utf-8") fmt.Fprint(w, body) } func isHTMX(r *http.Request) bool { return r.Header.Get("HX-Request") == "true" } // respond writes either a full page or an HTMX fragment depending on the request. func (s *Server) respond(w http.ResponseWriter, r *http.Request, title, fragment string) { if isHTMX(r) { renderFragment(w, fragment) return } renderPage(w, title, `

`+fragment+`

`) } // ─── GET / — book catalogue ─────────────────────────────────────────────────── const homeTmpl = `

libnovel

Browse Rankings

{{len .Books}} book{{if ne (len .Books) 1}}s{{end}} on disk

Scrape a new book

Available

{{if .Status}}{{.Status}}{{end}} {{if .TotalChapters}}{{.TotalChapters}} ch{{end}}

No books scraped yet.

` func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/" { http.NotFound(w, r) return } books, err := s.writer.ListBooks() if err != nil { http.Error(w, "failed to list books: "+err.Error(), http.StatusInternalServerError) return } // Load ranking items for the scrape-search autocomplete. // Failures are non-fatal — the form degrades to a plain URL input. rankingItems, _ := s.writer.ReadRankingItems() // Encode ranking items as JSON for embedding in the template. rankingJSON, _ := json.Marshal(rankingItems) t := template.Must(template.New("home").Parse(homeTmpl)) var buf bytes.Buffer _ = t.Execute(&buf, struct { Books interface{} RankingJSON template.JS }{ Books: books, RankingJSON: template.JS(rankingJSON), }) s.respond(w, r, "Home", buf.String()) } // ─── GET /ranking — ranking page ─────────────────────────────────────────────── const rankingTmpl = `

← All books

Novel Rankings

Top novels from novelfire.net

Cached {{.CachedAt}}

View Markdown

Fetch up to page: {{range .PageNums}} {{if eq .Num 0}} … {{else}} {{end}} {{end}}

Each page = ~20 novels from novelfire.net popular

{{range .Books}} {{if .Local}} {{if .Cover}} cover

{{if .Rank}}#{{.Rank}}{{end}}

In library

Source

{{if .Cover}} cover

{{if .Rank}}#{{.Rank}}{{end}}

Source

No ranking data. Use the page buttons above to fetch from novelfire.net.

` // rankingViewItem enriches a RankingItem with whether it is present in the // local book library, so the template can highlight it differently. type rankingViewItem struct { writer.RankingItem Local bool } // toRankingViewItems annotates items with Local=true for slugs found in localSlugs. func toRankingViewItems(items []writer.RankingItem, localSlugs map[string]bool) []rankingViewItem { out := make([]rankingViewItem, len(items)) for i, it := range items { out[i] = rankingViewItem{ RankingItem: it, Local: localSlugs[it.Slug], } } return out } // pageNum is one entry in the ranking pagination bar. // Num == 0 is a sentinel that renders as an ellipsis gap. type pageNum struct { Num int } // rankingPageNums builds the dynamic pagination list for 100 pages with // smart ellipsis: always show the first 3, last 3, and a sliding window of // 3 around the ends — compressed with "…" separators between runs. // // For a flat 100-page list it produces: // // 1 2 3 … 6 7 8 … 93 94 95 … 98 99 100 // // (The middle range is omitted intentionally; users click individual pages.) func rankingPageNums(total int) []pageNum { if total <= 0 { return nil } // Build the set of visible page numbers using a sliding-window approach. show := make(map[int]bool) // Always show first 3 and last 3. for i := 1; i <= 3 && i <= total; i++ { show[i] = true } for i := total - 2; i <= total; i++ { if i >= 1 { show[i] = true } } // For large ranges, show a few pages near the 1/4 and 3/4 marks. if total > 12 { q1 := total / 4 q3 := total * 3 / 4 for _, p := range []int{q1 - 1, q1, q1 + 1, q3 - 1, q3, q3 + 1} { if p >= 1 && p <= total { show[p] = true } } } // Collect sorted visible pages. pages := make([]int, 0, len(show)) for p := range show { pages = append(pages, p) } // Sort pages slice. for i := 0; i < len(pages); i++ { for j := i + 1; j < len(pages); j++ { if pages[j] < pages[i] { pages[i], pages[j] = pages[j], pages[i] } } } // Build output with ellipsis sentinels (Num==0) between non-consecutive pages. out := make([]pageNum, 0, len(pages)*2) for i, p := range pages { if i > 0 && p > pages[i-1]+1 { out = append(out, pageNum{0}) // ellipsis } out = append(out, pageNum{p}) } return out } // handleRanking serves the ranking page from the cached ranking.md file. // It does NOT trigger a live scrape; use POST /ranking/refresh for that. func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) { rankingItems, err := s.writer.ReadRankingItems() if err != nil { s.log.Error("failed to read cached ranking", "err", err) } cachedAt := "" if info, statErr := s.writer.RankingFileInfo(); statErr == nil { cachedAt = info.ModTime().Format("Jan 2, 2006 at 15:04") } t := template.Must(template.New("ranking").Parse(rankingTmpl)) var buf bytes.Buffer _ = t.Execute(&buf, struct { Books interface{} CachedAt string PageNums []pageNum }{ Books: toRankingViewItems(rankingItems, s.writer.LocalSlugs()), CachedAt: cachedAt, PageNums: rankingPageNums(100), }) s.respond(w, r, "Rankings", buf.String()) } // handleRankingRefresh starts an async scrape of novelfire.net/ranking and // immediately returns a polling badge. The browser polls /ui/ranking/status // until the job finishes, then follows an HX-Redirect back to /ranking. // // Accepts an optional form field "pages" (integer ≥ 1). 0 or absent means // fetch all pages; otherwise at most that many pages are scraped. func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) { _ = r.ParseForm() maxPages := 0 if p := strings.TrimSpace(r.FormValue("pages")); p != "" { if n, err := strconv.Atoi(p); err == nil && n > 0 { maxPages = n } } s.mu.Lock() if s.rankingRunning { s.mu.Unlock() renderFragment(w, rankingStatusHTML("running", "Ranking refresh already in progress…")) return } s.rankingRunning = true s.mu.Unlock() go func() { defer func() { s.mu.Lock() s.rankingRunning = false s.mu.Unlock() }() // Allow ~90 s per page; minimum 120 s for a single page. timeout := 120 * time.Second if maxPages > 1 { timeout = time.Duration(maxPages) * 90 * time.Second } ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() rankingCh, errCh := s.novel.ScrapeRanking(ctx, maxPages) var rankingItems []writer.RankingItem for rankingCh != nil || errCh != nil { select { case meta, ok := <-rankingCh: if !ok { rankingCh = nil } else { rankingItems = append(rankingItems, writer.RankingItem{ Rank: meta.Ranking, Slug: meta.Slug, Title: meta.Title, Author: meta.Author, Cover: meta.Cover, Status: meta.Status, Genres: meta.Genres, SourceURL: meta.SourceURL, }) } case err, ok := <-errCh: if !ok { errCh = nil } else if err != nil { s.log.Error("ranking scrape error", "err", err) } } } if len(rankingItems) > 0 { if err := s.writer.WriteRanking(rankingItems); err != nil { s.log.Error("failed to save ranking", "err", err) } } }() renderFragment(w, rankingStatusHTML("running", "Fetching rankings…")) } // handleRankingStatus is the HTMX polling endpoint for ranking refresh jobs. // While running it returns a self-replacing badge; when done it issues an // HX-Redirect so the browser navigates to /ranking. func (s *Server) handleRankingStatus(w http.ResponseWriter, r *http.Request) { s.mu.Lock() running := s.rankingRunning s.mu.Unlock() if running { renderFragment(w, rankingStatusHTML("running", "Fetching rankings…")) return } // Job done — redirect the HTMX request to the ranking page. w.Header().Set("HX-Redirect", "/ranking") w.WriteHeader(http.StatusOK) } // rankingStatusHTML returns a self-replacing polling badge for the ranking // refresh job. state is "running" or "done". func rankingStatusHTML(state, msg string) string { var colour, dot, poll string switch state { case "running": colour = "text-amber-300 bg-amber-950 border-amber-800" dot = `` poll = `hx-get="/ui/ranking/status" hx-trigger="every 3s" hx-target="this" hx-swap="outerHTML"` default: colour = "text-green-300 bg-green-950 border-green-800" dot = `` } return fmt.Sprintf( `

%s%s

`, colour, poll, dot, template.HTMLEscapeString(msg), ) } // ─── GET /ranking/view — view ranking markdown ───────────────────────────────── const rankingViewTmpl = `

← Back to Rankings

Ranking Data

{{.JSON}}

` func (s *Server) handleRankingView(w http.ResponseWriter, r *http.Request) { items, err := s.writer.ReadRankingItems() if err != nil { http.Error(w, "failed to read ranking: "+err.Error(), http.StatusInternalServerError) return } if len(items) == 0 { http.NotFound(w, r) return } pretty, err := json.MarshalIndent(items, "", " ") if err != nil { http.Error(w, "json marshal error: "+err.Error(), http.StatusInternalServerError) return } t := template.Must(template.New("rankingView").Parse(rankingViewTmpl)) var buf bytes.Buffer _ = t.Execute(&buf, struct{ JSON string }{JSON: string(pretty)}) s.respond(w, r, "Ranking Data", buf.String()) } // ─── GET /books/{slug} — chapter list ──────────────────────────────────────── const chapterPageSize = 50 const bookTmpl = `

← All books

{{if .Meta.Cover}} cover

{{if .Meta.SourceURL}} Source {{end}}

{{if .Meta.Status}}{{.Meta.Status}}{{end}} {{if .Meta.TotalChapters}}{{.Meta.TotalChapters}} ch total{{end}} {{.TotalDownloaded}} downloaded

Chapters

{{.Number}}
{{.Title}} {{if .Date}}{{.Date}}{{end}}
No chapters downloaded yet.

{{if gt .CurrentPage 1}} « {{end}} {{range pages .TotalPages}} {{if eq . $.CurrentPage}} {{.}} {{else}} {{.}} {{end}} {{end}} {{if lt .CurrentPage .TotalPages}} » {{end}}

` func (s *Server) handleBook(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") meta, ok, err := s.writer.ReadMetadata(slug) if err != nil { http.Error(w, "failed to read metadata: "+err.Error(), http.StatusInternalServerError) return } if !ok { http.NotFound(w, r) return } chapters, err := s.writer.ListChapters(slug) if err != nil { http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError) return } total := len(chapters) totalPages := (total + chapterPageSize - 1) / chapterPageSize if totalPages < 1 { totalPages = 1 } currentPage := 1 page := chapters if total > chapterPageSize { page = chapters[:chapterPageSize] } funcMap := template.FuncMap{ "pages": func(n int) []int { out := make([]int, n) for i := range out { out[i] = i + 1 } return out }, "prev": func(n int) int { return n - 1 }, "next": func(n int) int { return n + 1 }, } t := template.Must(template.New("book").Funcs(funcMap).Parse(bookTmpl)) var buf bytes.Buffer _ = t.Execute(&buf, struct { Slug string Meta interface{} Chapters interface{} TotalDownloaded int TotalPages int CurrentPage int }{ Slug: slug, Meta: meta, Chapters: page, TotalDownloaded: total, TotalPages: totalPages, CurrentPage: currentPage, }) s.respond(w, r, meta.Title, buf.String()) } // ─── GET /books/{slug}/chapters-page — paginated chapter list fragment ──────── const chapterPageTmpl = `{{range .Chapters}}

{{if gt .CurrentPage 1}} « {{end}} {{range pages .TotalPages}} {{if eq . $.CurrentPage}} {{.}} {{else}} {{.}} {{end}} {{end}} {{if lt .CurrentPage .TotalPages}} » {{end}}

` func (s *Server) handleBookChaptersPage(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") currentPage := 1 if p := r.URL.Query().Get("page"); p != "" { if n, err := strconv.Atoi(p); err == nil && n > 0 { currentPage = n } } chapters, err := s.writer.ListChapters(slug) if err != nil { http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError) return } total := len(chapters) totalPages := (total + chapterPageSize - 1) / chapterPageSize if totalPages < 1 { totalPages = 1 } start := (currentPage - 1) * chapterPageSize if start >= total { w.WriteHeader(http.StatusNoContent) return } end := start + chapterPageSize if end > total { end = total } funcMap := template.FuncMap{ "pages": func(n int) []int { out := make([]int, n) for i := range out { out[i] = i + 1 } return out }, "prev": func(n int) int { return n - 1 }, "next": func(n int) int { return n + 1 }, } t := template.Must(template.New("chapterPage").Funcs(funcMap).Parse(chapterPageTmpl)) var buf bytes.Buffer _ = t.Execute(&buf, struct { Slug string Chapters interface{} TotalPages int CurrentPage int }{ Slug: slug, Chapters: chapters[start:end], TotalPages: totalPages, CurrentPage: currentPage, }) w.Header().Set("Content-Type", "text/html; charset=utf-8") _, _ = buf.WriteTo(w) } // ─── GET /books/{slug}/chapters/{n} — chapter reader ───────────────────────── const chapterTmpl = `

← {{if .PrevN}} ← Prev {{else}} {{end}} {{if .NextN}} Next → {{else}} {{end}}

` func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") n, err := strconv.Atoi(r.PathValue("n")) if err != nil || n < 1 { http.NotFound(w, r) return } raw, err := s.writer.ReadChapter(slug, n) if err != nil { http.NotFound(w, r) return } var htmlBuf bytes.Buffer if err := md.Convert([]byte(raw), &htmlBuf); err != nil { http.Error(w, "markdown render error: "+err.Error(), http.StatusInternalServerError) return } chapters, _ := s.writer.ListChapters(slug) prevN, nextN := adjacentChapters(chapters, n) title := firstHeading(raw, fmt.Sprintf("Chapter %d", n)) t := template.Must(template.New("chapter").Parse(chapterTmpl)) var buf bytes.Buffer _ = t.Execute(&buf, struct { Slug string HTML template.HTML PrevN int NextN int ChapterN int Title string AllChapters interface{} Voices []string DefaultVoice string }{ Slug: slug, HTML: template.HTML(htmlBuf.String()), PrevN: prevN, NextN: nextN, ChapterN: n, Title: title, AllChapters: chapters, Voices: kokoroVoices, DefaultVoice: s.kokoroVoice, }) s.respond(w, r, title, buf.String()) } // ─── helpers ────────────────────────────────────────────────────────────────── // stripMarkdown removes Markdown syntax and returns clean plain text. func stripMarkdown(src string) string { src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "") src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "") src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "") src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "") src = regexp.MustCompile(`\[([^\]]+)\]$[^)]+$`).ReplaceAllString(src, "$1") src = regexp.MustCompile(`!\[[^\]]*\]$[^)]+$`).ReplaceAllString(src, "") src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "") src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "") src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n") return strings.TrimSpace(src) } // adjacentChapters returns the chapter numbers immediately before and after n // in the sorted chapters list. 0 means "does not exist". func adjacentChapters(chapters []writer.ChapterInfo, n int) (prev, next int) { for i, ch := range chapters { if ch.Number == n { if i > 0 { prev = chapters[i-1].Number } if i < len(chapters)-1 { next = chapters[i+1].Number } return } } return } // firstHeading returns the text of the first non-empty line, stripping a // leading "# " markdown heading marker. Falls back to fallback. func firstHeading(md, fallback string) string { for _, line := range strings.SplitN(md, "\n", 20) { line = strings.TrimSpace(line) if line == "" { continue } return strings.TrimPrefix(line, "# ") } return fallback } // ─── POST /ui/scrape/book — form submission ─────────────────────────────────── func (s *Server) handleUIScrapeBook(w http.ResponseWriter, r *http.Request) { bookURL := strings.TrimSpace(r.FormValue("url")) if bookURL == "" { renderFragment(w, scrapeStatusHTML("error", "Please enter a book URL.")) return } s.mu.Lock() already := s.running if !already { s.running = true } s.mu.Unlock() if already { renderFragment(w, scrapeStatusHTML("busy", "A scrape job is already running. Please wait.")) return } cfg := s.oCfg cfg.SingleBookURL = bookURL go func() { defer func() { s.mu.Lock() s.running = false s.mu.Unlock() }() ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) defer cancel() o := orchestrator.New(cfg, s.novel, s.log) if err := o.Run(ctx); err != nil { s.log.Error("UI scrape job failed", "url", bookURL, "err", err) } }() // Return a status badge that polls until the job finishes. renderFragment(w, scrapeStatusHTML("running", "Scraping "+bookURL+"…")) } // ─── GET /ui/scrape/status — polling endpoint ───────────────────────────────── func (s *Server) handleUIScrapeStatus(w http.ResponseWriter, r *http.Request) { s.mu.Lock() running := s.running s.mu.Unlock() if running { // Keep polling every 3 s while the job is in progress. renderFragment(w, scrapeStatusHTML("running", "Scraping in progress…")) return } // Job finished — show a done badge and stop polling. renderFragment(w, scrapeStatusHTML("done", "Done! Refresh the page to see new books.")) } // scrapeStatusHTML returns a self-contained status badge fragment. // state is one of: "running" | "done" | "busy" | "error". func scrapeStatusHTML(state, msg string) string { var colour, dot, poll string switch state { case "running": colour = "text-amber-300 bg-amber-950 border-amber-800" dot = `` poll = `hx-get="/ui/scrape/status" hx-trigger="every 3s" hx-target="this" hx-swap="outerHTML"` case "done": colour = "text-green-300 bg-green-950 border-green-800" dot = `` case "busy": colour = "text-yellow-300 bg-yellow-950 border-yellow-800" dot = `` default: // error colour = "text-red-300 bg-red-950 border-red-800" dot = `` } return fmt.Sprintf( `

%s%s

`, colour, poll, dot, template.HTMLEscapeString(msg), ) }

libnovel

Scrape a new book

Continue reading

Available

{{.Title}}

Novel Rankings

{{.Title}}

{{.Title}}

Ranking Data

{{.Meta.Title}}

Chapters

{{.Title}}