package server import ( "bytes" "context" "fmt" "html/template" "net/http" "regexp" "strconv" "strings" "time" "github.com/libnovel/scraper/internal/orchestrator" "github.com/libnovel/scraper/internal/writer" "github.com/yuin/goldmark" "github.com/yuin/goldmark/extension" goldhtml "github.com/yuin/goldmark/renderer/html" ) // md is the shared goldmark instance used for all markdown→HTML conversions. var md = goldmark.New( goldmark.WithExtensions(extension.Typographer, extension.Table), goldmark.WithRendererOptions(goldhtml.WithUnsafe()), ) // kokoroVoices is the full list of voices shipped with Kokoro-FastAPI, // grouped loosely by language prefix: // // af_ / am_ American English female / male // bf_ / bm_ British English female / male // ef_ / em_ Spanish female / male // ff_ French female // hf_ / hm_ Hindi female / male // if_ / im_ Italian female / male // jf_ / jm_ Japanese female / male // pf_ / pm_ Portuguese female / male // zf_ / zm_ Chinese female / male var kokoroVoices = []string{ // American English "af_alloy", "af_aoede", "af_bella", "af_heart", "af_jadzia", "af_jessica", "af_kore", "af_nicole", "af_nova", "af_river", "af_sarah", "af_sky", "am_adam", "am_echo", "am_eric", "am_fenrir", "am_liam", "am_michael", "am_onyx", "am_puck", // British English "bf_alice", "bf_emma", "bf_lily", "bm_daniel", "bm_fable", "bm_george", "bm_lewis", // Spanish "ef_dora", "em_alex", // French "ff_siwis", // Hindi "hf_alpha", "hf_beta", "hm_omega", "hm_psi", // Italian "if_sara", "im_nicola", // Japanese "jf_alpha", "jf_gongitsune", "jf_nezumi", "jf_tebukuro", "jm_kumo", // Portuguese "pf_dora", "pm_alex", // Chinese "zf_xiaobei", "zf_xiaoni", "zf_xiaoxiao", "zf_xiaoyi", "zm_yunjian", "zm_yunxi", "zm_yunxia", "zm_yunyang", } // ─── shared layout ──────────────────────────────────────────────────────────── const layoutHead = ` {{.Title}} — libnovel ` const layoutFoot = `` func renderPage(w http.ResponseWriter, title, body string) { t := template.Must(template.New("layout").Parse(layoutHead + body + layoutFoot)) w.Header().Set("Content-Type", "text/html; charset=utf-8") _ = t.Execute(w, struct{ Title string }{Title: title}) } func renderFragment(w http.ResponseWriter, body string) { w.Header().Set("Content-Type", "text/html; charset=utf-8") fmt.Fprint(w, body) } func isHTMX(r *http.Request) bool { return r.Header.Get("HX-Request") == "true" } // respond writes either a full page or an HTMX fragment depending on the request. func (s *Server) respond(w http.ResponseWriter, r *http.Request, title, fragment string) { if isHTMX(r) { renderFragment(w, fragment) return } renderPage(w, title, `
`+fragment+`
`) } // ─── GET / — book catalogue ─────────────────────────────────────────────────── const homeTmpl = `

libnovel

Browse Rankings

{{len .Books}} book{{if ne (len .Books) 1}}s{{end}} on disk

Scrape a new book

{{range .Books}}
{{if .Cover}} cover {{end}}

{{.Title}}

{{if .Author}}

{{.Author}}

{{end}}
{{if .Status}}{{.Status}}{{end}} {{if .TotalChapters}}{{.TotalChapters}} ch{{end}}
{{else}}

No books scraped yet.

{{end}}
` func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/" { http.NotFound(w, r) return } books, err := s.writer.ListBooks() if err != nil { http.Error(w, "failed to list books: "+err.Error(), http.StatusInternalServerError) return } t := template.Must(template.New("home").Parse(homeTmpl)) var buf bytes.Buffer _ = t.Execute(&buf, struct{ Books interface{} }{Books: books}) s.respond(w, r, "Home", buf.String()) } // ─── GET /ranking — ranking page ─────────────────────────────────────────────── const rankingTmpl = `
← All books

Novel Rankings

Top novels from novelfire.net

{{if .CachedAt}}

Cached {{.CachedAt}}

{{end}}
View Markdown
{{range .Books}} {{if .Local}} {{/* Book is in local library — wrap entire card in a clickable link */}}
{{if .Cover}} cover {{end}}
{{if .Rank}}#{{.Rank}}{{end}}

{{.Title}}

In library
{{if .Author}}

{{.Author}}

{{end}}
{{if .Status}}{{.Status}}{{end}}
{{if .Genres}}
{{range .Genres}}{{.}}{{end}}
{{end}}
{{else}} {{/* Book not yet in local library */}}
{{if .Cover}} cover {{end}}
{{if .Rank}}#{{.Rank}}{{end}}

{{.Title}}

{{if .Author}}

{{.Author}}

{{end}}
{{if .Status}}{{.Status}}{{end}}
{{if .Genres}}
{{range .Genres}}{{.}}{{end}}
{{end}} {{if .SourceURL}}
{{end}}
{{end}} {{else}}

No ranking data available. Click "Refresh Rankings" to fetch from novelfire.net.

{{end}}
` // rankingViewItem enriches a RankingItem with whether it is present in the // local book library, so the template can highlight it differently. type rankingViewItem struct { writer.RankingItem Local bool } // toRankingViewItems annotates items with Local=true for slugs found in localSlugs. func toRankingViewItems(items []writer.RankingItem, localSlugs map[string]bool) []rankingViewItem { out := make([]rankingViewItem, len(items)) for i, it := range items { out[i] = rankingViewItem{ RankingItem: it, Local: localSlugs[it.Slug], } } return out } // handleRanking serves the ranking page from the cached ranking.md file. // It does NOT trigger a live scrape; use POST /ranking/refresh for that. func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) { rankingItems, err := s.writer.ReadRankingItems() if err != nil { s.log.Error("failed to read cached ranking", "err", err) } cachedAt := "" if info, statErr := s.writer.RankingFileInfo(); statErr == nil { cachedAt = info.ModTime().Format("Jan 2, 2006 at 15:04") } t := template.Must(template.New("ranking").Parse(rankingTmpl)) var buf bytes.Buffer _ = t.Execute(&buf, struct { Books interface{} CachedAt string }{Books: toRankingViewItems(rankingItems, s.writer.LocalSlugs()), CachedAt: cachedAt}) s.respond(w, r, "Rankings", buf.String()) } // handleRankingRefresh triggers a live scrape of novelfire.net/ranking, // persists the result to ranking.md, then re-renders the ranking page. func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) defer cancel() rankingCh, errCh := s.novel.ScrapeRanking(ctx) var rankingItems []writer.RankingItem for { select { case meta, ok := <-rankingCh: if !ok { rankingCh = nil continue } rankingItems = append(rankingItems, writer.RankingItem{ Rank: meta.Ranking, Slug: meta.Slug, Title: meta.Title, Author: meta.Author, Cover: meta.Cover, Status: meta.Status, Genres: meta.Genres, SourceURL: meta.SourceURL, }) case err, ok := <-errCh: if !ok { errCh = nil continue } if err != nil { s.log.Error("ranking scrape error", "err", err) } } if rankingCh == nil && errCh == nil { break } } if len(rankingItems) > 0 { if err := s.writer.WriteRanking(rankingItems); err != nil { s.log.Error("failed to save ranking", "err", err) } } cachedAt := "" if info, statErr := s.writer.RankingFileInfo(); statErr == nil { cachedAt = info.ModTime().Format("Jan 2, 2006 at 15:04") } t := template.Must(template.New("ranking").Parse(rankingTmpl)) var buf bytes.Buffer _ = t.Execute(&buf, struct { Books interface{} CachedAt string }{Books: toRankingViewItems(rankingItems, s.writer.LocalSlugs()), CachedAt: cachedAt}) s.respond(w, r, "Rankings", buf.String()) } // ─── GET /ranking/view — view ranking markdown ───────────────────────────────── const rankingViewTmpl = `
← Back to Rankings

Ranking Data

{{.HTML}}
` func (s *Server) handleRankingView(w http.ResponseWriter, r *http.Request) { markdown, err := s.writer.ReadRanking() if err != nil { http.Error(w, "failed to read ranking: "+err.Error(), http.StatusInternalServerError) return } if markdown == "" { http.NotFound(w, r) return } var htmlBuf bytes.Buffer if err := md.Convert([]byte(markdown), &htmlBuf); err != nil { http.Error(w, "markdown render error: "+err.Error(), http.StatusInternalServerError) return } t := template.Must(template.New("rankingView").Parse(rankingViewTmpl)) var buf bytes.Buffer _ = t.Execute(&buf, struct{ HTML template.HTML }{HTML: template.HTML(htmlBuf.String())}) s.respond(w, r, "Ranking Data", buf.String()) } // ─── GET /books/{slug} — chapter list ──────────────────────────────────────── const bookTmpl = `
← All books
{{if .Meta.Cover}} cover {{end}}

{{.Meta.Title}}

{{if .Meta.SourceURL}}
{{end}}
{{if .Meta.Author}}

{{.Meta.Author}}

{{end}}
{{if .Meta.Status}}{{.Meta.Status}}{{end}} {{if .Meta.TotalChapters}}{{.Meta.TotalChapters}} ch total{{end}} {{len .Chapters}} downloaded
{{if .Meta.Summary}}

{{.Meta.Summary}}

{{end}}

Chapters

` func (s *Server) handleBook(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") meta, ok, err := s.writer.ReadMetadata(slug) if err != nil { http.Error(w, "failed to read metadata: "+err.Error(), http.StatusInternalServerError) return } if !ok { http.NotFound(w, r) return } chapters, err := s.writer.ListChapters(slug) if err != nil { http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError) return } t := template.Must(template.New("book").Parse(bookTmpl)) var buf bytes.Buffer _ = t.Execute(&buf, struct { Slug string Meta interface{} Chapters interface{} }{Slug: slug, Meta: meta, Chapters: chapters}) s.respond(w, r, meta.Title, buf.String()) } // ─── GET /books/{slug}/chapters/{n} — chapter reader ───────────────────────── const chapterTmpl = `
← Chapter list
{{if .PrevN}} ← Prev {{end}} {{if .NextN}} Next → {{end}}
Speed 1.0×
{{.HTML}}
{{if .PrevN}} ← Previous chapter {{else}}{{end}} {{if .NextN}} Next chapter → {{else}}{{end}}
` func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") n, err := strconv.Atoi(r.PathValue("n")) if err != nil || n < 1 { http.NotFound(w, r) return } raw, err := s.writer.ReadChapter(slug, n) if err != nil { http.NotFound(w, r) return } var htmlBuf bytes.Buffer if err := md.Convert([]byte(raw), &htmlBuf); err != nil { http.Error(w, "markdown render error: "+err.Error(), http.StatusInternalServerError) return } chapters, _ := s.writer.ListChapters(slug) prevN, nextN := adjacentChapters(chapters, n) title := firstHeading(raw, fmt.Sprintf("Chapter %d", n)) t := template.Must(template.New("chapter").Parse(chapterTmpl)) var buf bytes.Buffer _ = t.Execute(&buf, struct { Slug string HTML template.HTML PrevN int NextN int ChapterN int KokoroURL string Voices []string DefaultVoice string }{ Slug: slug, HTML: template.HTML(htmlBuf.String()), PrevN: prevN, NextN: nextN, ChapterN: n, KokoroURL: s.kokoroURL, Voices: kokoroVoices, DefaultVoice: s.kokoroVoice, }) s.respond(w, r, title, buf.String()) } // ─── helpers ────────────────────────────────────────────────────────────────── // stripMarkdown removes Markdown syntax and returns clean plain text. func stripMarkdown(src string) string { src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "") src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "") src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "") src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "") src = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`).ReplaceAllString(src, "$1") src = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`).ReplaceAllString(src, "") src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "") src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "") src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n") return strings.TrimSpace(src) } // adjacentChapters returns the chapter numbers immediately before and after n // in the sorted chapters list. 0 means "does not exist". func adjacentChapters(chapters []writer.ChapterInfo, n int) (prev, next int) { for i, ch := range chapters { if ch.Number == n { if i > 0 { prev = chapters[i-1].Number } if i < len(chapters)-1 { next = chapters[i+1].Number } return } } return } // firstHeading returns the text of the first non-empty line, stripping a // leading "# " markdown heading marker. Falls back to fallback. func firstHeading(md, fallback string) string { for _, line := range strings.SplitN(md, "\n", 20) { line = strings.TrimSpace(line) if line == "" { continue } return strings.TrimPrefix(line, "# ") } return fallback } // ─── POST /ui/scrape/book — form submission ─────────────────────────────────── func (s *Server) handleUIScrapeBook(w http.ResponseWriter, r *http.Request) { bookURL := strings.TrimSpace(r.FormValue("url")) if bookURL == "" { renderFragment(w, scrapeStatusHTML("error", "Please enter a book URL.")) return } s.mu.Lock() already := s.running if !already { s.running = true } s.mu.Unlock() if already { renderFragment(w, scrapeStatusHTML("busy", "A scrape job is already running. Please wait.")) return } cfg := s.oCfg cfg.SingleBookURL = bookURL go func() { defer func() { s.mu.Lock() s.running = false s.mu.Unlock() }() ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) defer cancel() o := orchestrator.New(cfg, s.novel, s.log) if err := o.Run(ctx); err != nil { s.log.Error("UI scrape job failed", "url", bookURL, "err", err) } }() // Return a status badge that polls until the job finishes. renderFragment(w, scrapeStatusHTML("running", "Scraping "+bookURL+"…")) } // ─── GET /ui/scrape/status — polling endpoint ───────────────────────────────── func (s *Server) handleUIScrapeStatus(w http.ResponseWriter, r *http.Request) { s.mu.Lock() running := s.running s.mu.Unlock() if running { // Keep polling every 3 s while the job is in progress. renderFragment(w, scrapeStatusHTML("running", "Scraping in progress…")) return } // Job finished — show a done badge and stop polling. renderFragment(w, scrapeStatusHTML("done", "Done! Refresh the page to see new books.")) } // scrapeStatusHTML returns a self-contained status badge fragment. // state is one of: "running" | "done" | "busy" | "error". func scrapeStatusHTML(state, msg string) string { var colour, dot, poll string switch state { case "running": colour = "text-amber-300 bg-amber-950 border-amber-800" dot = `` poll = `hx-get="/ui/scrape/status" hx-trigger="every 3s" hx-target="this" hx-swap="outerHTML"` case "done": colour = "text-green-300 bg-green-950 border-green-800" dot = `` case "busy": colour = "text-yellow-300 bg-yellow-950 border-yellow-800" dot = `` default: // error colour = "text-red-300 bg-red-950 border-red-800" dot = `` } return fmt.Sprintf( `
%s%s
`, colour, poll, dot, template.HTMLEscapeString(msg), ) }