package server import ( "bytes" "context" "encoding/json" "fmt" "html/template" "net/http" "regexp" "strconv" "strings" "time" "github.com/libnovel/scraper/internal/orchestrator" "github.com/libnovel/scraper/internal/writer" "github.com/yuin/goldmark" "github.com/yuin/goldmark/extension" goldhtml "github.com/yuin/goldmark/renderer/html" ) // md is the shared goldmark instance used for all markdown→HTML conversions. var md = goldmark.New( goldmark.WithExtensions(extension.Typographer, extension.Table), goldmark.WithRendererOptions(goldhtml.WithUnsafe()), ) // kokoroVoices is the full list of voices shipped with Kokoro-FastAPI, // grouped loosely by language prefix: // // af_ / am_ American English female / male // bf_ / bm_ British English female / male // ef_ / em_ Spanish female / male // ff_ French female // hf_ / hm_ Hindi female / male // if_ / im_ Italian female / male // jf_ / jm_ Japanese female / male // pf_ / pm_ Portuguese female / male // zf_ / zm_ Chinese female / male var kokoroVoices = []string{ // American English "af_alloy", "af_aoede", "af_bella", "af_heart", "af_jadzia", "af_jessica", "af_kore", "af_nicole", "af_nova", "af_river", "af_sarah", "af_sky", "am_adam", "am_echo", "am_eric", "am_fenrir", "am_liam", "am_michael", "am_onyx", "am_puck", // British English "bf_alice", "bf_emma", "bf_lily", "bm_daniel", "bm_fable", "bm_george", "bm_lewis", // Spanish "ef_dora", "em_alex", // French "ff_siwis", // Hindi "hf_alpha", "hf_beta", "hm_omega", "hm_psi", // Italian "if_sara", "im_nicola", // Japanese "jf_alpha", "jf_gongitsune", "jf_nezumi", "jf_tebukuro", "jm_kumo", // Portuguese "pf_dora", "pm_alex", // Chinese "zf_xiaobei", "zf_xiaoni", "zf_xiaoxiao", "zf_xiaoyi", "zm_yunjian", "zm_yunxi", "zm_yunxia", "zm_yunyang", } // ─── shared layout ──────────────────────────────────────────────────────────── const layoutHead = ` {{.Title}} — libnovel ` const layoutFoot = `` func renderPage(w http.ResponseWriter, title, body string) { t := template.Must(template.New("layout").Parse(layoutHead + body + layoutFoot)) w.Header().Set("Content-Type", "text/html; charset=utf-8") _ = t.Execute(w, struct{ Title string }{Title: title}) } func renderFragment(w http.ResponseWriter, body string) { w.Header().Set("Content-Type", "text/html; charset=utf-8") fmt.Fprint(w, body) } func isHTMX(r *http.Request) bool { return r.Header.Get("HX-Request") == "true" } // respond writes either a full page or an HTMX fragment depending on the request. func (s *Server) respond(w http.ResponseWriter, r *http.Request, title, fragment string) { if isHTMX(r) { renderFragment(w, fragment) return } renderPage(w, title, `

`+fragment+`

`) } // ─── GET / — book catalogue ─────────────────────────────────────────────────── const homeTmpl = `

libnovel

+ Add Browse Rankings

{{len .Books}} book{{if ne (len .Books) 1}}s{{end}} on disk

Available

{{if .Status}}{{.Status}}{{end}} {{if .TotalChapters}}{{.TotalChapters}} ch{{end}}

No books scraped yet.

` func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/" { http.NotFound(w, r) return } books, err := s.writer.ListBooks() if err != nil { http.Error(w, "failed to list books: "+err.Error(), http.StatusInternalServerError) return } t := template.Must(template.New("home").Parse(homeTmpl)) var buf bytes.Buffer _ = t.Execute(&buf, struct { Books interface{} }{ Books: books, }) s.respond(w, r, "Home", buf.String()) } // ─── GET /scrape — add a new book ───────────────────────────────────────────── const scrapeTmpl = `

← All books

Add a book

Search the rankings or paste a novelfire.net URL to scrape a new book.

` func (s *Server) handleScrape(w http.ResponseWriter, r *http.Request) { rankingItems, _ := s.writer.ReadRankingItems() rankingJSON, _ := json.Marshal(rankingItems) t := template.Must(template.New("scrape").Parse(scrapeTmpl)) var buf bytes.Buffer _ = t.Execute(&buf, struct { RankingJSON template.JS }{ RankingJSON: template.JS(rankingJSON), }) s.respond(w, r, "Add a book", buf.String()) } // ─── GET /ranking — ranking page ─────────────────────────────────────────────── const rankingTmpl = `

← All books

Novel Rankings

Top novels from novelfire.net

Cached {{.CachedAt}}

View Markdown

Page: {{$cur := .CurrentPage}} {{range .DisplayNums}} {{if eq .Num 0}} … {{else if eq .Num $cur}} {{.Num}} {{else}} {{.Num}} {{end}} {{end}}

Showing {{.TotalItems}} cached novels · page {{.CurrentPage}} of {{.TotalPages}}

Fetch more from novelfire.net

Fetch up to page: {{range .FetchNums}} {{if eq .Num 0}} … {{else}} {{end}} {{end}}

Each page ≈ 20 novels from novelfire.net popular

{{range .Books}} {{if .Local}} {{if .Cover}} cover

{{if .Rank}}#{{.Rank}}{{end}}

In library

{{if .Cover}} cover

{{if .Rank}}#{{.Rank}}{{end}}

Source

No ranking data. Use the page buttons above to fetch from novelfire.net.

` // rankingViewItem enriches a RankingItem with whether it is present in the // local book library, so the template can highlight it differently. type rankingViewItem struct { writer.RankingItem Local bool } // toRankingViewItems annotates items with Local=true for slugs found in localSlugs. func toRankingViewItems(items []writer.RankingItem, localSlugs map[string]bool) []rankingViewItem { out := make([]rankingViewItem, len(items)) for i, it := range items { out[i] = rankingViewItem{ RankingItem: it, Local: localSlugs[it.Slug], } } return out } // pageNum is one entry in the ranking pagination bar. // Num == 0 is a sentinel that renders as an ellipsis gap. type pageNum struct { Num int } // rankingPageNums builds a pagination list with smart ellipsis. // It always shows: first 2, last 2, and a ±2 window around current. // Gaps between non-consecutive runs are filled with a sentinel (Num==0) for "…". // Pass current=0 when there is no concept of a current page (e.g. fetch bar). func rankingPageNums(total, current int) []pageNum { if total <= 0 { return nil } show := make(map[int]bool) // First 2 and last 2. for i := 1; i <= 2 && i <= total; i++ { show[i] = true } for i := total - 1; i <= total; i++ { if i >= 1 { show[i] = true } } // ±2 window around current page. if current > 0 { for i := current - 2; i <= current+2; i++ { if i >= 1 && i <= total { show[i] = true } } } // Collect and sort. pages := make([]int, 0, len(show)) for p := range show { pages = append(pages, p) } for i := 0; i < len(pages); i++ { for j := i + 1; j < len(pages); j++ { if pages[j] < pages[i] { pages[i], pages[j] = pages[j], pages[i] } } } // Build output with ellipsis sentinels between non-consecutive pages. out := make([]pageNum, 0, len(pages)*2) for i, p := range pages { if i > 0 && p > pages[i-1]+1 { out = append(out, pageNum{0}) } out = append(out, pageNum{p}) } return out } const rankingPageSize = 20 // handleRanking serves the ranking page from the cached ranking.json file. // It does NOT trigger a live scrape; use POST /ranking/refresh for that. // Supports ?page=N for browsing through cached items (20 per page). func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) { rankingItems, err := s.writer.ReadRankingItems() if err != nil { s.log.Error("failed to read cached ranking", "err", err) } cachedAt := "" if info, statErr := s.writer.RankingFileInfo(); statErr == nil { cachedAt = info.ModTime().Format("Jan 2, 2006 at 15:04") } // Parse requested display page (1-indexed). currentPage := 1 if p := r.URL.Query().Get("page"); p != "" { if n, err2 := strconv.Atoi(p); err2 == nil && n > 0 { currentPage = n } } totalItems := len(rankingItems) totalPages := 1 if totalItems > 0 { totalPages = (totalItems + rankingPageSize - 1) / rankingPageSize } if currentPage > totalPages { currentPage = totalPages } // Slice items for the current display page. start := (currentPage - 1) * rankingPageSize end := start + rankingPageSize if end > totalItems { end = totalItems } pageItems := rankingItems if totalItems > 0 { pageItems = rankingItems[start:end] } t := template.Must(template.New("ranking").Parse(rankingTmpl)) var buf bytes.Buffer // Collect distinct genres and statuses across ALL items for facet filters. genreSet := map[string]bool{} statusSet := map[string]bool{} for _, it := range rankingItems { if it.Status != "" { statusSet[it.Status] = true } for _, g := range it.Genres { genreSet[g] = true } } allGenres := sortedKeys(genreSet) allStatuses := sortedKeys(statusSet) _ = t.Execute(&buf, struct { Books interface{} CachedAt string FetchNums []pageNum DisplayNums []pageNum CurrentPage int TotalPages int TotalItems int AllGenres []string AllStatuses []string }{ Books: toRankingViewItems(pageItems, s.writer.LocalSlugs()), CachedAt: cachedAt, FetchNums: rankingPageNums(100, 0), DisplayNums: rankingPageNums(totalPages, currentPage), CurrentPage: currentPage, TotalPages: totalPages, TotalItems: totalItems, AllGenres: allGenres, AllStatuses: allStatuses, }) s.respond(w, r, "Rankings", buf.String()) } // handleRankingRefresh starts an async scrape of novelfire.net/ranking and // immediately returns a polling badge. The browser polls /ui/ranking/status // until the job finishes, then follows an HX-Redirect back to /ranking. // // Accepts an optional form field "pages" (integer ≥ 1). 0 or absent means // fetch all pages; otherwise at most that many pages are scraped. func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) { _ = r.ParseForm() maxPages := 0 if p := strings.TrimSpace(r.FormValue("pages")); p != "" { if n, err := strconv.Atoi(p); err == nil && n > 0 { maxPages = n } } s.mu.Lock() if s.rankingRunning { s.mu.Unlock() renderFragment(w, rankingStatusHTML("running", "Ranking refresh already in progress…")) return } s.rankingRunning = true s.mu.Unlock() go func() { defer func() { s.mu.Lock() s.rankingRunning = false s.mu.Unlock() }() // Allow ~90 s per page; minimum 120 s for a single page. timeout := 120 * time.Second if maxPages > 1 { timeout = time.Duration(maxPages) * 90 * time.Second } ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() rankingCh, errCh := s.novel.ScrapeRanking(ctx, maxPages) var rankingItems []writer.RankingItem for rankingCh != nil || errCh != nil { select { case meta, ok := <-rankingCh: if !ok { rankingCh = nil } else { rankingItems = append(rankingItems, writer.RankingItem{ Rank: meta.Ranking, Slug: meta.Slug, Title: meta.Title, Author: meta.Author, Cover: meta.Cover, Status: meta.Status, Genres: meta.Genres, SourceURL: meta.SourceURL, }) } case err, ok := <-errCh: if !ok { errCh = nil } else if err != nil { s.log.Error("ranking scrape error", "err", err) } } } if len(rankingItems) > 0 { if err := s.writer.WriteRanking(rankingItems); err != nil { s.log.Error("failed to save ranking", "err", err) } } }() renderFragment(w, rankingStatusHTML("running", "Fetching rankings…")) } // handleRankingStatus is the HTMX polling endpoint for ranking refresh jobs. // While running it returns a self-replacing badge; when done it issues an // HX-Redirect so the browser navigates to /ranking. func (s *Server) handleRankingStatus(w http.ResponseWriter, r *http.Request) { s.mu.Lock() running := s.rankingRunning s.mu.Unlock() if running { renderFragment(w, rankingStatusHTML("running", "Fetching rankings…")) return } // Job done — redirect the HTMX request to the ranking page. w.Header().Set("HX-Redirect", "/ranking") w.WriteHeader(http.StatusOK) } // rankingStatusHTML returns a self-replacing polling badge for the ranking // refresh job. state is "running" or "done". func rankingStatusHTML(state, msg string) string { var colour, dot, poll string switch state { case "running": colour = "text-amber-300 bg-amber-950 border-amber-800" dot = `` poll = `hx-get="/ui/ranking/status" hx-trigger="every 3s" hx-target="this" hx-swap="outerHTML"` default: colour = "text-green-300 bg-green-950 border-green-800" dot = `` } return fmt.Sprintf( `

%s%s

`, colour, poll, dot, template.HTMLEscapeString(msg), ) } // ─── GET /ranking/view — view ranking markdown ───────────────────────────────── const rankingViewTmpl = `

← Back to Rankings

Ranking Data

{{.JSON}}

` func (s *Server) handleRankingView(w http.ResponseWriter, r *http.Request) { items, err := s.writer.ReadRankingItems() if err != nil { http.Error(w, "failed to read ranking: "+err.Error(), http.StatusInternalServerError) return } if len(items) == 0 { http.NotFound(w, r) return } pretty, err := json.MarshalIndent(items, "", " ") if err != nil { http.Error(w, "json marshal error: "+err.Error(), http.StatusInternalServerError) return } t := template.Must(template.New("rankingView").Parse(rankingViewTmpl)) var buf bytes.Buffer _ = t.Execute(&buf, struct{ JSON string }{JSON: string(pretty)}) s.respond(w, r, "Ranking Data", buf.String()) } // ─── GET /books/{slug} — chapter list ──────────────────────────────────────── const chapterPageSize = 50 const bookTmpl = `

← All books

{{if .Meta.Cover}} cover

{{if .Meta.SourceURL}} Source {{end}}

{{if .Meta.Status}}{{.Meta.Status}}{{end}} {{if .Meta.TotalChapters}}{{.Meta.TotalChapters}} ch total{{end}} {{.TotalDownloaded}} downloaded

Chapters

{{.Number}}
{{.Title}} {{if .Date}}{{.Date}}{{end}}
No chapters downloaded yet.

{{if gt .CurrentPage 1}} « {{end}} {{range pages .TotalPages}} {{if eq . $.CurrentPage}} {{.}} {{else}} {{.}} {{end}} {{end}} {{if lt .CurrentPage .TotalPages}} » {{end}}

` func (s *Server) handleBook(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") meta, ok, err := s.writer.ReadMetadata(slug) if err != nil { http.Error(w, "failed to read metadata: "+err.Error(), http.StatusInternalServerError) return } if !ok { http.NotFound(w, r) return } chapters, err := s.writer.ListChapters(slug) if err != nil { http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError) return } total := len(chapters) totalPages := (total + chapterPageSize - 1) / chapterPageSize if totalPages < 1 { totalPages = 1 } currentPage := 1 page := chapters if total > chapterPageSize { page = chapters[:chapterPageSize] } funcMap := template.FuncMap{ "pages": func(n int) []int { out := make([]int, n) for i := range out { out[i] = i + 1 } return out }, "prev": func(n int) int { return n - 1 }, "next": func(n int) int { return n + 1 }, } t := template.Must(template.New("book").Funcs(funcMap).Parse(bookTmpl)) var buf bytes.Buffer _ = t.Execute(&buf, struct { Slug string Meta interface{} Chapters interface{} TotalDownloaded int TotalPages int CurrentPage int }{ Slug: slug, Meta: meta, Chapters: page, TotalDownloaded: total, TotalPages: totalPages, CurrentPage: currentPage, }) s.respond(w, r, meta.Title, buf.String()) } // ─── GET /books/{slug}/chapters-page — paginated chapter list fragment ──────── const chapterPageTmpl = `{{range .Chapters}}

{{if gt .CurrentPage 1}} « {{end}} {{range pages .TotalPages}} {{if eq . $.CurrentPage}} {{.}} {{else}} {{.}} {{end}} {{end}} {{if lt .CurrentPage .TotalPages}} » {{end}}

` func (s *Server) handleBookChaptersPage(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") currentPage := 1 if p := r.URL.Query().Get("page"); p != "" { if n, err := strconv.Atoi(p); err == nil && n > 0 { currentPage = n } } chapters, err := s.writer.ListChapters(slug) if err != nil { http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError) return } total := len(chapters) totalPages := (total + chapterPageSize - 1) / chapterPageSize if totalPages < 1 { totalPages = 1 } start := (currentPage - 1) * chapterPageSize if start >= total { w.WriteHeader(http.StatusNoContent) return } end := start + chapterPageSize if end > total { end = total } funcMap := template.FuncMap{ "pages": func(n int) []int { out := make([]int, n) for i := range out { out[i] = i + 1 } return out }, "prev": func(n int) int { return n - 1 }, "next": func(n int) int { return n + 1 }, } t := template.Must(template.New("chapterPage").Funcs(funcMap).Parse(chapterPageTmpl)) var buf bytes.Buffer _ = t.Execute(&buf, struct { Slug string Chapters interface{} TotalPages int CurrentPage int }{ Slug: slug, Chapters: chapters[start:end], TotalPages: totalPages, CurrentPage: currentPage, }) w.Header().Set("Content-Type", "text/html; charset=utf-8") _, _ = buf.WriteTo(w) } // ─── GET /books/{slug}/chapters/{n} — chapter reader ───────────────────────── const chapterTmpl = `

☰ {{if .PrevN}} ← Prev {{end}} {{if .NextN}} Next → {{end}}

` func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) { slug := r.PathValue("slug") n, err := strconv.Atoi(r.PathValue("n")) if err != nil || n < 1 { http.NotFound(w, r) return } raw, err := s.writer.ReadChapter(slug, n) if err != nil { http.NotFound(w, r) return } var htmlBuf bytes.Buffer if err := md.Convert([]byte(raw), &htmlBuf); err != nil { http.Error(w, "markdown render error: "+err.Error(), http.StatusInternalServerError) return } chapters, _ := s.writer.ListChapters(slug) prevN, nextN := adjacentChapters(chapters, n) title := firstHeading(raw, fmt.Sprintf("Chapter %d", n)) chapterTitle, chapterDate := writer.SplitChapterTitle(title) t := template.Must(template.New("chapter").Parse(chapterTmpl)) var buf bytes.Buffer _ = t.Execute(&buf, struct { Slug string HTML template.HTML PrevN int NextN int ChapterN int Title string ChapterDate string AllChapters interface{} Voices []string DefaultVoice string }{ Slug: slug, HTML: template.HTML(htmlBuf.String()), PrevN: prevN, NextN: nextN, ChapterN: n, Title: chapterTitle, ChapterDate: chapterDate, AllChapters: chapters, Voices: kokoroVoices, DefaultVoice: s.kokoroVoice, }) s.respond(w, r, chapterTitle, buf.String()) } // ─── helpers ────────────────────────────────────────────────────────────────── // sortedKeys returns the keys of a string-bool map in sorted order. func sortedKeys(m map[string]bool) []string { out := make([]string, 0, len(m)) for k := range m { out = append(out, k) } // Simple insertion sort — sets are small (< 100 items). for i := 1; i < len(out); i++ { for j := i; j > 0 && out[j] < out[j-1]; j-- { out[j], out[j-1] = out[j-1], out[j] } } return out } // stripMarkdown removes Markdown syntax and returns clean plain text. func stripMarkdown(src string) string { src = regexp.MustCompile(`(?m)^#{1,6}\s+`).ReplaceAllString(src, "") src = regexp.MustCompile(`\*{1,3}|_{1,3}`).ReplaceAllString(src, "") src = regexp.MustCompile("(?s)```.*?```").ReplaceAllString(src, "") src = regexp.MustCompile("`[^`]*`").ReplaceAllString(src, "") src = regexp.MustCompile(`\[([^\]]+)\]$[^)]+$`).ReplaceAllString(src, "$1") src = regexp.MustCompile(`!\[[^\]]*\]$[^)]+$`).ReplaceAllString(src, "") src = regexp.MustCompile(`(?m)^>\s?`).ReplaceAllString(src, "") src = regexp.MustCompile(`(?m)^[-*_]{3,}\s*$`).ReplaceAllString(src, "") src = regexp.MustCompile(`\n{3,}`).ReplaceAllString(src, "\n\n") return strings.TrimSpace(src) } // adjacentChapters returns the chapter numbers immediately before and after n // in the sorted chapters list. 0 means "does not exist". func adjacentChapters(chapters []writer.ChapterInfo, n int) (prev, next int) { for i, ch := range chapters { if ch.Number == n { if i > 0 { prev = chapters[i-1].Number } if i < len(chapters)-1 { next = chapters[i+1].Number } return } } return } // firstHeading returns the text of the first non-empty line, stripping a // leading "# " markdown heading marker. Falls back to fallback. func firstHeading(md, fallback string) string { for _, line := range strings.SplitN(md, "\n", 20) { line = strings.TrimSpace(line) if line == "" { continue } return strings.TrimPrefix(line, "# ") } return fallback } // ─── POST /ui/scrape/book — form submission ─────────────────────────────────── func (s *Server) handleUIScrapeBook(w http.ResponseWriter, r *http.Request) { bookURL := strings.TrimSpace(r.FormValue("url")) if bookURL == "" { renderFragment(w, scrapeStatusHTML("error", "Please enter a book URL.")) return } s.mu.Lock() already := s.running if !already { s.running = true } s.mu.Unlock() if already { renderFragment(w, scrapeStatusHTML("busy", "A scrape job is already running. Please wait.")) return } cfg := s.oCfg cfg.SingleBookURL = bookURL go func() { defer func() { s.mu.Lock() s.running = false s.mu.Unlock() }() ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) defer cancel() o := orchestrator.New(cfg, s.novel, s.log) if err := o.Run(ctx); err != nil { s.log.Error("UI scrape job failed", "url", bookURL, "err", err) } }() // Return a status badge that polls until the job finishes. renderFragment(w, scrapeStatusHTML("running", "Scraping "+bookURL+"…")) } // ─── GET /ui/scrape/status — polling endpoint ───────────────────────────────── func (s *Server) handleUIScrapeStatus(w http.ResponseWriter, r *http.Request) { s.mu.Lock() running := s.running s.mu.Unlock() if running { // Keep polling every 3 s while the job is in progress. renderFragment(w, scrapeStatusHTML("running", "Scraping in progress…")) return } // Job finished — show a done badge and stop polling. renderFragment(w, scrapeStatusHTML("done", "Done! Refresh the page to see new books.")) } // scrapeStatusHTML returns a self-contained status badge fragment. // state is one of: "running" | "done" | "busy" | "error". func scrapeStatusHTML(state, msg string) string { var colour, dot, poll string switch state { case "running": colour = "text-amber-300 bg-amber-950 border-amber-800" dot = `` poll = `hx-get="/ui/scrape/status" hx-trigger="every 3s" hx-target="this" hx-swap="outerHTML"` case "done": colour = "text-green-300 bg-green-950 border-green-800" dot = `` case "busy": colour = "text-yellow-300 bg-yellow-950 border-yellow-800" dot = `` default: // error colour = "text-red-300 bg-red-950 border-red-800" dot = `` } return fmt.Sprintf( `

%s%s

`, colour, poll, dot, template.HTMLEscapeString(msg), ) }

libnovel

Continue reading

Available

{{.Title}}

Add a book

Novel Rankings

{{.Title}}

{{.Title}}

Ranking Data

{{.Meta.Title}}

Chapters

{{.Title}}