kamil/libnovel
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 162 Wiki Activity

steps 6-8: wire HybridStore into orchestrator, server, and main

Browse Source 
- Add storage/hybrid.go: HybridStore composing PocketBase + MinIO backends
- Rewrite orchestrator to accept storage.Store instead of *writer.Writer
- Replace *writer.Writer with storage.Store in server.go and ui.go
- Wire audio cache, reading progress, chapter reads/writes through store
- Add rankingCacheAdapter in main.go to bridge context-free RankingPageCacher
  interface to HybridStore's context-aware methods
This commit is contained in:
Admin
2026-03-02 14:44:02 +05:00
parent 9add9033b9
commit 18e76c9668
5 changed files with 443 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
scraper/internal/server/server.go
View File

@@ -22,7 +22,7 @@ import (
"github.com/libnovel/scraper/internal/orchestrator"
"github.com/libnovel/scraper/internal/scraper"
"github.com/libnovel/scraper/internal/writer"
"github.com/libnovel/scraper/internal/storage"
)
// Server wraps an HTTP mux with the scraping endpoints.
@@ -31,7 +31,7 @@ type Server struct {
oCfg orchestrator.Config
novel scraper.NovelScraper
log *slog.Logger
writer *writer.Writer
store storage.Store
mu sync.Mutex
running bool
rankingRunning bool
@@ -42,26 +42,23 @@ type Server struct {
voiceMu sync.RWMutex
cachedVoices []string // populated on first request from Kokoro /v1/audio/voices
// audioMu guards audioCache and audioInFlight.
// audioCache maps a cache key to the Kokoro download filename returned by
// POST /v1/audio/speech with return_download_link=true.
// audioMu guards audioInFlight only.
// Completed audio filenames are persisted to the Store (PocketBase).
// audioInFlight deduplicates concurrent generation requests for the same key.
audioMu sync.Mutex
audioCache map[string]string // cacheKey → kokoro download filename
audioInFlight map[string]chan struct{} // cacheKey → closed when done
}
// New creates a new Server.
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, kokoroURL, kokoroVoice string) *Server {
func New(addr string, oCfg orchestrator.Config, novel scraper.NovelScraper, log *slog.Logger, store storage.Store, kokoroURL, kokoroVoice string) *Server {
return &Server{
addr: addr,
oCfg: oCfg,
novel: novel,
log: log,
writer: writer.New(oCfg.StaticRoot),
store: store,
kokoroURL: kokoroURL,
kokoroVoice: kokoroVoice,
audioCache: make(map[string]string),
audioInFlight: make(map[string]chan struct{}),
}
}
@@ -174,7 +171,7 @@ func (s *Server) handleChapterText(w http.ResponseWriter, r *http.Request) {
http.NotFound(w, r)
return
}
raw, err := s.writer.ReadChapter(slug, n)
raw, err := s.store.ReadChapter(r.Context(), slug, n)
if err != nil {
http.NotFound(w, r)
return
@@ -223,15 +220,14 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
// Fast path: already generated this session.
s.audioMu.Lock()
if filename, ok := s.audioCache[cacheKey]; ok {
s.audioMu.Unlock()
// Fast path: already generated (check persistent store first).
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
s.writeAudioResponse(w, slug, n, voice, speed, filename)
return
}
// Deduplicate concurrent generation for the same key.
s.audioMu.Lock()
if ch, ok := s.audioInFlight[cacheKey]; ok {
s.audioMu.Unlock()
select {
@@ -240,10 +236,8 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
http.Error(w, `{"error":"request cancelled"}`, http.StatusServiceUnavailable)
return
}
s.audioMu.Lock()
filename, ok := s.audioCache[cacheKey]
s.audioMu.Unlock()
if ok {
// Check store again after waiting.
if filename, ok := s.store.GetAudioCache(r.Context(), cacheKey); ok {
s.writeAudioResponse(w, slug, n, voice, speed, filename)
} else {
http.Error(w, `{"error":"audio generation failed"}`, http.StatusInternalServerError)
@@ -262,7 +256,7 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
}()
// Load and validate chapter text.
raw, err := s.writer.ReadChapter(slug, n)
raw, err := s.store.ReadChapter(r.Context(), slug, n)
if err != nil {
http.Error(w, `{"error":"chapter not found"}`, http.StatusNotFound)
return
@@ -287,9 +281,7 @@ func (s *Server) handleAudioGenerate(w http.ResponseWriter, r *http.Request) {
return
}
s.audioMu.Lock()
s.audioCache[cacheKey] = filename
s.audioMu.Unlock()
_ = s.store.SetAudioCache(r.Context(), cacheKey, filename)
s.log.Info("audio generated", "slug", slug, "chapter", n, "filename", filename)
s.writeAudioResponse(w, slug, n, voice, speed, filename)
@@ -378,10 +370,7 @@ func (s *Server) handleAudioProxy(w http.ResponseWriter, r *http.Request) {
}
cacheKey := fmt.Sprintf("%s/%d/%s/%.2f", slug, n, voice, speed)
s.audioMu.Lock()
filename, ok := s.audioCache[cacheKey]
s.audioMu.Unlock()
filename, ok := s.store.GetAudioCache(r.Context(), cacheKey)
if !ok {
http.Error(w, "audio not generated yet", http.StatusNotFound)
return
@@ -462,7 +451,7 @@ func (s *Server) runAsync(w http.ResponseWriter, cfg orchestrator.Config) {
ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
defer cancel()
o := orchestrator.New(cfg, s.novel, s.log)
o := orchestrator.New(cfg, s.novel, s.log, s.store)
if err := o.Run(ctx); err != nil {
s.log.Error("scrape job failed", "err", fmt.Sprintf("%v", err))
}

76
scraper/internal/server/ui.go
View File

@@ -15,7 +15,7 @@ import (
"github.com/libnovel/scraper/internal/orchestrator"
"github.com/libnovel/scraper/internal/scraper"
"github.com/libnovel/scraper/internal/writer"
"github.com/libnovel/scraper/internal/storage"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/extension"
goldhtml "github.com/yuin/goldmark/renderer/html"
@@ -629,7 +629,7 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) {
return
}
books, err := s.writer.ListBooks()
books, err := s.store.ListBooks(r.Context())
if err != nil {
http.Error(w, "failed to list books: "+err.Error(), http.StatusInternalServerError)
return
@@ -639,8 +639,8 @@ func (s *Server) handleHome(w http.ResponseWriter, r *http.Request) {
for i, b := range books {
items[i] = homeBookItem{
BookMeta: b,
Downloaded: s.writer.CountChapters(b.Slug),
AddedAt: s.writer.MetadataMtime(b.Slug),
Downloaded: s.store.CountChapters(r.Context(), b.Slug),
AddedAt: s.store.MetadataMtime(r.Context(), b.Slug),
}
}
@@ -877,7 +877,7 @@ const scrapeTmpl = `
</script>`
func (s *Server) handleScrape(w http.ResponseWriter, r *http.Request) {
rankingItems, _ := s.writer.ReadRankingItems()
rankingItems, _ := s.store.ReadRankingItems(r.Context())
rankingJSON, _ := json.Marshal(rankingItems)
t := template.Must(template.New("scrape").Parse(scrapeTmpl))
@@ -1324,12 +1324,12 @@ const rankingTmpl = `
// rankingViewItem enriches a RankingItem with whether it is present in the
// local book library, so the template can highlight it differently.
type rankingViewItem struct {
writer.RankingItem
storage.RankingItem
Local bool
}
// toRankingViewItems annotates items with Local=true for slugs found in localSlugs.
func toRankingViewItems(items []writer.RankingItem, localSlugs map[string]bool) []rankingViewItem {
func toRankingViewItems(items []storage.RankingItem, localSlugs map[string]bool) []rankingViewItem {
out := make([]rankingViewItem, len(items))
for i, it := range items {
out[i] = rankingViewItem{
@@ -1403,13 +1403,13 @@ const rankingPageSize = 20
// It does NOT trigger a live scrape; use POST /ranking/refresh for that.
// Supports ?page=N for browsing through cached items (20 per page).
func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) {
rankingItems, err := s.writer.ReadRankingItems()
rankingItems, err := s.store.ReadRankingItems(r.Context())
if err != nil {
s.log.Error("failed to read cached ranking", "err", err)
}
cachedAt := ""
if info, statErr := s.writer.RankingFileInfo(); statErr == nil {
if info, statErr := s.store.RankingFileInfo(r.Context()); statErr == nil && info != nil {
cachedAt = info.ModTime().Format("Jan 2, 2006 at 15:04")
}
@@ -1482,7 +1482,7 @@ func (s *Server) handleRanking(w http.ResponseWriter, r *http.Request) {
}
// Encode full dataset + local slugs for client-side cross-page filtering.
localSlugs := s.writer.LocalSlugs()
localSlugs, _ := s.store.LocalSlugs(r.Context())
type rankingJSONItem struct {
Rank int `json:"rank"`
Slug string `json:"slug"`
@@ -1577,14 +1577,14 @@ func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) {
rankingCh, errCh := s.novel.ScrapeRanking(ctx, maxPages)
var rankingItems []writer.RankingItem
var rankingItems []storage.RankingItem
for rankingCh != nil || errCh != nil {
select {
case meta, ok := <-rankingCh:
if !ok {
rankingCh = nil
} else {
rankingItems = append(rankingItems, writer.RankingItem{
rankingItems = append(rankingItems, storage.RankingItem{
Rank: meta.Ranking,
Slug: meta.Slug,
Title: meta.Title,
@@ -1605,7 +1605,7 @@ func (s *Server) handleRankingRefresh(w http.ResponseWriter, r *http.Request) {
}
if len(rankingItems) > 0 {
if err := s.writer.WriteRanking(rankingItems); err != nil {
if err := s.store.WriteRanking(ctx, rankingItems); err != nil {
s.log.Error("failed to save ranking", "err", err)
}
}
@@ -1669,7 +1669,7 @@ const rankingViewTmpl = `
</div>`
func (s *Server) handleRankingView(w http.ResponseWriter, r *http.Request) {
items, err := s.writer.ReadRankingItems()
items, err := s.store.ReadRankingItems(r.Context())
if err != nil {
http.Error(w, "failed to read ranking: "+err.Error(), http.StatusInternalServerError)
return
@@ -1927,7 +1927,7 @@ const bookTmpl = `
func (s *Server) handleBook(w http.ResponseWriter, r *http.Request) {
slug := r.PathValue("slug")
meta, ok, err := s.writer.ReadMetadata(slug)
meta, ok, err := s.store.ReadMetadata(r.Context(), slug)
if err != nil {
http.Error(w, "failed to read metadata: "+err.Error(), http.StatusInternalServerError)
return
@@ -1937,7 +1937,7 @@ func (s *Server) handleBook(w http.ResponseWriter, r *http.Request) {
return
}
chapters, err := s.writer.ListChapters(slug)
chapters, err := s.store.ListChapters(r.Context(), slug)
if err != nil {
http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError)
return
@@ -2054,7 +2054,7 @@ func (s *Server) handleBookChaptersPage(w http.ResponseWriter, r *http.Request)
}
}
chapters, err := s.writer.ListChapters(slug)
chapters, err := s.store.ListChapters(r.Context(), slug)
if err != nil {
http.Error(w, "failed to list chapters: "+err.Error(), http.StatusInternalServerError)
return
@@ -3035,7 +3035,7 @@ func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) {
return
}
raw, err := s.writer.ReadChapter(slug, n)
raw, err := s.store.ReadChapter(r.Context(), slug, n)
if err != nil {
http.NotFound(w, r)
return
@@ -3051,15 +3051,15 @@ func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) {
return
}
chapters, _ := s.writer.ListChapters(slug)
chapters, _ := s.store.ListChapters(r.Context(), slug)
prevN, nextN := adjacentChapters(chapters, n)
title := firstHeading(raw, fmt.Sprintf("Chapter %d", n))
chapterTitle, chapterDate := writer.SplitChapterTitle(title)
chapterTitle, chapterDate := splitChapterTitle(title)
// Load cover URL for Media Session artwork (best-effort; ignore errors).
var coverURL string
if meta, ok, err := s.writer.ReadMetadata(slug); err == nil && ok {
if meta, ok, err := s.store.ReadMetadata(r.Context(), slug); err == nil && ok {
coverURL = meta.Cover
}
@@ -3096,6 +3096,36 @@ func (s *Server) handleChapter(w http.ResponseWriter, r *http.Request) {
// ─── helpers ──────────────────────────────────────────────────────────────────
// splitChapterTitle splits a raw chapter heading into a human-readable title
// and a trailing relative-date string (e.g. "1 year ago"). It mirrors the
// same logic in internal/writer and internal/storage/hybrid.
func splitChapterTitle(raw string) (title, date string) {
raw = strings.TrimSpace(raw)
// Strip leading numeric index.
if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 {
prefix := raw[:idx]
allDigit := true
for _, c := range prefix {
if c < '0' || c > '9' {
allDigit = false
break
}
}
if allDigit {
raw = strings.TrimSpace(raw[idx:])
}
}
// Strip "Chapter N - N: " prefix.
chNumRe := regexp.MustCompile(`(?i)^chapter\s+\d+(?:\s*-\s*\d+)?\s*:\s*`)
raw = strings.TrimSpace(chNumRe.ReplaceAllString(raw, ""))
// Detect trailing relative date.
dateRe := regexp.MustCompile(`\s*(\d+\s+(?:second|minute|hour|day|week|month|year)s?\s+ago)\s*$`)
if m := dateRe.FindStringSubmatchIndex(raw); m != nil {
return strings.TrimSpace(raw[:m[0]]), strings.TrimSpace(raw[m[2]:m[3]])
}
return raw, ""
}
// sortedKeys returns the keys of a string-bool map in sorted order.
func sortedKeys(m map[string]bool) []string {
out := make([]string, 0, len(m))
@@ -3127,7 +3157,7 @@ func stripMarkdown(src string) string {
// adjacentChapters returns the chapter numbers immediately before and after n
// in the sorted chapters list. 0 means "does not exist".
func adjacentChapters(chapters []writer.ChapterInfo, n int) (prev, next int) {
func adjacentChapters(chapters []storage.ChapterInfo, n int) (prev, next int) {
for i, ch := range chapters {
if ch.Number == n {
if i > 0 {
@@ -3211,7 +3241,7 @@ func (s *Server) handleUIScrapeBook(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
defer cancel()
o := orchestrator.New(cfg, s.novel, s.log)
o := orchestrator.New(cfg, s.novel, s.log, s.store)
if err := o.Run(ctx); err != nil {
s.log.Error("UI scrape job failed", "url", bookURL, "err", err)
}