// Package writer handles persistence of scraped chapters and metadata. // // Directory layout: // // static/books/ // ├── {book-slug}/ // │ ├── metadata.yaml // │ ├── vol-0/ (no volume grouping) // │ │ ├── 1-50/ // │ │ │ ├── chapter-1.md // │ │ │ └── … // │ │ └── 51-100/ // │ │ └── … // │ └── vol-1/ // │ └── … package writer import ( "encoding/json" "fmt" "os" "path/filepath" "regexp" "sort" "strconv" "strings" "github.com/libnovel/scraper/internal/scraper" "gopkg.in/yaml.v3" ) const chaptersPerFolder = 50 // Writer persists scraped content under a configurable root directory. type Writer struct { root string // e.g. "./static/books" } // New creates a Writer that stores files under root. func New(root string) *Writer { return &Writer{root: root} } // ─── Metadata ───────────────────────────────────────────────────────────────── // WriteMetadata serialises meta to static/books/{slug}/metadata.yaml. // It creates the directory if it does not exist and overwrites any existing file. func (w *Writer) WriteMetadata(meta scraper.BookMeta) error { dir := w.bookDir(meta.Slug) if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("writer: mkdir %s: %w", dir, err) } path := filepath.Join(dir, "metadata.yaml") f, err := os.Create(path) if err != nil { return fmt.Errorf("writer: create metadata %s: %w", path, err) } defer f.Close() enc := yaml.NewEncoder(f) enc.SetIndent(2) if err := enc.Encode(meta); err != nil { return fmt.Errorf("writer: encode metadata: %w", err) } return enc.Close() } // ReadMetadata reads the metadata.yaml for slug if it exists. // Returns (zero-value, false, nil) when the file does not exist. func (w *Writer) ReadMetadata(slug string) (scraper.BookMeta, bool, error) { path := filepath.Join(w.bookDir(slug), "metadata.yaml") data, err := os.ReadFile(path) if err != nil { if os.IsNotExist(err) { return scraper.BookMeta{}, false, nil } return scraper.BookMeta{}, false, fmt.Errorf("writer: read metadata %s: %w", path, err) } var meta scraper.BookMeta if err := yaml.Unmarshal(data, &meta); err != nil { return scraper.BookMeta{}, true, fmt.Errorf("writer: unmarshal metadata %s: %w", path, err) } return meta, true, nil } // ─── Chapters ───────────────────────────────────────────────────────────────── // ChapterExists returns true if the markdown file for ref already exists on disk. func (w *Writer) ChapterExists(slug string, ref scraper.ChapterRef) bool { _, err := os.Stat(w.chapterPath(slug, ref)) return err == nil } // WriteChapter writes chapter.Text to the appropriate markdown file. // The parent directories are created on demand. func (w *Writer) WriteChapter(slug string, chapter scraper.Chapter) error { path := w.chapterPath(slug, chapter.Ref) dir := filepath.Dir(path) if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("writer: mkdir %s: %w", dir, err) } // Build the markdown document. var sb strings.Builder sb.WriteString("# ") sb.WriteString(chapter.Ref.Title) sb.WriteString("\n\n") sb.WriteString(chapter.Text) sb.WriteString("\n") if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil { return fmt.Errorf("writer: write chapter %s: %w", path, err) } return nil } // ─── Catalogue helpers ──────────────────────────────────────────────────────── // ListBooks returns metadata for every book that has a metadata.yaml under root. // Books with unreadable metadata files are silently skipped. func (w *Writer) ListBooks() ([]scraper.BookMeta, error) { entries, err := os.ReadDir(w.root) if err != nil { if os.IsNotExist(err) { return nil, nil } return nil, fmt.Errorf("writer: list books: %w", err) } var books []scraper.BookMeta for _, e := range entries { if !e.IsDir() { continue } meta, ok, _ := w.ReadMetadata(e.Name()) if !ok { continue } books = append(books, meta) } sort.Slice(books, func(i, j int) bool { return books[i].Title < books[j].Title }) return books, nil } // LocalSlugs returns the set of book slugs that have a metadata.yaml on disk. // It is cheaper than ListBooks because it only checks for file existence rather // than fully parsing every YAML file. func (w *Writer) LocalSlugs() map[string]bool { entries, err := os.ReadDir(w.root) if err != nil { return map[string]bool{} } slugs := make(map[string]bool, len(entries)) for _, e := range entries { if !e.IsDir() { continue } metaPath := filepath.Join(w.root, e.Name(), "metadata.yaml") if _, err := os.Stat(metaPath); err == nil { slugs[e.Name()] = true } } return slugs } // ChapterInfo is a lightweight chapter descriptor derived from on-disk files. type ChapterInfo struct { Number int Title string // chapter name, cleaned of number prefix and trailing date Date string // relative date scraped alongside the title, e.g. "1 year ago" } // ListChapters returns all chapters on disk for slug, sorted by number. func (w *Writer) ListChapters(slug string) ([]ChapterInfo, error) { bookDir := w.bookDir(slug) var chapters []ChapterInfo // Walk vol-*/range-*/ directories. volDirs, err := filepath.Glob(filepath.Join(bookDir, "vol-*")) if err != nil { return nil, fmt.Errorf("writer: list chapters glob: %w", err) } for _, vd := range volDirs { rangeDirs, _ := filepath.Glob(filepath.Join(vd, "*-*")) for _, rd := range rangeDirs { files, _ := filepath.Glob(filepath.Join(rd, "chapter-*.md")) for _, f := range files { base := filepath.Base(f) // chapter-N.md numStr := strings.TrimSuffix(strings.TrimPrefix(base, "chapter-"), ".md") n, err := strconv.Atoi(numStr) if err != nil { continue } title, date := chapterTitle(f, n) chapters = append(chapters, ChapterInfo{Number: n, Title: title, Date: date}) } } } sort.Slice(chapters, func(i, j int) bool { return chapters[i].Number < chapters[j].Number }) return chapters, nil } // chapterTitle reads the first non-empty line of a markdown file and strips // the leading "# " heading marker. Falls back to "Chapter N". func chapterTitle(path string, n int) (title, date string) { data, err := os.ReadFile(path) if err != nil { return fmt.Sprintf("Chapter %d", n), "" } for _, line := range strings.SplitN(string(data), "\n", 10) { line = strings.TrimSpace(line) if line == "" { continue } line = strings.TrimPrefix(line, "# ") return splitChapterTitle(line) } return fmt.Sprintf("Chapter %d", n), "" } // splitChapterTitle separates the human-readable chapter name from the // trailing relative-date string that novelfire.net appends to the heading. // Examples of raw heading text (after stripping "# "): // // "1 Chapter 1 - 1: The Academy's Weakest1 year ago" // "2 Chapter 2 - Enter the Storm3 months ago" // // The pattern is: optional leading number+whitespace, then the real title, // then a date that matches /\d+\s+(second|minute|hour|day|week|month|year)s?\s+ago$/ func splitChapterTitle(raw string) (title, date string) { // Strip a leading chapter-number index that novelfire sometimes prepends. // It looks like "1 " or "12 " at the very start. raw = strings.TrimSpace(raw) if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 { prefix := raw[:idx] allDigit := true for _, c := range prefix { if c < '0' || c > '9' { allDigit = false break } } if allDigit { raw = strings.TrimSpace(raw[idx:]) } } // Match a trailing relative date: " [s] ago" dateRe := regexp.MustCompile(`\s*(\d+\s+(?:second|minute|hour|day|week|month|year)s?\s+ago)\s*$`) if m := dateRe.FindStringSubmatchIndex(raw); m != nil { return strings.TrimSpace(raw[:m[0]]), strings.TrimSpace(raw[m[2]:m[3]]) } return raw, "" } // ReadChapter returns the raw markdown content for chapter number n of slug. func (w *Writer) ReadChapter(slug string, n int) (string, error) { // Reconstruct path using the same bucketing formula as chapterPath. ref := scraper.ChapterRef{Number: n, Volume: 0} path := w.chapterPath(slug, ref) data, err := os.ReadFile(path) if err != nil { return "", fmt.Errorf("writer: read chapter %d: %w", n, err) } return string(data), nil } // ─── Ranking ───────────────────────────────────────────────────────────────── // RankingItem represents a single entry in the ranking. type RankingItem struct { Rank int `yaml:"rank" json:"rank"` Slug string `yaml:"slug" json:"slug"` Title string `yaml:"title" json:"title"` Author string `yaml:"author,omitempty" json:"author,omitempty"` Cover string `yaml:"cover,omitempty" json:"cover,omitempty"` Status string `yaml:"status,omitempty" json:"status,omitempty"` Genres []string `yaml:"genres,omitempty" json:"genres,omitempty"` SourceURL string `yaml:"source_url,omitempty" json:"source_url,omitempty"` } // WriteRanking saves the ranking items as JSON to static/books/ranking.json. // This replaces the old markdown table format with a structured format that // is faster to read back (no custom parsing) and safe for titles containing "|". func (w *Writer) WriteRanking(items []RankingItem) error { path := filepath.Clean(w.rankingPath()) dir := filepath.Dir(path) if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("writer: mkdir %s: %w", dir, err) } data, err := json.MarshalIndent(items, "", " ") if err != nil { return fmt.Errorf("writer: marshal ranking: %w", err) } if err := os.WriteFile(path, data, 0o644); err != nil { return fmt.Errorf("writer: write ranking %s: %w", path, err) } return nil } // ReadRankingItems parses ranking.json into a slice of RankingItem. // Returns nil slice (not an error) when the file does not exist yet. func (w *Writer) ReadRankingItems() ([]RankingItem, error) { data, err := os.ReadFile(w.rankingPath()) if err != nil { if os.IsNotExist(err) { return nil, nil } return nil, fmt.Errorf("writer: read ranking: %w", err) } var items []RankingItem if err := json.Unmarshal(data, &items); err != nil { return nil, fmt.Errorf("writer: parse ranking json: %w", err) } return items, nil } // RankingFileInfo returns os.FileInfo for the ranking.json file, if it exists. func (w *Writer) RankingFileInfo() (os.FileInfo, error) { return os.Stat(w.rankingPath()) } func (w *Writer) rankingPath() string { return filepath.Join(w.root, "ranking.json") } // ─── Ranking page HTML cache ────────────────────────────────────────────────── // rankingCacheDir returns the directory that stores per-page HTML caches. func (w *Writer) rankingCacheDir() string { return filepath.Join(w.root, "_ranking_cache") } // rankingPageCachePath returns the path for a cached ranking page HTML file. func (w *Writer) rankingPageCachePath(page int) string { return filepath.Join(w.rankingCacheDir(), fmt.Sprintf("page-%d.html", page)) } // WriteRankingPageCache persists raw HTML for the given ranking page number. func (w *Writer) WriteRankingPageCache(page int, html string) error { dir := w.rankingCacheDir() if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("writer: mkdir ranking cache %s: %w", dir, err) } path := w.rankingPageCachePath(page) if err := os.WriteFile(path, []byte(html), 0o644); err != nil { return fmt.Errorf("writer: write ranking page cache %s: %w", path, err) } return nil } // ReadRankingPageCache reads the cached HTML for the given ranking page. // Returns ("", nil) when no cache file exists yet. func (w *Writer) ReadRankingPageCache(page int) (string, error) { data, err := os.ReadFile(w.rankingPageCachePath(page)) if err != nil { if os.IsNotExist(err) { return "", nil } return "", fmt.Errorf("writer: read ranking page cache page %d: %w", page, err) } return string(data), nil } // RankingPageCacheInfo returns os.FileInfo for a cached ranking page file. // Returns (nil, nil) when the file does not exist. func (w *Writer) RankingPageCacheInfo(page int) (os.FileInfo, error) { info, err := os.Stat(w.rankingPageCachePath(page)) if err != nil { if os.IsNotExist(err) { return nil, nil } return nil, err } return info, nil } // bookDir returns the root directory for a book slug. func (w *Writer) bookDir(slug string) string { return filepath.Join(w.root, slug) } // AudioDir returns the directory used to cache generated MP3 files for a book. func (w *Writer) AudioDir(slug string) string { return filepath.Join(w.bookDir(slug), "audio") } // AudioPath returns the full path for a cached chapter audio file. // The filename is keyed by chapter number, voice, and speed so that different // settings never collide. Speed is formatted to one decimal place (e.g. "1.0"). func (w *Writer) AudioPath(slug string, n int, voice string, speed float64) string { safeVoice := sanitiseVoice(voice) filename := fmt.Sprintf("ch%d-%s-%.1f.mp3", n, safeVoice, speed) return filepath.Join(w.AudioDir(slug), filename) } // AudioPartPath returns the path for an individual audio chunk generated during // chunked TTS. Part files are named ch{n}-{voice}-{speed}.part{p}.mp3 and are // deleted after they have been merged into the final AudioPath file. func (w *Writer) AudioPartPath(slug string, n int, voice string, speed float64, part int) string { safeVoice := sanitiseVoice(voice) filename := fmt.Sprintf("ch%d-%s-%.1f.part%d.mp3", n, safeVoice, speed, part) return filepath.Join(w.AudioDir(slug), filename) } // sanitiseVoice converts a voice name into a string that is safe to embed in a // filename (only a-z, A-Z, 0-9, '_', '-' are kept; everything else becomes '_'). func sanitiseVoice(voice string) string { return strings.Map(func(r rune) rune { if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' || r == '-' { return r } return '_' }, voice) } // chapterPath computes the full file path for a chapter. // // vol-{volume}/{folderRange}/chapter-{number}.md // // Example: vol-0/1-50/chapter-1.md, vol-0/51-100/chapter-51.md func (w *Writer) chapterPath(slug string, ref scraper.ChapterRef) string { vol := ref.Volume // 0 == no volume grouping volDir := fmt.Sprintf("vol-%d", vol) // Folder group: chapters 1-50 → "1-50", 51-100 → "51-100", … lo := ((ref.Number-1)/chaptersPerFolder)*chaptersPerFolder + 1 hi := lo + chaptersPerFolder - 1 rangeDir := fmt.Sprintf("%d-%d", lo, hi) filename := fmt.Sprintf("chapter-%d.md", ref.Number) return filepath.Join(w.bookDir(slug), volDir, rangeDir, filename) }