libnovel/scraper/internal/writer/writer.go

// Package writer handles persistence of scraped chapters and metadata.
//
// Directory layout:
//
//	static/books/
//	├── {book-slug}/
//	│   ├── metadata.yaml
//	│   ├── vol-0/          (no volume grouping)
//	│   │   ├── 1-50/
//	│   │   │   ├── chapter-1.md
//	│   │   │   └── …
//	│   │   └── 51-100/
//	│   │       └── …
//	│   └── vol-1/
//	│       └── …
package writer

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/libnovel/scraper/internal/scraper"
	"gopkg.in/yaml.v3"
)

const chaptersPerFolder = 50

// Writer persists scraped content under a configurable root directory.
type Writer struct {
	root string // e.g. "./static/books"
}

// New creates a Writer that stores files under root.
func New(root string) *Writer {
	return &Writer{root: root}
}

// ─── Metadata ─────────────────────────────────────────────────────────────────

// WriteMetadata serialises meta to static/books/{slug}/metadata.yaml.
// It creates the directory if it does not exist and overwrites any existing file.
func (w *Writer) WriteMetadata(meta scraper.BookMeta) error {
	dir := w.bookDir(meta.Slug)
	if err := os.MkdirAll(dir, 0o755); err != nil {
		return fmt.Errorf("writer: mkdir %s: %w", dir, err)
	}

	path := filepath.Join(dir, "metadata.yaml")
	f, err := os.Create(path)
	if err != nil {
		return fmt.Errorf("writer: create metadata %s: %w", path, err)
	}
	defer f.Close()

	enc := yaml.NewEncoder(f)
	enc.SetIndent(2)
	if err := enc.Encode(meta); err != nil {
		return fmt.Errorf("writer: encode metadata: %w", err)
	}
	return enc.Close()
}

// ReadMetadata reads the metadata.yaml for slug if it exists.
// Returns (zero-value, false, nil) when the file does not exist.
func (w *Writer) ReadMetadata(slug string) (scraper.BookMeta, bool, error) {
	path := filepath.Join(w.bookDir(slug), "metadata.yaml")
	data, err := os.ReadFile(path)
	if err != nil {
		if os.IsNotExist(err) {
			return scraper.BookMeta{}, false, nil
		}
		return scraper.BookMeta{}, false, fmt.Errorf("writer: read metadata %s: %w", path, err)
	}

	var meta scraper.BookMeta
	if err := yaml.Unmarshal(data, &meta); err != nil {
		return scraper.BookMeta{}, true, fmt.Errorf("writer: unmarshal metadata %s: %w", path, err)
	}
	return meta, true, nil
}

// MetadataMtime returns the modification time (Unix seconds) of the
// metadata.yaml file for slug, or 0 if the file cannot be stat'd.
func (w *Writer) MetadataMtime(slug string) int64 {
	path := filepath.Join(w.bookDir(slug), "metadata.yaml")
	fi, err := os.Stat(path)
	if err != nil {
		return 0
	}
	return fi.ModTime().Unix()
}

// ─── Chapters ─────────────────────────────────────────────────────────────────

// ChapterExists returns true if the markdown file for ref already exists on disk.
func (w *Writer) ChapterExists(slug string, ref scraper.ChapterRef) bool {
	_, err := os.Stat(w.chapterPath(slug, ref))
	return err == nil
}

// WriteChapter writes chapter.Text to the appropriate markdown file.
// The parent directories are created on demand.
func (w *Writer) WriteChapter(slug string, chapter scraper.Chapter) error {
	path := w.chapterPath(slug, chapter.Ref)
	dir := filepath.Dir(path)

	if err := os.MkdirAll(dir, 0o755); err != nil {
		return fmt.Errorf("writer: mkdir %s: %w", dir, err)
	}

	// Build the markdown document.
	var sb strings.Builder
	sb.WriteString("# ")
	sb.WriteString(chapter.Ref.Title)
	sb.WriteString("\n\n")
	sb.WriteString(chapter.Text)
	sb.WriteString("\n")

	if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil {
		return fmt.Errorf("writer: write chapter %s: %w", path, err)
	}
	return nil
}

// ─── Catalogue helpers ────────────────────────────────────────────────────────

// ListBooks returns metadata for every book that has a metadata.yaml under root.
// Books with unreadable metadata files are silently skipped.
func (w *Writer) ListBooks() ([]scraper.BookMeta, error) {
	entries, err := os.ReadDir(w.root)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, fmt.Errorf("writer: list books: %w", err)
	}
	var books []scraper.BookMeta
	for _, e := range entries {
		if !e.IsDir() {
			continue
		}
		meta, ok, _ := w.ReadMetadata(e.Name())
		if !ok {
			continue
		}
		books = append(books, meta)
	}
	sort.Slice(books, func(i, j int) bool {
		return books[i].Title < books[j].Title
	})
	return books, nil
}

// LocalSlugs returns the set of book slugs that have a metadata.yaml on disk.
// It is cheaper than ListBooks because it only checks for file existence rather
// than fully parsing every YAML file.
func (w *Writer) LocalSlugs() map[string]bool {
	entries, err := os.ReadDir(w.root)
	if err != nil {
		return map[string]bool{}
	}
	slugs := make(map[string]bool, len(entries))
	for _, e := range entries {
		if !e.IsDir() {
			continue
		}
		metaPath := filepath.Join(w.root, e.Name(), "metadata.yaml")
		if _, err := os.Stat(metaPath); err == nil {
			slugs[e.Name()] = true
		}
	}
	return slugs
}

// ChapterInfo is a lightweight chapter descriptor derived from on-disk files.
type ChapterInfo struct {
	Number int
	Title  string // chapter name, cleaned of number prefix and trailing date
	Date   string // relative date scraped alongside the title, e.g. "1 year ago"
}

// ListChapters returns all chapters on disk for slug, sorted by number.
func (w *Writer) ListChapters(slug string) ([]ChapterInfo, error) {
	bookDir := w.bookDir(slug)
	var chapters []ChapterInfo

	// Walk vol-*/range-*/ directories.
	volDirs, err := filepath.Glob(filepath.Join(bookDir, "vol-*"))
	if err != nil {
		return nil, fmt.Errorf("writer: list chapters glob: %w", err)
	}
	for _, vd := range volDirs {
		rangeDirs, _ := filepath.Glob(filepath.Join(vd, "*-*"))
		for _, rd := range rangeDirs {
			files, _ := filepath.Glob(filepath.Join(rd, "chapter-*.md"))
			for _, f := range files {
				base := filepath.Base(f) // chapter-N.md
				numStr := strings.TrimSuffix(strings.TrimPrefix(base, "chapter-"), ".md")
				n, err := strconv.Atoi(numStr)
				if err != nil {
					continue
				}
				title, date := chapterTitle(f, n)
				chapters = append(chapters, ChapterInfo{Number: n, Title: title, Date: date})
			}
		}
	}
	sort.Slice(chapters, func(i, j int) bool {
		return chapters[i].Number < chapters[j].Number
	})
	return chapters, nil
}

// CountChapters returns the number of chapter markdown files on disk for slug.
// It is cheaper than ListChapters because it does not read file contents.
func (w *Writer) CountChapters(slug string) int {
	bookDir := w.bookDir(slug)
	volDirs, err := filepath.Glob(filepath.Join(bookDir, "vol-*"))
	if err != nil {
		return 0
	}
	count := 0
	for _, vd := range volDirs {
		rangeDirs, _ := filepath.Glob(filepath.Join(vd, "*-*"))
		for _, rd := range rangeDirs {
			files, _ := filepath.Glob(filepath.Join(rd, "chapter-*.md"))
			count += len(files)
		}
	}
	return count
}

// chapterTitle reads the first non-empty line of a markdown file and strips
// the leading "# " heading marker. Falls back to "Chapter N".
func chapterTitle(path string, n int) (title, date string) {
	data, err := os.ReadFile(path)
	if err != nil {
		return fmt.Sprintf("Chapter %d", n), ""
	}
	for _, line := range strings.SplitN(string(data), "\n", 10) {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}
		line = strings.TrimPrefix(line, "# ")
		return SplitChapterTitle(line)
	}
	return fmt.Sprintf("Chapter %d", n), ""
}

// SplitChapterTitle separates the human-readable chapter name from the
// trailing relative-date string that novelfire.net appends to the heading.
// Examples of raw heading text (after stripping "# "):
//
//	"1                                    Chapter 1 - 1: The Academy's Weakest1 year ago"
//	"2  Chapter 2 - Enter the Storm3 months ago"
//
// The pattern is: optional leading number+whitespace, then the real title,
// then a date that matches /\d+\s+(second|minute|hour|day|week|month|year)s?\s+ago$/
func SplitChapterTitle(raw string) (title, date string) {
	// Strip a leading chapter-number index that novelfire sometimes prepends.
	// It looks like "1   " or "12  " at the very start.
	raw = strings.TrimSpace(raw)
	if idx := strings.IndexFunc(raw, func(r rune) bool { return r == ' ' || r == '\t' }); idx > 0 {
		prefix := raw[:idx]
		allDigit := true
		for _, c := range prefix {
			if c < '0' || c > '9' {
				allDigit = false
				break
			}
		}
		if allDigit {
			raw = strings.TrimSpace(raw[idx:])
		}
	}

	// Match a trailing relative date: "<n> <unit>[s] ago"
	dateRe := regexp.MustCompile(`\s*(\d+\s+(?:second|minute|hour|day|week|month|year)s?\s+ago)\s*$`)
	if m := dateRe.FindStringSubmatchIndex(raw); m != nil {
		return strings.TrimSpace(raw[:m[0]]), strings.TrimSpace(raw[m[2]:m[3]])
	}
	return raw, ""
}

// ReadChapter returns the raw markdown content for chapter number n of slug.
func (w *Writer) ReadChapter(slug string, n int) (string, error) {
	// Reconstruct path using the same bucketing formula as chapterPath.
	ref := scraper.ChapterRef{Number: n, Volume: 0}
	path := w.chapterPath(slug, ref)
	data, err := os.ReadFile(path)
	if err != nil {
		return "", fmt.Errorf("writer: read chapter %d: %w", n, err)
	}
	return string(data), nil
}

// ─── Ranking ─────────────────────────────────────────────────────────────────

// RankingItem represents a single entry in the ranking.
type RankingItem struct {
	Rank      int      `yaml:"rank"       json:"rank"`
	Slug      string   `yaml:"slug"       json:"slug"`
	Title     string   `yaml:"title"      json:"title"`
	Author    string   `yaml:"author,omitempty"     json:"author,omitempty"`
	Cover     string   `yaml:"cover,omitempty"      json:"cover,omitempty"`
	Status    string   `yaml:"status,omitempty"     json:"status,omitempty"`
	Genres    []string `yaml:"genres,omitempty"     json:"genres,omitempty"`
	SourceURL string   `yaml:"source_url,omitempty" json:"source_url,omitempty"`
}

// WriteRanking saves the ranking items as JSON to static/books/ranking.json.
// This replaces the old markdown table format with a structured format that
// is faster to read back (no custom parsing) and safe for titles containing "|".
func (w *Writer) WriteRanking(items []RankingItem) error {
	path := filepath.Clean(w.rankingPath())
	dir := filepath.Dir(path)
	if err := os.MkdirAll(dir, 0o755); err != nil {
		return fmt.Errorf("writer: mkdir %s: %w", dir, err)
	}

	data, err := json.MarshalIndent(items, "", "  ")
	if err != nil {
		return fmt.Errorf("writer: marshal ranking: %w", err)
	}
	if err := os.WriteFile(path, data, 0o644); err != nil {
		return fmt.Errorf("writer: write ranking %s: %w", path, err)
	}
	return nil
}

// ReadRankingItems parses ranking.json into a slice of RankingItem.
// Returns nil slice (not an error) when the file does not exist yet.
func (w *Writer) ReadRankingItems() ([]RankingItem, error) {
	data, err := os.ReadFile(w.rankingPath())
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, fmt.Errorf("writer: read ranking: %w", err)
	}
	var items []RankingItem
	if err := json.Unmarshal(data, &items); err != nil {
		return nil, fmt.Errorf("writer: parse ranking json: %w", err)
	}
	return items, nil
}

// RankingFileInfo returns os.FileInfo for the ranking.json file, if it exists.
func (w *Writer) RankingFileInfo() (os.FileInfo, error) {
	return os.Stat(w.rankingPath())
}

func (w *Writer) rankingPath() string {
	return filepath.Join(w.root, "ranking.json")
}

// ─── Ranking page HTML cache ──────────────────────────────────────────────────

// rankingCacheDir returns the directory that stores per-page HTML caches.
func (w *Writer) rankingCacheDir() string {
	return filepath.Join(w.root, "_ranking_cache")
}

// rankingPageCachePath returns the path for a cached ranking page HTML file.
func (w *Writer) rankingPageCachePath(page int) string {
	return filepath.Join(w.rankingCacheDir(), fmt.Sprintf("page-%d.html", page))
}

// WriteRankingPageCache persists raw HTML for the given ranking page number.
func (w *Writer) WriteRankingPageCache(page int, html string) error {
	dir := w.rankingCacheDir()
	if err := os.MkdirAll(dir, 0o755); err != nil {
		return fmt.Errorf("writer: mkdir ranking cache %s: %w", dir, err)
	}
	path := w.rankingPageCachePath(page)
	if err := os.WriteFile(path, []byte(html), 0o644); err != nil {
		return fmt.Errorf("writer: write ranking page cache %s: %w", path, err)
	}
	return nil
}

// ReadRankingPageCache reads the cached HTML for the given ranking page.
// Returns ("", nil) when no cache file exists yet.
func (w *Writer) ReadRankingPageCache(page int) (string, error) {
	data, err := os.ReadFile(w.rankingPageCachePath(page))
	if err != nil {
		if os.IsNotExist(err) {
			return "", nil
		}
		return "", fmt.Errorf("writer: read ranking page cache page %d: %w", page, err)
	}
	return string(data), nil
}

// RankingPageCacheInfo returns os.FileInfo for a cached ranking page file.
// Returns (nil, nil) when the file does not exist.
func (w *Writer) RankingPageCacheInfo(page int) (os.FileInfo, error) {
	info, err := os.Stat(w.rankingPageCachePath(page))
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, err
	}
	return info, nil
}

// bookDir returns the root directory for a book slug.
func (w *Writer) bookDir(slug string) string {
	return filepath.Join(w.root, slug)
}

// AudioDir returns the directory used to cache generated MP3 files for a book.
func (w *Writer) AudioDir(slug string) string {
	return filepath.Join(w.bookDir(slug), "audio")
}

// AudioPath returns the full path for a cached chapter audio file.
// The filename is keyed by chapter number, voice, and speed so that different
// settings never collide.  Speed is formatted to one decimal place (e.g. "1.0").
func (w *Writer) AudioPath(slug string, n int, voice string, speed float64) string {
	safeVoice := sanitiseVoice(voice)
	filename := fmt.Sprintf("ch%d-%s-%.1f.mp3", n, safeVoice, speed)
	return filepath.Join(w.AudioDir(slug), filename)
}

// AudioPartPath returns the path for an individual audio chunk generated during
// chunked TTS.  Part files are named ch{n}-{voice}-{speed}.part{p}.mp3 and are
// deleted after they have been merged into the final AudioPath file.
func (w *Writer) AudioPartPath(slug string, n int, voice string, speed float64, part int) string {
	safeVoice := sanitiseVoice(voice)
	filename := fmt.Sprintf("ch%d-%s-%.1f.part%d.mp3", n, safeVoice, speed, part)
	return filepath.Join(w.AudioDir(slug), filename)
}

// sanitiseVoice converts a voice name into a string that is safe to embed in a
// filename (only a-z, A-Z, 0-9, '_', '-' are kept; everything else becomes '_').
func sanitiseVoice(voice string) string {
	return strings.Map(func(r rune) rune {
		if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' || r == '-' {
			return r
		}
		return '_'
	}, voice)
}

// chapterPath computes the full file path for a chapter.
//
//	vol-{volume}/{folderRange}/chapter-{number}.md
//
// Example: vol-0/1-50/chapter-1.md, vol-0/51-100/chapter-51.md
func (w *Writer) chapterPath(slug string, ref scraper.ChapterRef) string {
	vol := ref.Volume // 0 == no volume grouping
	volDir := fmt.Sprintf("vol-%d", vol)

	// Folder group: chapters 1-50 → "1-50", 51-100 → "51-100", …
	lo := ((ref.Number-1)/chaptersPerFolder)*chaptersPerFolder + 1
	hi := lo + chaptersPerFolder - 1
	rangeDir := fmt.Sprintf("%d-%d", lo, hi)

	filename := fmt.Sprintf("chapter-%d.md", ref.Number)

	return filepath.Join(w.bookDir(slug), volDir, rangeDir, filename)
}