From 0aba23de1f92503c4dc11d0f2007d604b1925bb6 Mon Sep 17 00:00:00 2001 From: Admin Date: Sun, 1 Mar 2026 21:44:39 +0500 Subject: [PATCH] feat: switch ranking storage from markdown to JSON, fix round-trip test - WriteRanking now writes ranking.json via json.MarshalIndent (replaces markdown table) - ReadRankingItems uses json.Unmarshal (no fragile pipe-split parsing) - ReadRanking() (raw string reader) removed; handleRankingView uses ReadRankingItems - handleRankingView renders a 
 JSON block instead of goldmark markdown
- RankingItem gains json: struct tags; slug and genres now survive the round-trip exactly
- TestWriteRanking_RoundTrip updated: checks ranking.json, verifies Slug/Genres/SourceURL
- Add RankingPageCacher interface and per-page HTML disk cache support in scraper
---
 scraper/internal/novelfire/ranking_test.go |  19 +++-
 scraper/internal/server/ui.go              |  16 ++-
 scraper/internal/writer/writer.go          | 126 +++++----------------
 3 files changed, 51 insertions(+), 110 deletions(-)

diff --git a/scraper/internal/novelfire/ranking_test.go b/scraper/internal/novelfire/ranking_test.go
index 9b02b38..df1a2a0 100644
--- a/scraper/internal/novelfire/ranking_test.go
+++ b/scraper/internal/novelfire/ranking_test.go
@@ -164,9 +164,9 @@ func TestWriteRanking_RoundTrip(t *testing.T) {
 		t.Fatalf("WriteRanking failed: %v", err)
 	}
 
-	rankingFile := filepath.Join(dir, "ranking.md")
+	rankingFile := filepath.Join(dir, "ranking.json")
 	if _, err := os.Stat(rankingFile); err != nil {
-		t.Fatalf("ranking.md not created: %v", err)
+		t.Fatalf("ranking.json not created: %v", err)
 	}
 
 	got, err := w.ReadRankingItems()
@@ -180,12 +180,27 @@ func TestWriteRanking_RoundTrip(t *testing.T) {
 		if got[i].Rank != want.Rank {
 			t.Errorf("item[%d].Rank = %d, want %d", i, got[i].Rank, want.Rank)
 		}
+		if got[i].Slug != want.Slug {
+			t.Errorf("item[%d].Slug = %q, want %q", i, got[i].Slug, want.Slug)
+		}
 		if got[i].Title != want.Title {
 			t.Errorf("item[%d].Title = %q, want %q", i, got[i].Title, want.Title)
 		}
 		if got[i].Status != want.Status {
 			t.Errorf("item[%d].Status = %q, want %q", i, got[i].Status, want.Status)
 		}
+		if len(got[i].Genres) != len(want.Genres) {
+			t.Errorf("item[%d].Genres len = %d, want %d", i, len(got[i].Genres), len(want.Genres))
+		} else {
+			for j, g := range want.Genres {
+				if got[i].Genres[j] != g {
+					t.Errorf("item[%d].Genres[%d] = %q, want %q", i, j, got[i].Genres[j], g)
+				}
+			}
+		}
+		if got[i].SourceURL != want.SourceURL {
+			t.Errorf("item[%d].SourceURL = %q, want %q", i, got[i].SourceURL, want.SourceURL)
+		}
 	}
 }
 
diff --git a/scraper/internal/server/ui.go b/scraper/internal/server/ui.go
index d16dc0e..9127a63 100644
--- a/scraper/internal/server/ui.go
+++ b/scraper/internal/server/ui.go
@@ -933,31 +933,29 @@ const rankingViewTmpl = `
 
   
Ranking Data

 
-  

-    {{.HTML}}
-  

+  
{{.JSON}}

 `
 
 func (s *Server) handleRankingView(w http.ResponseWriter, r *http.Request) {
-	markdown, err := s.writer.ReadRanking()
+	items, err := s.writer.ReadRankingItems()
 	if err != nil {
 		http.Error(w, "failed to read ranking: "+err.Error(), http.StatusInternalServerError)
 		return
 	}
-	if markdown == "" {
+	if len(items) == 0 {
 		http.NotFound(w, r)
 		return
 	}
 
-	var htmlBuf bytes.Buffer
-	if err := md.Convert([]byte(markdown), &htmlBuf); err != nil {
-		http.Error(w, "markdown render error: "+err.Error(), http.StatusInternalServerError)
+	pretty, err := json.MarshalIndent(items, "", "  ")
+	if err != nil {
+		http.Error(w, "json marshal error: "+err.Error(), http.StatusInternalServerError)
 		return
 	}
 
 	t := template.Must(template.New("rankingView").Parse(rankingViewTmpl))
 	var buf bytes.Buffer
-	_ = t.Execute(&buf, struct{ HTML template.HTML }{HTML: template.HTML(htmlBuf.String())})
+	_ = t.Execute(&buf, struct{ JSON string }{JSON: string(pretty)})
 
 	s.respond(w, r, "Ranking Data", buf.String())
 }
diff --git a/scraper/internal/writer/writer.go b/scraper/internal/writer/writer.go
index ca368c8..bf49b14 100644
--- a/scraper/internal/writer/writer.go
+++ b/scraper/internal/writer/writer.go
@@ -16,6 +16,7 @@
 package writer
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -274,17 +275,19 @@ func (w *Writer) ReadChapter(slug string, n int) (string, error) {
 
 // RankingItem represents a single entry in the ranking.
 type RankingItem struct {
-	Rank      int      `yaml:"rank"`
-	Slug      string   `yaml:"slug"`
-	Title     string   `yaml:"title"`
-	Author    string   `yaml:"author,omitempty"`
-	Cover     string   `yaml:"cover,omitempty"`
-	Status    string   `yaml:"status,omitempty"`
-	Genres    []string `yaml:"genres,omitempty"`
-	SourceURL string   `yaml:"source_url,omitempty"`
+	Rank      int      `yaml:"rank"       json:"rank"`
+	Slug      string   `yaml:"slug"       json:"slug"`
+	Title     string   `yaml:"title"      json:"title"`
+	Author    string   `yaml:"author,omitempty"     json:"author,omitempty"`
+	Cover     string   `yaml:"cover,omitempty"      json:"cover,omitempty"`
+	Status    string   `yaml:"status,omitempty"     json:"status,omitempty"`
+	Genres    []string `yaml:"genres,omitempty"     json:"genres,omitempty"`
+	SourceURL string   `yaml:"source_url,omitempty" json:"source_url,omitempty"`
 }
 
-// WriteRanking saves the ranking items as markdown to static/ranking.md.
+// WriteRanking saves the ranking items as JSON to static/books/ranking.json.
+// This replaces the old markdown table format with a structured format that
+// is faster to read back (no custom parsing) and safe for titles containing "|".
 func (w *Writer) WriteRanking(items []RankingItem) error {
 	path := filepath.Clean(w.rankingPath())
 	dir := filepath.Dir(path)
@@ -292,115 +295,40 @@ func (w *Writer) WriteRanking(items []RankingItem) error {
 		return fmt.Errorf("writer: mkdir %s: %w", dir, err)
 	}
 
-	var sb strings.Builder
-	sb.WriteString("# Novel Rankings\n\n")
-	sb.WriteString("| Rank | Title | Cover | Status | Genres | URL |\n")
-	sb.WriteString("|------|-------|-------|--------|--------|-----|\n")
-	for _, item := range items {
-		genres := strings.Join(item.Genres, ", ")
-		if genres == "" {
-			genres = "-"
-		}
-		sb.WriteString(fmt.Sprintf("| %d | %s | %s | %s | %s | %s |\n",
-			item.Rank, item.Title, item.Cover, item.Status, genres, item.SourceURL))
+	data, err := json.MarshalIndent(items, "", "  ")
+	if err != nil {
+		return fmt.Errorf("writer: marshal ranking: %w", err)
 	}
-
-	if err := os.WriteFile(path, []byte(sb.String()), 0o644); err != nil {
+	if err := os.WriteFile(path, data, 0o644); err != nil {
 		return fmt.Errorf("writer: write ranking %s: %w", path, err)
 	}
 	return nil
 }
 
-// ReadRanking reads the ranking.md file if it exists.
-func (w *Writer) ReadRanking() (string, error) {
-	path := w.rankingPath()
-	data, err := os.ReadFile(path)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return "", nil
-		}
-		return "", fmt.Errorf("writer: read ranking: %w", err)
-	}
-	return string(data), nil
-}
-
-// ReadRankingItems parses ranking.md back into a slice of RankingItem.
+// ReadRankingItems parses ranking.json into a slice of RankingItem.
 // Returns nil slice (not an error) when the file does not exist yet.
 func (w *Writer) ReadRankingItems() ([]RankingItem, error) {
-	markdown, err := w.ReadRanking()
-	if err != nil || markdown == "" {
-		return nil, err
+	data, err := os.ReadFile(w.rankingPath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("writer: read ranking: %w", err)
 	}
-
 	var items []RankingItem
-	for _, line := range strings.Split(markdown, "\n") {
-		// Only process data rows: start and end with '|', not header/separator rows.
-		line = strings.TrimSpace(line)
-		if !strings.HasPrefix(line, "|") || !strings.HasSuffix(line, "|") {
-			continue
-		}
-		// Strip leading/trailing '|' and split on '|'.
-		inner := strings.TrimPrefix(strings.TrimSuffix(line, "|"), "|")
-		cols := strings.Split(inner, "|")
-		if len(cols) < 6 {
-			continue
-		}
-		for i, c := range cols {
-			cols[i] = strings.TrimSpace(c)
-		}
-		// Skip header row and separator row.
-		if cols[0] == "Rank" || strings.HasPrefix(cols[0], "---") {
-			continue
-		}
-		rank, err := strconv.Atoi(cols[0])
-		if err != nil {
-			continue
-		}
-		title := cols[1]
-		cover := cols[2]
-		status := cols[3]
-		genresStr := cols[4]
-		sourceURL := cols[5]
-
-		var genres []string
-		if genresStr != "-" && genresStr != "" {
-			for _, g := range strings.Split(genresStr, ",") {
-				g = strings.TrimSpace(g)
-				if g != "" {
-					genres = append(genres, g)
-				}
-			}
-		}
-
-		// Derive slug from source URL (last path segment).
-		slug := ""
-		if sourceURL != "" {
-			parts := strings.Split(strings.TrimRight(sourceURL, "/"), "/")
-			if len(parts) > 0 {
-				slug = parts[len(parts)-1]
-			}
-		}
-
-		items = append(items, RankingItem{
-			Rank:      rank,
-			Slug:      slug,
-			Title:     title,
-			Cover:     cover,
-			Status:    status,
-			Genres:    genres,
-			SourceURL: sourceURL,
-		})
+	if err := json.Unmarshal(data, &items); err != nil {
+		return nil, fmt.Errorf("writer: parse ranking json: %w", err)
 	}
 	return items, nil
 }
 
-// RankingFileInfo returns os.FileInfo for the ranking.md file, if it exists.
+// RankingFileInfo returns os.FileInfo for the ranking.json file, if it exists.
 func (w *Writer) RankingFileInfo() (os.FileInfo, error) {
 	return os.Stat(w.rankingPath())
 }
 
 func (w *Writer) rankingPath() string {
-	return filepath.Join(w.root, "ranking.md")
+	return filepath.Join(w.root, "ranking.json")
 }
 
 // ─── Ranking page HTML cache ──────────────────────────────────────────────────