feat(storage): add scraping_tasks collection and Store interface methods

Adds a scraping_tasks PocketBase collection with fields for kind, status, progress counters, timestamps, and error info. Exposes CreateScrapeTask, UpdateScrapeTask, and ListScrapeTasks on the Store interface with implementations in HybridStore and PocketBaseStore.
2026-03-04 00:40:17 +05:00
parent 333c8ad868
commit 041099598b
3 changed files with 167 additions and 13 deletions
--- a/scraper/internal/storage/hybrid.go
+++ b/scraper/internal/storage/hybrid.go
@@ -237,6 +237,56 @@ func (h *HybridStore) PresignAudio(ctx context.Context, key string, expires time
 	return h.minio.PresignAudio(ctx, key, expires)
 }

+// ─── Scraping tasks ───────────────────────────────────────────────────────────
+
+func (h *HybridStore) CreateScrapeTask(ctx context.Context, kind, targetURL string) (string, error) {
+	return h.pb.CreateScrapingTask(ctx, kind, targetURL)
+}
+
+func (h *HybridStore) UpdateScrapeTask(ctx context.Context, id string, u ScrapeTaskUpdate) error {
+	data := map[string]interface{}{
+		"status":           u.Status,
+		"books_found":      u.BooksFound,
+		"chapters_scraped": u.ChaptersScraped,
+		"chapters_skipped": u.ChaptersSkipped,
+		"errors":           u.Errors,
+		"error_message":    u.ErrorMessage,
+	}
+	if !u.Finished.IsZero() {
+		data["finished"] = u.Finished.UTC().Format(time.RFC3339)
+	}
+	return h.pb.UpdateScrapingTask(ctx, id, data)
+}
+
+func (h *HybridStore) ListScrapeTasks(ctx context.Context) ([]ScrapeTask, error) {
+	rows, err := h.pb.ListScrapingTasks(ctx)
+	if err != nil {
+		return nil, err
+	}
+	tasks := make([]ScrapeTask, 0, len(rows))
+	for _, r := range rows {
+		t := ScrapeTask{
+			ID:              strVal(r, "id"),
+			Kind:            strVal(r, "kind"),
+			TargetURL:       strVal(r, "target_url"),
+			Status:          strVal(r, "status"),
+			BooksFound:      int(floatVal(r, "books_found")),
+			ChaptersScraped: int(floatVal(r, "chapters_scraped")),
+			ChaptersSkipped: int(floatVal(r, "chapters_skipped")),
+			Errors:          int(floatVal(r, "errors")),
+			ErrorMessage:    strVal(r, "error_message"),
+		}
+		if ts, ok := r["started"].(string); ok {
+			t.Started, _ = time.Parse(time.RFC3339, ts)
+		}
+		if ts, ok := r["finished"].(string); ok && ts != "" {
+			t.Finished, _ = time.Parse(time.RFC3339, ts)
+		}
+		tasks = append(tasks, t)
+	}
+	return tasks, nil
+}
+
 // ─── helpers ──────────────────────────────────────────────────────────────────

 func recToBookMeta(rec map[string]interface{}) scraper.BookMeta {
--- a/scraper/internal/storage/pocketbase.go
+++ b/scraper/internal/storage/pocketbase.go
@@ -2,14 +2,18 @@
 //
 // Collections expected in PocketBase:
 //
-//	books        — slug(text,unique), title, author, cover, status, genres(json),
-//	               summary, total_chapters(number), source_url, ranking(number), updated(date)
-//	chapters_idx — slug(text), number(number), title, date_label, updated(date)
-//	ranking      — rank(number), slug(text,unique), title, author, cover, status,
-//	               genres(json), source_url, updated(date)
-//	progress     — session_id(text), slug(text), chapter(number), updated(date)
-//	audio_cache  — cache_key(text,unique), filename(text), updated(date)
-//	app_users    — username(text,unique), password_hash(text), role(text), created(date)
+//	books          — slug(text,unique), title, author, cover, status, genres(json),
+//	                 summary, total_chapters(number), source_url, ranking(number), updated(date)
+//	chapters_idx   — slug(text), number(number), title, date_label, updated(date)
+//	ranking        — rank(number), slug(text,unique), title, author, cover, status,
+//	                 genres(json), source_url, updated(date)
+//	progress       — session_id(text), slug(text), chapter(number), updated(date)
+//	audio_cache    — cache_key(text,unique), filename(text), updated(date)
+//	app_users      — username(text,unique), password_hash(text), role(text), created(date)
+//	scraping_tasks — id(auto), kind(text), target_url(text), status(text),
+//	                 books_found(number), chapters_scraped(number),
+//	                 chapters_skipped(number), errors(number),
+//	                 started(date), finished(date), error_message(text)
 package storage

 import (
@@ -345,6 +349,22 @@ func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
 				{"name": "created", "type": "date"},
 			},
 		},
+		{
+			"name": "scraping_tasks",
+			"type": "base",
+			"fields": []map[string]interface{}{
+				{"name": "kind", "type": "text", "required": true},   // "catalogue" | "book"
+				{"name": "target_url", "type": "text"},               // set for single-book scrapes
+				{"name": "status", "type": "text", "required": true}, // "running" | "done" | "failed" | "cancelled"
+				{"name": "books_found", "type": "number"},
+				{"name": "chapters_scraped", "type": "number"},
+				{"name": "chapters_skipped", "type": "number"},
+				{"name": "errors", "type": "number"},
+				{"name": "started", "type": "date"},
+				{"name": "finished", "type": "date"},
+				{"name": "error_message", "type": "text"},
+			},
+		},
 	}
 	for _, col := range collections {
 		name, _ := col["name"].(string)
@@ -590,6 +610,58 @@ func (s *PocketBaseStore) GetAudioCache(ctx context.Context, cacheKey string) (s
 	return filename, filename != "", nil
 }

+// ─── Scraping tasks ───────────────────────────────────────────────────────────
+
+// CreateScrapingTask inserts a new scraping_tasks record with status="running"
+// and returns the newly created record's ID.
+func (s *PocketBaseStore) CreateScrapingTask(ctx context.Context, kind, targetURL string) (string, error) {
+	data := map[string]interface{}{
+		"kind":             kind,
+		"target_url":       targetURL,
+		"status":           "running",
+		"books_found":      0,
+		"chapters_scraped": 0,
+		"chapters_skipped": 0,
+		"errors":           0,
+		"started":          time.Now().UTC().Format(time.RFC3339),
+	}
+	resp, err := s.pb.do(ctx, http.MethodPost, "/api/collections/scraping_tasks/records", data)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	b, _ := io.ReadAll(resp.Body)
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
+		return "", fmt.Errorf("pocketbase: CreateScrapingTask: status %d: %s", resp.StatusCode, b)
+	}
+	var rec map[string]interface{}
+	if err := json.Unmarshal(b, &rec); err != nil {
+		return "", fmt.Errorf("pocketbase: CreateScrapingTask: decode: %w", err)
+	}
+	id, _ := rec["id"].(string)
+	return id, nil
+}
+
+// UpdateScrapingTask patches counters on an existing scraping_tasks record.
+func (s *PocketBaseStore) UpdateScrapingTask(ctx context.Context, id string, data map[string]interface{}) error {
+	resp, err := s.pb.do(ctx, http.MethodPatch,
+		fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), data)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		b, _ := io.ReadAll(resp.Body)
+		return fmt.Errorf("pocketbase: UpdateScrapingTask id=%s: status %d: %s", id, resp.StatusCode, b)
+	}
+	return nil
+}
+
+// ListScrapingTasks returns all scraping_tasks sorted by started descending.
+func (s *PocketBaseStore) ListScrapingTasks(ctx context.Context) ([]map[string]interface{}, error) {
+	return s.pb.listAll(ctx, "scraping_tasks", "", "-started")
+}
+
 // ─── helpers ──────────────────────────────────────────────────────────────────

 // pbEsc escapes a string for use in a PocketBase filter expression.
--- a/scraper/internal/storage/store.go
+++ b/scraper/internal/storage/store.go
@@ -30,11 +30,33 @@ type ReadingProgress struct {
 	UpdatedAt time.Time `json:"updated_at"`
 }

-// AudioCacheEntry maps a (slug, chapter, voice, speed) tuple to a Kokoro
-// download filename so audio is not re-generated after a server restart.
-type AudioCacheEntry struct {
-	CacheKey string `json:"cache_key"`
-	Filename string `json:"filename"`
+// ScrapeTask represents a single scraping job record from the scraping_tasks
+// collection.
+type ScrapeTask struct {
+	ID              string    `json:"id"`
+	Kind            string    `json:"kind"`       // "catalogue" | "book"
+	TargetURL       string    `json:"target_url"` // non-empty for single-book scrapes
+	Status          string    `json:"status"`     // "running" | "done" | "failed" | "cancelled"
+	BooksFound      int       `json:"books_found"`
+	ChaptersScraped int       `json:"chapters_scraped"`
+	ChaptersSkipped int       `json:"chapters_skipped"`
+	Errors          int       `json:"errors"`
+	Started         time.Time `json:"started"`
+	Finished        time.Time `json:"finished,omitempty"`
+	ErrorMessage    string    `json:"error_message,omitempty"`
+}
+
+// ScrapeTaskUpdate carries the fields that can be patched on a ScrapeTask.
+// Zero-value fields are still sent; callers should only include keys they want
+// to change via the map form used inside the store implementation.
+type ScrapeTaskUpdate struct {
+	Status          string
+	BooksFound      int
+	ChaptersScraped int
+	ChaptersSkipped int
+	Errors          int
+	Finished        time.Time // zero = not finished yet
+	ErrorMessage    string
 }

 // ─── Store interface ──────────────────────────────────────────────────────────
@@ -113,4 +135,14 @@ type Store interface {

 	// PresignAudio returns a presigned GET URL for an audio object.
 	PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error)
+
+	// ── Scraping tasks ─────────────────────────────────────────────────────
+
+	// CreateScrapeTask inserts a new scraping_tasks record with status="running"
+	// and returns the assigned ID.
+	CreateScrapeTask(ctx context.Context, kind, targetURL string) (string, error)
+	// UpdateScrapeTask patches an existing task record.
+	UpdateScrapeTask(ctx context.Context, id string, u ScrapeTaskUpdate) error
+	// ListScrapeTasks returns all tasks sorted by started descending.
+	ListScrapeTasks(ctx context.Context) ([]ScrapeTask, error)
 }