diff --git a/scraper/internal/storage/hybrid.go b/scraper/internal/storage/hybrid.go index 3fccf57..69710c4 100644 --- a/scraper/internal/storage/hybrid.go +++ b/scraper/internal/storage/hybrid.go @@ -237,6 +237,56 @@ func (h *HybridStore) PresignAudio(ctx context.Context, key string, expires time return h.minio.PresignAudio(ctx, key, expires) } +// ─── Scraping tasks ─────────────────────────────────────────────────────────── + +func (h *HybridStore) CreateScrapeTask(ctx context.Context, kind, targetURL string) (string, error) { + return h.pb.CreateScrapingTask(ctx, kind, targetURL) +} + +func (h *HybridStore) UpdateScrapeTask(ctx context.Context, id string, u ScrapeTaskUpdate) error { + data := map[string]interface{}{ + "status": u.Status, + "books_found": u.BooksFound, + "chapters_scraped": u.ChaptersScraped, + "chapters_skipped": u.ChaptersSkipped, + "errors": u.Errors, + "error_message": u.ErrorMessage, + } + if !u.Finished.IsZero() { + data["finished"] = u.Finished.UTC().Format(time.RFC3339) + } + return h.pb.UpdateScrapingTask(ctx, id, data) +} + +func (h *HybridStore) ListScrapeTasks(ctx context.Context) ([]ScrapeTask, error) { + rows, err := h.pb.ListScrapingTasks(ctx) + if err != nil { + return nil, err + } + tasks := make([]ScrapeTask, 0, len(rows)) + for _, r := range rows { + t := ScrapeTask{ + ID: strVal(r, "id"), + Kind: strVal(r, "kind"), + TargetURL: strVal(r, "target_url"), + Status: strVal(r, "status"), + BooksFound: int(floatVal(r, "books_found")), + ChaptersScraped: int(floatVal(r, "chapters_scraped")), + ChaptersSkipped: int(floatVal(r, "chapters_skipped")), + Errors: int(floatVal(r, "errors")), + ErrorMessage: strVal(r, "error_message"), + } + if ts, ok := r["started"].(string); ok { + t.Started, _ = time.Parse(time.RFC3339, ts) + } + if ts, ok := r["finished"].(string); ok && ts != "" { + t.Finished, _ = time.Parse(time.RFC3339, ts) + } + tasks = append(tasks, t) + } + return tasks, nil +} + // ─── helpers ────────────────────────────────────────────────────────────────── func recToBookMeta(rec map[string]interface{}) scraper.BookMeta { diff --git a/scraper/internal/storage/pocketbase.go b/scraper/internal/storage/pocketbase.go index 2893e3c..4f1d5d2 100644 --- a/scraper/internal/storage/pocketbase.go +++ b/scraper/internal/storage/pocketbase.go @@ -2,14 +2,18 @@ // // Collections expected in PocketBase: // -// books — slug(text,unique), title, author, cover, status, genres(json), -// summary, total_chapters(number), source_url, ranking(number), updated(date) -// chapters_idx — slug(text), number(number), title, date_label, updated(date) -// ranking — rank(number), slug(text,unique), title, author, cover, status, -// genres(json), source_url, updated(date) -// progress — session_id(text), slug(text), chapter(number), updated(date) -// audio_cache — cache_key(text,unique), filename(text), updated(date) -// app_users — username(text,unique), password_hash(text), role(text), created(date) +// books — slug(text,unique), title, author, cover, status, genres(json), +// summary, total_chapters(number), source_url, ranking(number), updated(date) +// chapters_idx — slug(text), number(number), title, date_label, updated(date) +// ranking — rank(number), slug(text,unique), title, author, cover, status, +// genres(json), source_url, updated(date) +// progress — session_id(text), slug(text), chapter(number), updated(date) +// audio_cache — cache_key(text,unique), filename(text), updated(date) +// app_users — username(text,unique), password_hash(text), role(text), created(date) +// scraping_tasks — id(auto), kind(text), target_url(text), status(text), +// books_found(number), chapters_scraped(number), +// chapters_skipped(number), errors(number), +// started(date), finished(date), error_message(text) package storage import ( @@ -345,6 +349,22 @@ func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error { {"name": "created", "type": "date"}, }, }, + { + "name": "scraping_tasks", + "type": "base", + "fields": []map[string]interface{}{ + {"name": "kind", "type": "text", "required": true}, // "catalogue" | "book" + {"name": "target_url", "type": "text"}, // set for single-book scrapes + {"name": "status", "type": "text", "required": true}, // "running" | "done" | "failed" | "cancelled" + {"name": "books_found", "type": "number"}, + {"name": "chapters_scraped", "type": "number"}, + {"name": "chapters_skipped", "type": "number"}, + {"name": "errors", "type": "number"}, + {"name": "started", "type": "date"}, + {"name": "finished", "type": "date"}, + {"name": "error_message", "type": "text"}, + }, + }, } for _, col := range collections { name, _ := col["name"].(string) @@ -590,6 +610,58 @@ func (s *PocketBaseStore) GetAudioCache(ctx context.Context, cacheKey string) (s return filename, filename != "", nil } +// ─── Scraping tasks ─────────────────────────────────────────────────────────── + +// CreateScrapingTask inserts a new scraping_tasks record with status="running" +// and returns the newly created record's ID. +func (s *PocketBaseStore) CreateScrapingTask(ctx context.Context, kind, targetURL string) (string, error) { + data := map[string]interface{}{ + "kind": kind, + "target_url": targetURL, + "status": "running", + "books_found": 0, + "chapters_scraped": 0, + "chapters_skipped": 0, + "errors": 0, + "started": time.Now().UTC().Format(time.RFC3339), + } + resp, err := s.pb.do(ctx, http.MethodPost, "/api/collections/scraping_tasks/records", data) + if err != nil { + return "", err + } + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + return "", fmt.Errorf("pocketbase: CreateScrapingTask: status %d: %s", resp.StatusCode, b) + } + var rec map[string]interface{} + if err := json.Unmarshal(b, &rec); err != nil { + return "", fmt.Errorf("pocketbase: CreateScrapingTask: decode: %w", err) + } + id, _ := rec["id"].(string) + return id, nil +} + +// UpdateScrapingTask patches counters on an existing scraping_tasks record. +func (s *PocketBaseStore) UpdateScrapingTask(ctx context.Context, id string, data map[string]interface{}) error { + resp, err := s.pb.do(ctx, http.MethodPatch, + fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), data) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return fmt.Errorf("pocketbase: UpdateScrapingTask id=%s: status %d: %s", id, resp.StatusCode, b) + } + return nil +} + +// ListScrapingTasks returns all scraping_tasks sorted by started descending. +func (s *PocketBaseStore) ListScrapingTasks(ctx context.Context) ([]map[string]interface{}, error) { + return s.pb.listAll(ctx, "scraping_tasks", "", "-started") +} + // ─── helpers ────────────────────────────────────────────────────────────────── // pbEsc escapes a string for use in a PocketBase filter expression. diff --git a/scraper/internal/storage/store.go b/scraper/internal/storage/store.go index 3d6811e..f3d2e09 100644 --- a/scraper/internal/storage/store.go +++ b/scraper/internal/storage/store.go @@ -30,11 +30,33 @@ type ReadingProgress struct { UpdatedAt time.Time `json:"updated_at"` } -// AudioCacheEntry maps a (slug, chapter, voice, speed) tuple to a Kokoro -// download filename so audio is not re-generated after a server restart. -type AudioCacheEntry struct { - CacheKey string `json:"cache_key"` - Filename string `json:"filename"` +// ScrapeTask represents a single scraping job record from the scraping_tasks +// collection. +type ScrapeTask struct { + ID string `json:"id"` + Kind string `json:"kind"` // "catalogue" | "book" + TargetURL string `json:"target_url"` // non-empty for single-book scrapes + Status string `json:"status"` // "running" | "done" | "failed" | "cancelled" + BooksFound int `json:"books_found"` + ChaptersScraped int `json:"chapters_scraped"` + ChaptersSkipped int `json:"chapters_skipped"` + Errors int `json:"errors"` + Started time.Time `json:"started"` + Finished time.Time `json:"finished,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` +} + +// ScrapeTaskUpdate carries the fields that can be patched on a ScrapeTask. +// Zero-value fields are still sent; callers should only include keys they want +// to change via the map form used inside the store implementation. +type ScrapeTaskUpdate struct { + Status string + BooksFound int + ChaptersScraped int + ChaptersSkipped int + Errors int + Finished time.Time // zero = not finished yet + ErrorMessage string } // ─── Store interface ────────────────────────────────────────────────────────── @@ -113,4 +135,14 @@ type Store interface { // PresignAudio returns a presigned GET URL for an audio object. PresignAudio(ctx context.Context, key string, expires time.Duration) (string, error) + + // ── Scraping tasks ───────────────────────────────────────────────────── + + // CreateScrapeTask inserts a new scraping_tasks record with status="running" + // and returns the assigned ID. + CreateScrapeTask(ctx context.Context, kind, targetURL string) (string, error) + // UpdateScrapeTask patches an existing task record. + UpdateScrapeTask(ctx context.Context, id string, u ScrapeTaskUpdate) error + // ListScrapeTasks returns all tasks sorted by started descending. + ListScrapeTasks(ctx context.Context) ([]ScrapeTask, error) }