feat(storage): add scraping_tasks collection and Store interface methods

Adds a scraping_tasks PocketBase collection with fields for kind, status,
progress counters, timestamps, and error info. Exposes CreateScrapeTask,
UpdateScrapeTask, and ListScrapeTasks on the Store interface with
implementations in HybridStore and PocketBaseStore.
This commit is contained in:
Admin
2026-03-04 00:40:17 +05:00
parent 333c8ad868
commit 041099598b
3 changed files with 167 additions and 13 deletions

View File

@@ -2,14 +2,18 @@
//
// Collections expected in PocketBase:
//
// books — slug(text,unique), title, author, cover, status, genres(json),
// summary, total_chapters(number), source_url, ranking(number), updated(date)
// chapters_idx — slug(text), number(number), title, date_label, updated(date)
// ranking — rank(number), slug(text,unique), title, author, cover, status,
// genres(json), source_url, updated(date)
// progress — session_id(text), slug(text), chapter(number), updated(date)
// audio_cache — cache_key(text,unique), filename(text), updated(date)
// app_users — username(text,unique), password_hash(text), role(text), created(date)
// books — slug(text,unique), title, author, cover, status, genres(json),
// summary, total_chapters(number), source_url, ranking(number), updated(date)
// chapters_idx — slug(text), number(number), title, date_label, updated(date)
// ranking — rank(number), slug(text,unique), title, author, cover, status,
// genres(json), source_url, updated(date)
// progress — session_id(text), slug(text), chapter(number), updated(date)
// audio_cache — cache_key(text,unique), filename(text), updated(date)
// app_users — username(text,unique), password_hash(text), role(text), created(date)
// scraping_tasks — id(auto), kind(text), target_url(text), status(text),
// books_found(number), chapters_scraped(number),
// chapters_skipped(number), errors(number),
// started(date), finished(date), error_message(text)
package storage
import (
@@ -345,6 +349,22 @@ func (s *PocketBaseStore) EnsureCollections(ctx context.Context) error {
{"name": "created", "type": "date"},
},
},
{
"name": "scraping_tasks",
"type": "base",
"fields": []map[string]interface{}{
{"name": "kind", "type": "text", "required": true}, // "catalogue" | "book"
{"name": "target_url", "type": "text"}, // set for single-book scrapes
{"name": "status", "type": "text", "required": true}, // "running" | "done" | "failed" | "cancelled"
{"name": "books_found", "type": "number"},
{"name": "chapters_scraped", "type": "number"},
{"name": "chapters_skipped", "type": "number"},
{"name": "errors", "type": "number"},
{"name": "started", "type": "date"},
{"name": "finished", "type": "date"},
{"name": "error_message", "type": "text"},
},
},
}
for _, col := range collections {
name, _ := col["name"].(string)
@@ -590,6 +610,58 @@ func (s *PocketBaseStore) GetAudioCache(ctx context.Context, cacheKey string) (s
return filename, filename != "", nil
}
// ─── Scraping tasks ───────────────────────────────────────────────────────────
// CreateScrapingTask inserts a new scraping_tasks record with status="running"
// and returns the newly created record's ID.
func (s *PocketBaseStore) CreateScrapingTask(ctx context.Context, kind, targetURL string) (string, error) {
data := map[string]interface{}{
"kind": kind,
"target_url": targetURL,
"status": "running",
"books_found": 0,
"chapters_scraped": 0,
"chapters_skipped": 0,
"errors": 0,
"started": time.Now().UTC().Format(time.RFC3339),
}
resp, err := s.pb.do(ctx, http.MethodPost, "/api/collections/scraping_tasks/records", data)
if err != nil {
return "", err
}
defer resp.Body.Close()
b, _ := io.ReadAll(resp.Body)
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
return "", fmt.Errorf("pocketbase: CreateScrapingTask: status %d: %s", resp.StatusCode, b)
}
var rec map[string]interface{}
if err := json.Unmarshal(b, &rec); err != nil {
return "", fmt.Errorf("pocketbase: CreateScrapingTask: decode: %w", err)
}
id, _ := rec["id"].(string)
return id, nil
}
// UpdateScrapingTask patches counters on an existing scraping_tasks record.
func (s *PocketBaseStore) UpdateScrapingTask(ctx context.Context, id string, data map[string]interface{}) error {
resp, err := s.pb.do(ctx, http.MethodPatch,
fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id), data)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return fmt.Errorf("pocketbase: UpdateScrapingTask id=%s: status %d: %s", id, resp.StatusCode, b)
}
return nil
}
// ListScrapingTasks returns all scraping_tasks sorted by started descending.
func (s *PocketBaseStore) ListScrapingTasks(ctx context.Context) ([]map[string]interface{}, error) {
return s.pb.listAll(ctx, "scraping_tasks", "", "-started")
}
// ─── helpers ──────────────────────────────────────────────────────────────────
// pbEsc escapes a string for use in a PocketBase filter expression.