v2.6.50: notifications overhaul, fix blank page, fix chapter review loading
All checks were successful
Release / Test backend (push) Successful in 3m15s
Release / Check ui (push) Successful in 1m49s
Release / Docker (push) Successful in 5m53s
Release / Gitea Release (push) Successful in 40s

- svelte.config.js: paths.relative=false so CSS uses absolute /_app/ paths (fixes blank home page after redirect)
- ai-jobs: fix openReview() mutating stale alias r instead of $state review — was causing 'Loading results...' to never resolve for chapter-names/image-gen/description
- notifications bell: redesign with All/Unread tabs, per-item dismiss (×), mark-all-read, clear-all, 'View all' footer link
- /admin/notifications: new dedicated full-page notifications view
- api/notifications proxy: add PATCH (mark-all-read) and DELETE (clear-all, dismiss) handlers
- runner: add CreateNotification calls on success/failure in runScrapeTask, runAudioTask, runTranslationTask
- storage/import.go: real PDF (dslipak/pdf) and EPUB (archive/zip + x/net/html) parsing replacing stubs
- translation admin page: stream jobs Promise instead of blocking navigation
- store.go: DeleteNotification, ClearAllNotifications, MarkAllNotificationsRead methods
- handlers_notifications.go + server.go: PATCH /api/notifications, DELETE /api/notifications, DELETE /api/notifications/{id}
This commit is contained in:
root
2026-04-09 15:14:00 +05:00
parent 2571c243c9
commit 2ca1ab2250
17 changed files with 980 additions and 164 deletions

View File

@@ -12,8 +12,10 @@ require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/clipperhouse/uax29/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dslipak/pdf v0.0.2 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/getsentry/sentry-go v0.43.0 // indirect
@@ -23,16 +25,21 @@ require (
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
github.com/hhrutter/lzw v1.0.0 // indirect
github.com/hhrutter/pkcs7 v0.2.0 // indirect
github.com/hhrutter/tiff v1.0.2 // indirect
github.com/hibiken/asynq v0.26.0 // indirect
github.com/hibiken/asynq/x v0.0.0-20260203063626-d704b68a426d // indirect
github.com/klauspost/compress v1.18.2 // indirect
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/mattn/go-runewidth v0.0.19 // indirect
github.com/meilisearch/meilisearch-go v0.36.1 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.23.2 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
@@ -61,6 +68,7 @@ require (
go.yaml.in/yaml/v2 v2.4.2 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.48.0 // indirect
golang.org/x/image v0.32.0 // indirect
golang.org/x/sys v0.41.0 // indirect
golang.org/x/text v0.34.0 // indirect
golang.org/x/time v0.14.0 // indirect
@@ -68,5 +76,6 @@ require (
google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect
google.golang.org/grpc v1.79.2 // indirect
google.golang.org/protobuf v1.36.11 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

View File

@@ -6,10 +6,14 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY=
github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dslipak/pdf v0.0.2 h1:djAvcM5neg9Ush+zR6QXB+VMJzR6TdnX766HPIg1JmI=
github.com/dslipak/pdf v0.0.2/go.mod h1:2L3SnkI9cQwnAS9gfPz2iUoLC0rUZwbucpbKi5R1mUo=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
@@ -29,6 +33,12 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c=
github.com/hhrutter/lzw v1.0.0 h1:laL89Llp86W3rRs83LvKbwYRx6INE8gDn0XNb1oXtm0=
github.com/hhrutter/lzw v1.0.0/go.mod h1:2HC6DJSn/n6iAZfgM3Pg+cP1KxeWc3ezG8bBqW5+WEo=
github.com/hhrutter/pkcs7 v0.2.0 h1:i4HN2XMbGQpZRnKBLsUwO3dSckzgX142TNqY/KfXg+I=
github.com/hhrutter/pkcs7 v0.2.0/go.mod h1:aEzKz0+ZAlz7YaEMY47jDHL14hVWD6iXt0AgqgAvWgE=
github.com/hhrutter/tiff v1.0.2 h1:7H3FQQpKu/i5WaSChoD1nnJbGx4MxU5TlNqqpxw55z8=
github.com/hhrutter/tiff v1.0.2/go.mod h1:pcOeuK5loFUE7Y/WnzGw20YxUdnqjY1P0Jlcieb/cCw=
github.com/hibiken/asynq v0.26.0 h1:1Zxr92MlDnb1Zt/QR5g2vSCqUS03i95lUfqx5X7/wrw=
github.com/hibiken/asynq v0.26.0/go.mod h1:Qk4e57bTnWDoyJ67VkchuV6VzSM9IQW2nPvAGuDyw58=
github.com/hibiken/asynq/x v0.0.0-20260203063626-d704b68a426d h1:Ld5m8EIK5QVOq/owOexKIbETij3skACg4eU1pArHsrw=
@@ -40,6 +50,8 @@ github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4O
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
github.com/meilisearch/meilisearch-go v0.36.1 h1:mJTCJE5g7tRvaqKco6DfqOuJEjX+rRltDEnkEC02Y0M=
github.com/meilisearch/meilisearch-go v0.36.1/go.mod h1:hWcR0MuWLSzHfbz9GGzIr3s9rnXLm1jqkmHkJPbUSvM=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
@@ -50,8 +62,12 @@ github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRi
github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/pdfcpu/pdfcpu v0.11.1 h1:htHBSkGH5jMKWC6e0sihBFbcKZ8vG1M67c8/dJxhjas=
github.com/pdfcpu/pdfcpu v0.11.1/go.mod h1:pP3aGga7pRvwFWAm9WwFvo+V68DfANi9kxSQYioNYcw=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
@@ -120,6 +136,8 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
golang.org/x/image v0.32.0 h1:6lZQWq75h7L5IWNk0r+SCpUJ6tUVd3v4ZHnbRKLkUDQ=
golang.org/x/image v0.32.0/go.mod h1:/R37rrQmKXtO6tYXAjtDLwQgFLHmhW+V6ayXlxzP2Pc=
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
@@ -139,5 +157,7 @@ google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -7,6 +7,63 @@ import (
"github.com/libnovel/backend/internal/storage"
)
// handleDismissNotification handles DELETE /api/notifications/{id}.
func (s *Server) handleDismissNotification(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
if id == "" {
jsonError(w, http.StatusBadRequest, "notification id required")
return
}
store, ok := s.deps.Producer.(*storage.Store)
if !ok {
jsonError(w, http.StatusInternalServerError, "storage not available")
return
}
if err := store.DeleteNotification(r.Context(), id); err != nil {
jsonError(w, http.StatusInternalServerError, "dismiss notification: "+err.Error())
return
}
writeJSON(w, 0, map[string]any{"success": true})
}
// handleClearAllNotifications handles DELETE /api/notifications?user_id=...
func (s *Server) handleClearAllNotifications(w http.ResponseWriter, r *http.Request) {
userID := r.URL.Query().Get("user_id")
if userID == "" {
jsonError(w, http.StatusBadRequest, "user_id required")
return
}
store, ok := s.deps.Producer.(*storage.Store)
if !ok {
jsonError(w, http.StatusInternalServerError, "storage not available")
return
}
if err := store.ClearAllNotifications(r.Context(), userID); err != nil {
jsonError(w, http.StatusInternalServerError, "clear notifications: "+err.Error())
return
}
writeJSON(w, 0, map[string]any{"success": true})
}
// handleMarkAllNotificationsRead handles PATCH /api/notifications?user_id=...
func (s *Server) handleMarkAllNotificationsRead(w http.ResponseWriter, r *http.Request) {
userID := r.URL.Query().Get("user_id")
if userID == "" {
jsonError(w, http.StatusBadRequest, "user_id required")
return
}
store, ok := s.deps.Producer.(*storage.Store)
if !ok {
jsonError(w, http.StatusInternalServerError, "storage not available")
return
}
if err := store.MarkAllNotificationsRead(r.Context(), userID); err != nil {
jsonError(w, http.StatusInternalServerError, "mark all read: "+err.Error())
return
}
writeJSON(w, 0, map[string]any{"success": true})
}
type notification struct {
ID string `json:"id"`
UserID string `json:"user_id"`

View File

@@ -251,7 +251,10 @@ func (s *Server) ListenAndServe(ctx context.Context) error {
// Notifications
mux.HandleFunc("GET /api/notifications", s.handleListNotifications)
mux.HandleFunc("PATCH /api/notifications", s.handleMarkAllNotificationsRead)
mux.HandleFunc("PATCH /api/notifications/{id}", s.handleMarkNotificationRead)
mux.HandleFunc("DELETE /api/notifications", s.handleClearAllNotifications)
mux.HandleFunc("DELETE /api/notifications/{id}", s.handleDismissNotification)
// Voices list
mux.HandleFunc("GET /api/voices", s.handleVoices)

View File

@@ -503,9 +503,21 @@ func (r *Runner) runScrapeTask(ctx context.Context, task domain.ScrapeTask) {
if result.ErrorMessage != "" {
r.tasksFailed.Add(1)
span.SetStatus(codes.Error, result.ErrorMessage)
if r.deps.Notifier != nil {
_ = r.deps.Notifier.CreateNotification(ctx, "admin",
"Scrape Failed",
fmt.Sprintf("Scrape task (%s) failed: %s", task.Kind, result.ErrorMessage),
"/admin/tasks")
}
} else {
r.tasksCompleted.Add(1)
span.SetStatus(codes.Ok, "")
if r.deps.Notifier != nil {
_ = r.deps.Notifier.CreateNotification(ctx, "admin",
"Scrape Complete",
fmt.Sprintf("Scraped %d chapters, skipped %d (%s)", result.ChaptersScraped, result.ChaptersSkipped, task.Kind),
"/admin/tasks")
}
}
log.Info("runner: scrape task finished",
@@ -585,6 +597,12 @@ func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) {
if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil {
log.Error("runner: FinishAudioTask failed", "err", err)
}
if r.deps.Notifier != nil {
_ = r.deps.Notifier.CreateNotification(ctx, "admin",
"Audio Failed",
fmt.Sprintf("Ch.%d of %s (%s): %s", task.Chapter, task.Slug, task.Voice, msg),
fmt.Sprintf("/books/%s", task.Slug))
}
}
raw, err := r.deps.BookReader.ReadChapter(ctx, task.Slug, task.Chapter)
@@ -649,6 +667,12 @@ func (r *Runner) runAudioTask(ctx context.Context, task domain.AudioTask) {
if err := r.deps.Consumer.FinishAudioTask(ctx, task.ID, result); err != nil {
log.Error("runner: FinishAudioTask failed", "err", err)
}
if r.deps.Notifier != nil {
_ = r.deps.Notifier.CreateNotification(ctx, "admin",
"Audio Ready",
fmt.Sprintf("Ch.%d of %s (%s) is ready", task.Chapter, task.Slug, task.Voice),
fmt.Sprintf("/books/%s", task.Slug))
}
log.Info("runner: audio task finished", "key", key)
}

View File

@@ -53,6 +53,12 @@ func (r *Runner) runTranslationTask(ctx context.Context, task domain.Translation
if err := r.deps.Consumer.FinishTranslationTask(ctx, task.ID, result); err != nil {
log.Error("runner: FinishTranslationTask failed", "err", err)
}
if r.deps.Notifier != nil {
_ = r.deps.Notifier.CreateNotification(ctx, "admin",
"Translation Failed",
fmt.Sprintf("Ch.%d of %s (%s): %s", task.Chapter, task.Slug, task.Lang, msg),
fmt.Sprintf("/books/%s", task.Slug))
}
}
// Guard: LibreTranslate must be configured.
@@ -93,5 +99,11 @@ func (r *Runner) runTranslationTask(ctx context.Context, task domain.Translation
if err := r.deps.Consumer.FinishTranslationTask(ctx, task.ID, result); err != nil {
log.Error("runner: FinishTranslationTask failed", "err", err)
}
if r.deps.Notifier != nil {
_ = r.deps.Notifier.CreateNotification(ctx, "admin",
"Translation Ready",
fmt.Sprintf("Ch.%d of %s translated to %s", task.Chapter, task.Slug, task.Lang),
fmt.Sprintf("/books/%s", task.Slug))
}
log.Info("runner: translation task finished", "key", key)
}

View File

@@ -1,20 +1,27 @@
package storage
import (
"archive/zip"
"bytes"
"context"
"errors"
"fmt"
"io"
"regexp"
"sort"
"strconv"
"strings"
"github.com/dslipak/pdf"
"github.com/libnovel/backend/internal/bookstore"
"github.com/libnovel/backend/internal/domain"
minio "github.com/minio/minio-go/v7"
"golang.org/x/net/html"
)
var (
chapterPattern = regexp.MustCompile(`(?i)chapter\s+(\d+)|The\s+Eminence\s+in\s+Shadow\s+(\d+)\s*-\s*(\d+)`)
)
// chapterHeadingRE matches common chapter heading patterns:
// "Chapter 1", "Chapter 1:", "Chapter 1 -", "CHAPTER ONE", "1.", "Part 1", etc.
var chapterHeadingRE = regexp.MustCompile(
`(?i)^(?:chapter|ch\.?|part|episode|book)\s+(\d+|[ivxlcdm]+)\b|^\d{1,4}[\.\)]\s+\S`)
type importer struct {
mc *minioClient
@@ -42,73 +49,453 @@ func (i *importer) Import(ctx context.Context, objectKey, fileType string) ([]bo
}
if fileType == "pdf" {
return i.parsePDF(data)
return parsePDF(data)
}
return i.parseEPUB(data)
return parseEPUB(data)
}
func (i *importer) parsePDF(data []byte) ([]bookstore.Chapter, error) {
return nil, errors.New("PDF parsing not yet implemented - requires external library")
// ── PDF parsing ───────────────────────────────────────────────────────────────
func parsePDF(data []byte) ([]bookstore.Chapter, error) {
r, err := pdf.NewReader(bytes.NewReader(data), int64(len(data)))
if err != nil {
return nil, fmt.Errorf("open PDF: %w", err)
}
// Extract per-page text so we can detect chapter boundaries.
numPages := r.NumPage()
if numPages == 0 {
return nil, fmt.Errorf("PDF has no pages")
}
// Collect full text first with page markers so we can split by chapter.
var sb strings.Builder
fonts := make(map[string]*pdf.Font)
for i := 1; i <= numPages; i++ {
page := r.Page(i)
if page.V.IsNull() {
continue
}
text, err := page.GetPlainText(fonts)
if err != nil {
continue
}
sb.WriteString(text)
sb.WriteByte('\n')
}
return extractChaptersFromText(sb.String()), nil
}
func (i *importer) parseEPUB(data []byte) ([]bookstore.Chapter, error) {
return nil, errors.New("EPUB parsing not yet implemented - requires external library")
}
// ── EPUB parsing ──────────────────────────────────────────────────────────────
func parseEPUB(data []byte) ([]bookstore.Chapter, error) {
zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
if err != nil {
return nil, fmt.Errorf("open EPUB zip: %w", err)
}
// 1. Read META-INF/container.xml → find rootfile (content.opf path).
opfPath, err := epubRootfilePath(zr)
if err != nil {
return nil, fmt.Errorf("epub container: %w", err)
}
// 2. Parse content.opf → spine order of chapter files.
spineFiles, titleMap, err := epubSpine(zr, opfPath)
if err != nil {
return nil, fmt.Errorf("epub spine: %w", err)
}
if len(spineFiles) == 0 {
return nil, fmt.Errorf("EPUB spine is empty")
}
// Base directory of the OPF file for resolving relative hrefs.
opfDir := ""
if idx := strings.LastIndex(opfPath, "/"); idx >= 0 {
opfDir = opfPath[:idx+1]
}
// extractChaptersFromText is a helper that splits raw text into chapters.
// Used as a fallback when the PDF parser library returns raw text.
func extractChaptersFromText(text string) []bookstore.Chapter {
var chapters []bookstore.Chapter
var currentChapter *bookstore.Chapter
for i, href := range spineFiles {
fullPath := opfDir + href
content, err := epubFileContent(zr, fullPath)
if err != nil {
continue
}
text := htmlToText(content)
if strings.TrimSpace(text) == "" {
continue
}
title := titleMap[href]
if title == "" {
title = fmt.Sprintf("Chapter %d", i+1)
}
chapters = append(chapters, bookstore.Chapter{
Number: i + 1,
Title: title,
Content: text,
})
}
if len(chapters) == 0 {
return nil, fmt.Errorf("no readable chapters found in EPUB")
}
return chapters, nil
}
// epubRootfilePath parses META-INF/container.xml and returns the full-path
// of the OPF package document.
func epubRootfilePath(zr *zip.Reader) (string, error) {
f := zipFile(zr, "META-INF/container.xml")
if f == nil {
return "", fmt.Errorf("META-INF/container.xml not found")
}
rc, err := f.Open()
if err != nil {
return "", err
}
defer rc.Close()
doc, err := html.Parse(rc)
if err != nil {
return "", err
}
var path string
var walk func(*html.Node)
walk = func(n *html.Node) {
if n.Type == html.ElementNode && strings.EqualFold(n.Data, "rootfile") {
for _, a := range n.Attr {
if strings.EqualFold(a.Key, "full-path") {
path = a.Val
return
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(doc)
if path == "" {
return "", fmt.Errorf("rootfile full-path not found in container.xml")
}
return path, nil
}
// epubSpine parses the OPF document and returns the spine item hrefs in order,
// plus a map from href → nav title (if available from NCX/NAV).
func epubSpine(zr *zip.Reader, opfPath string) ([]string, map[string]string, error) {
f := zipFile(zr, opfPath)
if f == nil {
return nil, nil, fmt.Errorf("OPF file %q not found in EPUB", opfPath)
}
rc, err := f.Open()
if err != nil {
return nil, nil, err
}
defer rc.Close()
opfData, err := io.ReadAll(rc)
if err != nil {
return nil, nil, err
}
// Build id→href map from <manifest>.
idToHref := make(map[string]string)
// Also keep a href→navTitle map (populated from NCX later).
hrefTitle := make(map[string]string)
// Parse OPF XML with html.Parse (handles malformed XML too).
doc, _ := html.Parse(bytes.NewReader(opfData))
var manifestItems []struct{ id, href, mediaType string }
var spineIdrefs []string
var ncxID string
var walk func(*html.Node)
walk = func(n *html.Node) {
if n.Type == html.ElementNode {
tag := strings.ToLower(n.Data)
switch tag {
case "item":
var id, href, mt string
for _, a := range n.Attr {
switch strings.ToLower(a.Key) {
case "id":
id = a.Val
case "href":
href = a.Val
case "media-type":
mt = a.Val
}
}
if id != "" && href != "" {
manifestItems = append(manifestItems, struct{ id, href, mediaType string }{id, href, mt})
idToHref[id] = href
}
case "itemref":
for _, a := range n.Attr {
if strings.ToLower(a.Key) == "idref" {
spineIdrefs = append(spineIdrefs, a.Val)
}
}
case "spine":
for _, a := range n.Attr {
if strings.ToLower(a.Key) == "toc" {
ncxID = a.Val
}
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(doc)
// Build ordered spine href list.
var spineHrefs []string
for _, idref := range spineIdrefs {
if href, ok := idToHref[idref]; ok {
spineHrefs = append(spineHrefs, href)
}
}
// If no explicit spine, fall back to all XHTML items in manifest order.
if len(spineHrefs) == 0 {
sort.Slice(manifestItems, func(i, j int) bool {
return manifestItems[i].href < manifestItems[j].href
})
for _, it := range manifestItems {
mt := strings.ToLower(it.mediaType)
if strings.Contains(mt, "html") || strings.HasSuffix(strings.ToLower(it.href), ".html") || strings.HasSuffix(strings.ToLower(it.href), ".xhtml") {
spineHrefs = append(spineHrefs, it.href)
}
}
}
// Try to get chapter titles from NCX (toc.ncx).
opfDir := ""
if idx := strings.LastIndex(opfPath, "/"); idx >= 0 {
opfDir = opfPath[:idx+1]
}
if ncxHref, ok := idToHref[ncxID]; ok {
ncxPath := opfDir + ncxHref
if ncxFile := zipFile(zr, ncxPath); ncxFile != nil {
if ncxRC, err := ncxFile.Open(); err == nil {
defer ncxRC.Close()
parseNCXTitles(ncxRC, hrefTitle)
}
}
}
return spineHrefs, hrefTitle, nil
}
// parseNCXTitles extracts navPoint label→src mappings from a toc.ncx.
func parseNCXTitles(r io.Reader, out map[string]string) {
doc, err := html.Parse(r)
if err != nil {
return
}
// Collect navPoints: each has a <navLabel><text>…</text></navLabel> and
// a <content src="…"/> child.
var walk func(*html.Node)
walk = func(n *html.Node) {
if n.Type == html.ElementNode && strings.EqualFold(n.Data, "navpoint") {
var label, src string
var inner func(*html.Node)
inner = func(c *html.Node) {
if c.Type == html.ElementNode {
if strings.EqualFold(c.Data, "text") && label == "" {
if c.FirstChild != nil && c.FirstChild.Type == html.TextNode {
label = strings.TrimSpace(c.FirstChild.Data)
}
}
if strings.EqualFold(c.Data, "content") {
for _, a := range c.Attr {
if strings.EqualFold(a.Key, "src") {
// Strip fragment identifier (#...).
src = strings.SplitN(a.Val, "#", 2)[0]
}
}
}
}
for child := c.FirstChild; child != nil; child = child.NextSibling {
inner(child)
}
}
inner(n)
if label != "" && src != "" {
out[src] = label
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(doc)
}
// epubFileContent returns the raw bytes of a file inside the EPUB zip.
func epubFileContent(zr *zip.Reader, path string) ([]byte, error) {
f := zipFile(zr, path)
if f == nil {
return nil, fmt.Errorf("file %q not in EPUB", path)
}
rc, err := f.Open()
if err != nil {
return nil, err
}
defer rc.Close()
return io.ReadAll(rc)
}
// zipFile finds a file by name (case-insensitive) in a zip.Reader.
func zipFile(zr *zip.Reader, name string) *zip.File {
nameLower := strings.ToLower(name)
for _, f := range zr.File {
if strings.ToLower(f.Name) == nameLower {
return f
}
}
return nil
}
// htmlToText converts HTML/XHTML content to plain text suitable for storage.
func htmlToText(data []byte) string {
doc, err := html.Parse(bytes.NewReader(data))
if err != nil {
return string(data)
}
var sb strings.Builder
var walk func(*html.Node)
walk = func(n *html.Node) {
if n.Type == html.TextNode {
text := strings.TrimSpace(n.Data)
if text != "" {
sb.WriteString(text)
sb.WriteByte(' ')
}
}
if n.Type == html.ElementNode {
switch strings.ToLower(n.Data) {
case "p", "div", "br", "h1", "h2", "h3", "h4", "h5", "h6", "li", "tr":
// Block-level: ensure newline before content.
if sb.Len() > 0 {
s := sb.String()
if s[len(s)-1] != '\n' {
sb.WriteByte('\n')
}
}
case "script", "style", "head":
// Skip entirely.
return
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
if n.Type == html.ElementNode {
switch strings.ToLower(n.Data) {
case "p", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "tr":
sb.WriteByte('\n')
}
}
}
walk(doc)
// Collapse multiple blank lines.
lines := strings.Split(sb.String(), "\n")
var out []string
blanks := 0
for _, l := range lines {
l = strings.TrimSpace(l)
if l == "" {
blanks++
if blanks <= 1 {
out = append(out, "")
}
} else {
blanks = 0
out = append(out, l)
}
}
return strings.TrimSpace(strings.Join(out, "\n"))
}
// ── Chapter segmentation (shared by PDF and plain-text paths) ─────────────────
// extractChaptersFromText splits a block of plain text into chapters by
// detecting heading lines that match chapterHeadingRE.
// Falls back to paragraph-splitting when no headings are found.
func extractChaptersFromText(text string) []bookstore.Chapter {
lines := strings.Split(text, "\n")
chapterNum := 0
type segment struct {
title string
number int
lines []string
}
var segments []segment
var cur *segment
chNum := 0
for _, line := range lines {
line = strings.TrimSpace(line)
if len(line) < 3 {
continue
}
matches := chapterPattern.FindStringSubmatch(line)
if matches != nil {
if currentChapter != nil && currentChapter.Content != "" {
chapters = append(chapters, *currentChapter)
if chapterHeadingRE.MatchString(line) {
if cur != nil {
segments = append(segments, *cur)
}
chapterNum++
if matches[1] != "" {
chapterNum, _ = fmt.Sscanf(matches[1], "%d", &chapterNum)
chNum++
// Try to parse the explicit chapter number from the heading.
if m := regexp.MustCompile(`\d+`).FindString(line); m != "" {
if n, err := strconv.Atoi(m); err == nil && n > 0 && n < 100000 {
chNum = n
}
}
currentChapter = &bookstore.Chapter{
Number: chapterNum,
Title: line,
Content: "",
}
continue
}
if currentChapter != nil {
if currentChapter.Content != "" {
currentChapter.Content += " "
}
currentChapter.Content += line
cur = &segment{title: line, number: chNum}
} else if cur != nil && line != "" {
cur.lines = append(cur.lines, line)
}
}
if currentChapter != nil && currentChapter.Content != "" {
chapters = append(chapters, *currentChapter)
if cur != nil {
segments = append(segments, *cur)
}
// If no chapters found via regex, try splitting by double newlines
// Require segments to have meaningful content (>= 100 chars).
var chapters []bookstore.Chapter
for _, seg := range segments {
content := strings.Join(seg.lines, "\n")
if len(strings.TrimSpace(content)) < 50 {
continue
}
chapters = append(chapters, bookstore.Chapter{
Number: seg.number,
Title: seg.title,
Content: content,
})
}
// Fallback: no headings found — split by double newlines (paragraph blocks).
if len(chapters) == 0 {
paragraphs := strings.Split(text, "\n\n")
for i, para := range paragraphs {
n := 0
for _, para := range paragraphs {
para = strings.TrimSpace(para)
if len(para) > 50 {
if len(para) > 100 {
n++
chapters = append(chapters, bookstore.Chapter{
Number: i + 1,
Title: fmt.Sprintf("Chapter %d", i+1),
Content: para,
Number: n,
Title: fmt.Sprintf("Chapter %d", n),
Content: para,
})
}
}
@@ -117,28 +504,31 @@ func extractChaptersFromText(text string) []bookstore.Chapter {
return chapters
}
// IngestChapters stores extracted chapters for a book via BookWriter.
// This is called by the runner after extracting chapters from PDF/EPUB.
// ── Chapter ingestion ─────────────────────────────────────────────────────────
// IngestChapters stores extracted chapters for a book.
// Each chapter is written as a markdown file in the chapters MinIO bucket
// and its index record is upserted in PocketBase via WriteChapter.
func (s *Store) IngestChapters(ctx context.Context, slug string, chapters []bookstore.Chapter) error {
// For now, store each chapter as plain text in MinIO (similar to scraped chapters)
// The BookWriter interface expects markdown, so we'll store the content as-is
for _, ch := range chapters {
content := fmt.Sprintf("# Chapter %d\n\n%s", ch.Number, ch.Content)
if ch.Title != "" {
content = fmt.Sprintf("# %s\n\n%s", ch.Title, ch.Content)
var mdContent string
if ch.Title != "" && ch.Title != fmt.Sprintf("Chapter %d", ch.Number) {
mdContent = fmt.Sprintf("# %s\n\n%s", ch.Title, ch.Content)
} else {
mdContent = fmt.Sprintf("# Chapter %d\n\n%s", ch.Number, ch.Content)
}
key := fmt.Sprintf("books/%s/chapters/%d.md", slug, ch.Number)
if err := s.mc.putObject(ctx, "books", key, "text/markdown", []byte(content)); err != nil {
return fmt.Errorf("put chapter %d: %w", ch.Number, err)
domainCh := domain.Chapter{
Ref: domain.ChapterRef{Number: ch.Number, Title: ch.Title},
Text: mdContent,
}
if err := s.WriteChapter(ctx, slug, domainCh); err != nil {
return fmt.Errorf("ingest chapter %d: %w", ch.Number, err)
}
}
// Also create a simple metadata entry in the books collection
// (in a real implementation, we'd update the existing book or create a placeholder)
return nil
}
// GetImportObjectKey returns the MinIO object key for an uploaded import file.
func GetImportObjectKey(filename string) string {
return fmt.Sprintf("imports/%s", filename)
}
}

View File

@@ -708,6 +708,48 @@ func (s *Store) MarkNotificationRead(ctx context.Context, id string) error {
map[string]any{"read": true})
}
// DeleteNotification deletes a single notification by ID.
func (s *Store) DeleteNotification(ctx context.Context, id string) error {
return s.pb.delete(ctx, fmt.Sprintf("/api/collections/notifications/records/%s", id))
}
// ClearAllNotifications deletes all notifications for a user.
func (s *Store) ClearAllNotifications(ctx context.Context, userID string) error {
filter := fmt.Sprintf("user_id='%s'", userID)
items, err := s.pb.listAll(ctx, "notifications", filter, "")
if err != nil {
return fmt.Errorf("ClearAllNotifications list: %w", err)
}
for _, raw := range items {
var rec struct {
ID string `json:"id"`
}
if json.Unmarshal(raw, &rec) == nil && rec.ID != "" {
_ = s.pb.delete(ctx, fmt.Sprintf("/api/collections/notifications/records/%s", rec.ID))
}
}
return nil
}
// MarkAllNotificationsRead marks all notifications for a user as read.
func (s *Store) MarkAllNotificationsRead(ctx context.Context, userID string) error {
filter := fmt.Sprintf("user_id='%s'&&read=false", userID)
items, err := s.pb.listAll(ctx, "notifications", filter, "")
if err != nil {
return fmt.Errorf("MarkAllNotificationsRead list: %w", err)
}
for _, raw := range items {
var rec struct {
ID string `json:"id"`
}
if json.Unmarshal(raw, &rec) == nil && rec.ID != "" {
_ = s.pb.patch(ctx, fmt.Sprintf("/api/collections/notifications/records/%s", rec.ID),
map[string]any{"read": true})
}
}
return nil
}
func (s *Store) CancelTask(ctx context.Context, id string) error {
// Try scraping_tasks first, then audio_jobs, then translation_jobs.
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id),