v2.6.50: notifications overhaul, fix blank page, fix chapter review loading
All checks were successful
Release / Test backend (push) Successful in 3m15s
Release / Check ui (push) Successful in 1m49s
Release / Docker (push) Successful in 5m53s
Release / Gitea Release (push) Successful in 40s

- svelte.config.js: paths.relative=false so CSS uses absolute /_app/ paths (fixes blank home page after redirect)
- ai-jobs: fix openReview() mutating stale alias r instead of $state review — was causing 'Loading results...' to never resolve for chapter-names/image-gen/description
- notifications bell: redesign with All/Unread tabs, per-item dismiss (×), mark-all-read, clear-all, 'View all' footer link
- /admin/notifications: new dedicated full-page notifications view
- api/notifications proxy: add PATCH (mark-all-read) and DELETE (clear-all, dismiss) handlers
- runner: add CreateNotification calls on success/failure in runScrapeTask, runAudioTask, runTranslationTask
- storage/import.go: real PDF (dslipak/pdf) and EPUB (archive/zip + x/net/html) parsing replacing stubs
- translation admin page: stream jobs Promise instead of blocking navigation
- store.go: DeleteNotification, ClearAllNotifications, MarkAllNotificationsRead methods
- handlers_notifications.go + server.go: PATCH /api/notifications, DELETE /api/notifications, DELETE /api/notifications/{id}
This commit is contained in:
root
2026-04-09 15:14:00 +05:00
parent 2571c243c9
commit 2ca1ab2250
17 changed files with 980 additions and 164 deletions

View File

@@ -1,20 +1,27 @@
package storage
import (
"archive/zip"
"bytes"
"context"
"errors"
"fmt"
"io"
"regexp"
"sort"
"strconv"
"strings"
"github.com/dslipak/pdf"
"github.com/libnovel/backend/internal/bookstore"
"github.com/libnovel/backend/internal/domain"
minio "github.com/minio/minio-go/v7"
"golang.org/x/net/html"
)
var (
chapterPattern = regexp.MustCompile(`(?i)chapter\s+(\d+)|The\s+Eminence\s+in\s+Shadow\s+(\d+)\s*-\s*(\d+)`)
)
// chapterHeadingRE matches common chapter heading patterns:
// "Chapter 1", "Chapter 1:", "Chapter 1 -", "CHAPTER ONE", "1.", "Part 1", etc.
var chapterHeadingRE = regexp.MustCompile(
`(?i)^(?:chapter|ch\.?|part|episode|book)\s+(\d+|[ivxlcdm]+)\b|^\d{1,4}[\.\)]\s+\S`)
type importer struct {
mc *minioClient
@@ -42,73 +49,453 @@ func (i *importer) Import(ctx context.Context, objectKey, fileType string) ([]bo
}
if fileType == "pdf" {
return i.parsePDF(data)
return parsePDF(data)
}
return i.parseEPUB(data)
return parseEPUB(data)
}
func (i *importer) parsePDF(data []byte) ([]bookstore.Chapter, error) {
return nil, errors.New("PDF parsing not yet implemented - requires external library")
// ── PDF parsing ───────────────────────────────────────────────────────────────
func parsePDF(data []byte) ([]bookstore.Chapter, error) {
r, err := pdf.NewReader(bytes.NewReader(data), int64(len(data)))
if err != nil {
return nil, fmt.Errorf("open PDF: %w", err)
}
// Extract per-page text so we can detect chapter boundaries.
numPages := r.NumPage()
if numPages == 0 {
return nil, fmt.Errorf("PDF has no pages")
}
// Collect full text first with page markers so we can split by chapter.
var sb strings.Builder
fonts := make(map[string]*pdf.Font)
for i := 1; i <= numPages; i++ {
page := r.Page(i)
if page.V.IsNull() {
continue
}
text, err := page.GetPlainText(fonts)
if err != nil {
continue
}
sb.WriteString(text)
sb.WriteByte('\n')
}
return extractChaptersFromText(sb.String()), nil
}
func (i *importer) parseEPUB(data []byte) ([]bookstore.Chapter, error) {
return nil, errors.New("EPUB parsing not yet implemented - requires external library")
}
// ── EPUB parsing ──────────────────────────────────────────────────────────────
func parseEPUB(data []byte) ([]bookstore.Chapter, error) {
zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
if err != nil {
return nil, fmt.Errorf("open EPUB zip: %w", err)
}
// 1. Read META-INF/container.xml → find rootfile (content.opf path).
opfPath, err := epubRootfilePath(zr)
if err != nil {
return nil, fmt.Errorf("epub container: %w", err)
}
// 2. Parse content.opf → spine order of chapter files.
spineFiles, titleMap, err := epubSpine(zr, opfPath)
if err != nil {
return nil, fmt.Errorf("epub spine: %w", err)
}
if len(spineFiles) == 0 {
return nil, fmt.Errorf("EPUB spine is empty")
}
// Base directory of the OPF file for resolving relative hrefs.
opfDir := ""
if idx := strings.LastIndex(opfPath, "/"); idx >= 0 {
opfDir = opfPath[:idx+1]
}
// extractChaptersFromText is a helper that splits raw text into chapters.
// Used as a fallback when the PDF parser library returns raw text.
func extractChaptersFromText(text string) []bookstore.Chapter {
var chapters []bookstore.Chapter
var currentChapter *bookstore.Chapter
for i, href := range spineFiles {
fullPath := opfDir + href
content, err := epubFileContent(zr, fullPath)
if err != nil {
continue
}
text := htmlToText(content)
if strings.TrimSpace(text) == "" {
continue
}
title := titleMap[href]
if title == "" {
title = fmt.Sprintf("Chapter %d", i+1)
}
chapters = append(chapters, bookstore.Chapter{
Number: i + 1,
Title: title,
Content: text,
})
}
if len(chapters) == 0 {
return nil, fmt.Errorf("no readable chapters found in EPUB")
}
return chapters, nil
}
// epubRootfilePath parses META-INF/container.xml and returns the full-path
// of the OPF package document.
func epubRootfilePath(zr *zip.Reader) (string, error) {
f := zipFile(zr, "META-INF/container.xml")
if f == nil {
return "", fmt.Errorf("META-INF/container.xml not found")
}
rc, err := f.Open()
if err != nil {
return "", err
}
defer rc.Close()
doc, err := html.Parse(rc)
if err != nil {
return "", err
}
var path string
var walk func(*html.Node)
walk = func(n *html.Node) {
if n.Type == html.ElementNode && strings.EqualFold(n.Data, "rootfile") {
for _, a := range n.Attr {
if strings.EqualFold(a.Key, "full-path") {
path = a.Val
return
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(doc)
if path == "" {
return "", fmt.Errorf("rootfile full-path not found in container.xml")
}
return path, nil
}
// epubSpine parses the OPF document and returns the spine item hrefs in order,
// plus a map from href → nav title (if available from NCX/NAV).
func epubSpine(zr *zip.Reader, opfPath string) ([]string, map[string]string, error) {
f := zipFile(zr, opfPath)
if f == nil {
return nil, nil, fmt.Errorf("OPF file %q not found in EPUB", opfPath)
}
rc, err := f.Open()
if err != nil {
return nil, nil, err
}
defer rc.Close()
opfData, err := io.ReadAll(rc)
if err != nil {
return nil, nil, err
}
// Build id→href map from <manifest>.
idToHref := make(map[string]string)
// Also keep a href→navTitle map (populated from NCX later).
hrefTitle := make(map[string]string)
// Parse OPF XML with html.Parse (handles malformed XML too).
doc, _ := html.Parse(bytes.NewReader(opfData))
var manifestItems []struct{ id, href, mediaType string }
var spineIdrefs []string
var ncxID string
var walk func(*html.Node)
walk = func(n *html.Node) {
if n.Type == html.ElementNode {
tag := strings.ToLower(n.Data)
switch tag {
case "item":
var id, href, mt string
for _, a := range n.Attr {
switch strings.ToLower(a.Key) {
case "id":
id = a.Val
case "href":
href = a.Val
case "media-type":
mt = a.Val
}
}
if id != "" && href != "" {
manifestItems = append(manifestItems, struct{ id, href, mediaType string }{id, href, mt})
idToHref[id] = href
}
case "itemref":
for _, a := range n.Attr {
if strings.ToLower(a.Key) == "idref" {
spineIdrefs = append(spineIdrefs, a.Val)
}
}
case "spine":
for _, a := range n.Attr {
if strings.ToLower(a.Key) == "toc" {
ncxID = a.Val
}
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(doc)
// Build ordered spine href list.
var spineHrefs []string
for _, idref := range spineIdrefs {
if href, ok := idToHref[idref]; ok {
spineHrefs = append(spineHrefs, href)
}
}
// If no explicit spine, fall back to all XHTML items in manifest order.
if len(spineHrefs) == 0 {
sort.Slice(manifestItems, func(i, j int) bool {
return manifestItems[i].href < manifestItems[j].href
})
for _, it := range manifestItems {
mt := strings.ToLower(it.mediaType)
if strings.Contains(mt, "html") || strings.HasSuffix(strings.ToLower(it.href), ".html") || strings.HasSuffix(strings.ToLower(it.href), ".xhtml") {
spineHrefs = append(spineHrefs, it.href)
}
}
}
// Try to get chapter titles from NCX (toc.ncx).
opfDir := ""
if idx := strings.LastIndex(opfPath, "/"); idx >= 0 {
opfDir = opfPath[:idx+1]
}
if ncxHref, ok := idToHref[ncxID]; ok {
ncxPath := opfDir + ncxHref
if ncxFile := zipFile(zr, ncxPath); ncxFile != nil {
if ncxRC, err := ncxFile.Open(); err == nil {
defer ncxRC.Close()
parseNCXTitles(ncxRC, hrefTitle)
}
}
}
return spineHrefs, hrefTitle, nil
}
// parseNCXTitles extracts navPoint label→src mappings from a toc.ncx.
func parseNCXTitles(r io.Reader, out map[string]string) {
doc, err := html.Parse(r)
if err != nil {
return
}
// Collect navPoints: each has a <navLabel><text>…</text></navLabel> and
// a <content src="…"/> child.
var walk func(*html.Node)
walk = func(n *html.Node) {
if n.Type == html.ElementNode && strings.EqualFold(n.Data, "navpoint") {
var label, src string
var inner func(*html.Node)
inner = func(c *html.Node) {
if c.Type == html.ElementNode {
if strings.EqualFold(c.Data, "text") && label == "" {
if c.FirstChild != nil && c.FirstChild.Type == html.TextNode {
label = strings.TrimSpace(c.FirstChild.Data)
}
}
if strings.EqualFold(c.Data, "content") {
for _, a := range c.Attr {
if strings.EqualFold(a.Key, "src") {
// Strip fragment identifier (#...).
src = strings.SplitN(a.Val, "#", 2)[0]
}
}
}
}
for child := c.FirstChild; child != nil; child = child.NextSibling {
inner(child)
}
}
inner(n)
if label != "" && src != "" {
out[src] = label
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
}
walk(doc)
}
// epubFileContent returns the raw bytes of a file inside the EPUB zip.
func epubFileContent(zr *zip.Reader, path string) ([]byte, error) {
f := zipFile(zr, path)
if f == nil {
return nil, fmt.Errorf("file %q not in EPUB", path)
}
rc, err := f.Open()
if err != nil {
return nil, err
}
defer rc.Close()
return io.ReadAll(rc)
}
// zipFile finds a file by name (case-insensitive) in a zip.Reader.
func zipFile(zr *zip.Reader, name string) *zip.File {
nameLower := strings.ToLower(name)
for _, f := range zr.File {
if strings.ToLower(f.Name) == nameLower {
return f
}
}
return nil
}
// htmlToText converts HTML/XHTML content to plain text suitable for storage.
func htmlToText(data []byte) string {
doc, err := html.Parse(bytes.NewReader(data))
if err != nil {
return string(data)
}
var sb strings.Builder
var walk func(*html.Node)
walk = func(n *html.Node) {
if n.Type == html.TextNode {
text := strings.TrimSpace(n.Data)
if text != "" {
sb.WriteString(text)
sb.WriteByte(' ')
}
}
if n.Type == html.ElementNode {
switch strings.ToLower(n.Data) {
case "p", "div", "br", "h1", "h2", "h3", "h4", "h5", "h6", "li", "tr":
// Block-level: ensure newline before content.
if sb.Len() > 0 {
s := sb.String()
if s[len(s)-1] != '\n' {
sb.WriteByte('\n')
}
}
case "script", "style", "head":
// Skip entirely.
return
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
walk(c)
}
if n.Type == html.ElementNode {
switch strings.ToLower(n.Data) {
case "p", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "tr":
sb.WriteByte('\n')
}
}
}
walk(doc)
// Collapse multiple blank lines.
lines := strings.Split(sb.String(), "\n")
var out []string
blanks := 0
for _, l := range lines {
l = strings.TrimSpace(l)
if l == "" {
blanks++
if blanks <= 1 {
out = append(out, "")
}
} else {
blanks = 0
out = append(out, l)
}
}
return strings.TrimSpace(strings.Join(out, "\n"))
}
// ── Chapter segmentation (shared by PDF and plain-text paths) ─────────────────
// extractChaptersFromText splits a block of plain text into chapters by
// detecting heading lines that match chapterHeadingRE.
// Falls back to paragraph-splitting when no headings are found.
func extractChaptersFromText(text string) []bookstore.Chapter {
lines := strings.Split(text, "\n")
chapterNum := 0
type segment struct {
title string
number int
lines []string
}
var segments []segment
var cur *segment
chNum := 0
for _, line := range lines {
line = strings.TrimSpace(line)
if len(line) < 3 {
continue
}
matches := chapterPattern.FindStringSubmatch(line)
if matches != nil {
if currentChapter != nil && currentChapter.Content != "" {
chapters = append(chapters, *currentChapter)
if chapterHeadingRE.MatchString(line) {
if cur != nil {
segments = append(segments, *cur)
}
chapterNum++
if matches[1] != "" {
chapterNum, _ = fmt.Sscanf(matches[1], "%d", &chapterNum)
chNum++
// Try to parse the explicit chapter number from the heading.
if m := regexp.MustCompile(`\d+`).FindString(line); m != "" {
if n, err := strconv.Atoi(m); err == nil && n > 0 && n < 100000 {
chNum = n
}
}
currentChapter = &bookstore.Chapter{
Number: chapterNum,
Title: line,
Content: "",
}
continue
}
if currentChapter != nil {
if currentChapter.Content != "" {
currentChapter.Content += " "
}
currentChapter.Content += line
cur = &segment{title: line, number: chNum}
} else if cur != nil && line != "" {
cur.lines = append(cur.lines, line)
}
}
if currentChapter != nil && currentChapter.Content != "" {
chapters = append(chapters, *currentChapter)
if cur != nil {
segments = append(segments, *cur)
}
// If no chapters found via regex, try splitting by double newlines
// Require segments to have meaningful content (>= 100 chars).
var chapters []bookstore.Chapter
for _, seg := range segments {
content := strings.Join(seg.lines, "\n")
if len(strings.TrimSpace(content)) < 50 {
continue
}
chapters = append(chapters, bookstore.Chapter{
Number: seg.number,
Title: seg.title,
Content: content,
})
}
// Fallback: no headings found — split by double newlines (paragraph blocks).
if len(chapters) == 0 {
paragraphs := strings.Split(text, "\n\n")
for i, para := range paragraphs {
n := 0
for _, para := range paragraphs {
para = strings.TrimSpace(para)
if len(para) > 50 {
if len(para) > 100 {
n++
chapters = append(chapters, bookstore.Chapter{
Number: i + 1,
Title: fmt.Sprintf("Chapter %d", i+1),
Content: para,
Number: n,
Title: fmt.Sprintf("Chapter %d", n),
Content: para,
})
}
}
@@ -117,28 +504,31 @@ func extractChaptersFromText(text string) []bookstore.Chapter {
return chapters
}
// IngestChapters stores extracted chapters for a book via BookWriter.
// This is called by the runner after extracting chapters from PDF/EPUB.
// ── Chapter ingestion ─────────────────────────────────────────────────────────
// IngestChapters stores extracted chapters for a book.
// Each chapter is written as a markdown file in the chapters MinIO bucket
// and its index record is upserted in PocketBase via WriteChapter.
func (s *Store) IngestChapters(ctx context.Context, slug string, chapters []bookstore.Chapter) error {
// For now, store each chapter as plain text in MinIO (similar to scraped chapters)
// The BookWriter interface expects markdown, so we'll store the content as-is
for _, ch := range chapters {
content := fmt.Sprintf("# Chapter %d\n\n%s", ch.Number, ch.Content)
if ch.Title != "" {
content = fmt.Sprintf("# %s\n\n%s", ch.Title, ch.Content)
var mdContent string
if ch.Title != "" && ch.Title != fmt.Sprintf("Chapter %d", ch.Number) {
mdContent = fmt.Sprintf("# %s\n\n%s", ch.Title, ch.Content)
} else {
mdContent = fmt.Sprintf("# Chapter %d\n\n%s", ch.Number, ch.Content)
}
key := fmt.Sprintf("books/%s/chapters/%d.md", slug, ch.Number)
if err := s.mc.putObject(ctx, "books", key, "text/markdown", []byte(content)); err != nil {
return fmt.Errorf("put chapter %d: %w", ch.Number, err)
domainCh := domain.Chapter{
Ref: domain.ChapterRef{Number: ch.Number, Title: ch.Title},
Text: mdContent,
}
if err := s.WriteChapter(ctx, slug, domainCh); err != nil {
return fmt.Errorf("ingest chapter %d: %w", ch.Number, err)
}
}
// Also create a simple metadata entry in the books collection
// (in a real implementation, we'd update the existing book or create a placeholder)
return nil
}
// GetImportObjectKey returns the MinIO object key for an uploaded import file.
func GetImportObjectKey(filename string) string {
return fmt.Sprintf("imports/%s", filename)
}
}

View File

@@ -708,6 +708,48 @@ func (s *Store) MarkNotificationRead(ctx context.Context, id string) error {
map[string]any{"read": true})
}
// DeleteNotification deletes a single notification by ID.
func (s *Store) DeleteNotification(ctx context.Context, id string) error {
return s.pb.delete(ctx, fmt.Sprintf("/api/collections/notifications/records/%s", id))
}
// ClearAllNotifications deletes all notifications for a user.
func (s *Store) ClearAllNotifications(ctx context.Context, userID string) error {
filter := fmt.Sprintf("user_id='%s'", userID)
items, err := s.pb.listAll(ctx, "notifications", filter, "")
if err != nil {
return fmt.Errorf("ClearAllNotifications list: %w", err)
}
for _, raw := range items {
var rec struct {
ID string `json:"id"`
}
if json.Unmarshal(raw, &rec) == nil && rec.ID != "" {
_ = s.pb.delete(ctx, fmt.Sprintf("/api/collections/notifications/records/%s", rec.ID))
}
}
return nil
}
// MarkAllNotificationsRead marks all notifications for a user as read.
func (s *Store) MarkAllNotificationsRead(ctx context.Context, userID string) error {
filter := fmt.Sprintf("user_id='%s'&&read=false", userID)
items, err := s.pb.listAll(ctx, "notifications", filter, "")
if err != nil {
return fmt.Errorf("MarkAllNotificationsRead list: %w", err)
}
for _, raw := range items {
var rec struct {
ID string `json:"id"`
}
if json.Unmarshal(raw, &rec) == nil && rec.ID != "" {
_ = s.pb.patch(ctx, fmt.Sprintf("/api/collections/notifications/records/%s", rec.ID),
map[string]any{"read": true})
}
}
return nil
}
func (s *Store) CancelTask(ctx context.Context, id string) error {
// Try scraping_tasks first, then audio_jobs, then translation_jobs.
if err := s.pb.patch(ctx, fmt.Sprintf("/api/collections/scraping_tasks/records/%s", id),