Compare commits

...

3 Commits

Author SHA1 Message Date
root
d82aa9d4b4 fix(import): decrypt owner-encrypted PDFs with pdfcpu; add imports bucket to minio-init
All checks were successful
Release / Test backend (push) Successful in 2m10s
Release / Check ui (push) Successful in 1m55s
Release / Docker (push) Successful in 7m17s
Release / Gitea Release (push) Successful in 48s
- parsePDF now attempts to strip encryption via pdfcpu (empty user password)
  before handing bytes to dslipak/pdf — fixes '256-bit encryption key' error
  on publisher PDFs that use owner-only encryption (copy/print restrictions)
- Add pdfcpu v0.11.1 as direct dependency (was already indirect)
- docker-compose.yml minio-init: add 'imports' and 'translations' buckets
  so a fresh deploy creates all required buckets
2026-04-09 20:08:12 +05:00
root
ae08382b81 fix(import): wire ImportFileStore to bypass Asynq type assertion; add pb-init collections
All checks were successful
Release / Test backend (push) Successful in 44s
Release / Check ui (push) Successful in 1m56s
Release / Docker (push) Successful in 6m10s
Release / Gitea Release (push) Successful in 37s
- Add ImportFileStore interface to bookstore package
- Add ImportFileStore field to backend.Dependencies
- Wire ImportFileStore: store in cmd/backend/main.go
- handlers_import.go: use s.deps.ImportFileStore.PutImportFile instead of
  broken s.deps.Producer.(*storage.Store) type assertion (fails when Asynq active)
- pb-init-v3.sh: add import_tasks and notifications collection definitions
2026-04-09 19:05:11 +05:00
root
b9f8008c2c chore: embed git credentials in remote URL; update AGENTS.md 2026-04-09 17:09:53 +05:00
9 changed files with 89 additions and 5 deletions

View File

@@ -47,11 +47,21 @@ Sub-directories have their own `AGENTS.md` with deeper context (e.g. `ios/AGENTS
- `release.yaml` — runs on `v*` tags (build Docker images, upload source maps, create Gitea release)
- Secrets: `DOCKER_USER`, `DOCKER_TOKEN`, `GITEA_TOKEN`, `GLITCHTIP_AUTH_TOKEN`
### Git credentials
Credentials are embedded in the remote URL — no `HOME=/root` or credential helper needed for push:
```
https://kamil:95782641Apple%24@gitea.kalekber.cc/kamil/libnovel.git
```
All git commands still use `HOME=/root` prefix for consistency (picks up `/root/.gitconfig` for user name/email), but push auth works without it.
### Releasing a new version
```bash
git tag v2.5.X -m "Short title\n\nOptional longer body"
git push origin v2.5.X
HOME=/root git tag v2.6.X -m "Short title"
HOME=/root git push origin v3-cleanup --tags
```
CI will build all Docker images, upload source maps to GlitchTip, and create a Gitea release automatically.

View File

@@ -189,6 +189,7 @@ func run() error {
ChapterImageStore: store,
Producer: producer,
TaskReader: store,
ImportFileStore: store,
SearchIndex: searchIndex,
Kokoro: kokoroClient,
PocketTTS: pocketTTSClient,

View File

@@ -38,6 +38,7 @@ require (
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pdfcpu/pdfcpu v0.11.1 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect

View File

@@ -98,12 +98,11 @@ func (s *Server) handleAdminImport(w http.ResponseWriter, r *http.Request) {
// Upload to MinIO for actual import
objectKey = fmt.Sprintf("imports/%d_%s", time.Now().Unix(), header.Filename)
store, ok := s.deps.Producer.(*storage.Store)
if !ok {
if s.deps.ImportFileStore == nil {
jsonError(w, http.StatusInternalServerError, "storage not available")
return
}
if err := store.PutImportFile(r.Context(), objectKey, data); err != nil {
if err := s.deps.ImportFileStore.PutImportFile(r.Context(), objectKey, data); err != nil {
jsonError(w, http.StatusInternalServerError, "upload file: "+err.Error())
return
}

View File

@@ -85,6 +85,9 @@ type Dependencies struct {
// BookWriter writes book metadata and chapter refs to PocketBase.
// Used by admin text-gen apply endpoints.
BookWriter bookstore.BookWriter
// ImportFileStore uploads raw PDF/EPUB files to MinIO for the runner to process.
// Always wired to the concrete *storage.Store (not the Asynq wrapper).
ImportFileStore bookstore.ImportFileStore
// AIJobStore tracks long-running AI generation jobs in PocketBase.
// If nil, job persistence is disabled (jobs still run but are not recorded).
AIJobStore bookstore.AIJobStore

View File

@@ -215,3 +215,10 @@ type BookImporter interface {
// Returns the extracted chapters or an error.
Import(ctx context.Context, objectKey, fileType string) ([]Chapter, error)
}
// ImportFileStore uploads raw import files to object storage.
// Kept separate from BookImporter so the HTTP handler can upload the file
// without a concrete type assertion, regardless of which Producer is wired.
type ImportFileStore interface {
PutImportFile(ctx context.Context, objectKey string, data []byte) error
}

View File

@@ -15,6 +15,8 @@ import (
"github.com/libnovel/backend/internal/bookstore"
"github.com/libnovel/backend/internal/domain"
minio "github.com/minio/minio-go/v7"
"github.com/pdfcpu/pdfcpu/pkg/api"
"github.com/pdfcpu/pdfcpu/pkg/pdfcpu/model"
"golang.org/x/net/html"
)
@@ -90,7 +92,33 @@ func AnalyzeFile(data []byte, fileType string) (chapterCount int, firstLines []s
// decryptPDF strips encryption from a PDF using an empty user password.
// Returns the decrypted bytes, or an error if decryption is not possible.
// This handles the common case of "owner-only" encrypted PDFs (copy/print
// restrictions) which use an empty user password and open normally in readers.
func decryptPDF(data []byte) ([]byte, error) {
conf := model.NewDefaultConfiguration()
conf.UserPW = ""
conf.OwnerPW = ""
var out bytes.Buffer
err := api.Decrypt(bytes.NewReader(data), &out, conf)
if err != nil {
return nil, err
}
return out.Bytes(), nil
}
func parsePDF(data []byte) ([]bookstore.Chapter, error) {
// If the PDF is encrypted, try to decrypt it with an empty password.
// Many publisher PDFs use owner-only encryption (copy/print restrictions)
// with an empty user password, so they open normally but confuse parsers.
decrypted, err := decryptPDF(data)
if err == nil {
data = decrypted
}
// (if decryption fails we still attempt to parse — maybe it works anyway)
r, err := pdf.NewReader(bytes.NewReader(data), int64(len(data)))
if err != nil {
return nil, fmt.Errorf("open PDF: %w", err)

View File

@@ -58,6 +58,8 @@ services:
mc mb --ignore-existing local/audio;
mc mb --ignore-existing local/avatars;
mc mb --ignore-existing local/catalogue;
mc mb --ignore-existing local/translations;
mc mb --ignore-existing local/imports;
echo 'buckets ready';
"
environment:

View File

@@ -299,6 +299,39 @@ create "translation_jobs" '{
{"name":"heartbeat_at", "type":"date"}
]}'
create "import_tasks" '{
"name":"import_tasks","type":"base","fields":[
{"name":"slug", "type":"text", "required":true},
{"name":"title", "type":"text", "required":true},
{"name":"file_name", "type":"text"},
{"name":"file_type", "type":"text"},
{"name":"object_key", "type":"text"},
{"name":"author", "type":"text"},
{"name":"cover_url", "type":"text"},
{"name":"genres", "type":"text"},
{"name":"summary", "type":"text"},
{"name":"book_status", "type":"text"},
{"name":"worker_id", "type":"text"},
{"name":"initiator_user_id", "type":"text"},
{"name":"status", "type":"text", "required":true},
{"name":"chapters_done", "type":"number"},
{"name":"chapters_total", "type":"number"},
{"name":"error_message", "type":"text"},
{"name":"started", "type":"date"},
{"name":"finished", "type":"date"},
{"name":"heartbeat_at", "type":"date"}
]}'
create "notifications" '{
"name":"notifications","type":"base","fields":[
{"name":"user_id", "type":"text","required":true},
{"name":"title", "type":"text","required":true},
{"name":"message", "type":"text"},
{"name":"link", "type":"text"},
{"name":"read", "type":"bool"},
{"name":"created", "type":"date"}
]}'
create "ai_jobs" '{
"name":"ai_jobs","type":"base","fields":[
{"name":"kind", "type":"text", "required":true},