fix(acquisition): accept PDFs that carry only an owner password

CHANGELOG.md

@@ -10,6 +10,12 @@ equivalents for Python package metadata.
 
 ### Fixed
 
+- Accept uploaded PDFs that carry only an owner password, as publisher PDFs that
+  restrict printing or copying usually do: they open with the empty user password
+  and their text is extracted. A PDF that needs a password to open is still
+  refused, no other password is tried, and page limits and page-tree checks still
+  apply. A file whose content this server cannot decrypt (AES without a pypdf
+  crypto backend) is refused as encrypted instead of as structurally invalid.
 - Distinguish voice-provider disconnects, failed handshakes and refused redirects
   from internal relay failures with fixed operator diagnostics. Public session-end
   reasons, settlement and the no-redirect credential boundary remain unchanged.

services/api/COMMUNITY_EXPORT_MANIFEST.json

@@ -223,10 +223,10 @@
       "sha256": "6927d6272b210e1ee553ce1e549ab8aaadd77f7a08f2a7390b857139b7990f53"
     },
     {
-      "bytes": 54579,
+      "bytes": 55423,
       "mode": "0644",
       "path": "src/sixsentences_server/acquisition/upload.py",
-      "sha256": "ee2b197e484725fa9f0e2dd7ee4b4be17cf5ba6ef85bb59ef09fee9d4ecdbb86"
+      "sha256": "e5d87f92ab5eab72b844dea2f74294533e5f5314f1b20026d53aa2407282f545"
     },
     {
       "bytes": 7775,
@@ -1777,10 +1777,10 @@
       "sha256": "09cdfc2336300cc44daf4bee9d237d0b761483a9ade212ea82f841f2d8d644af"
     },
     {
-      "bytes": 79982,
+      "bytes": 83794,
       "mode": "0644",
       "path": "tests/test_documents.py",
-      "sha256": "36b9d26944b06c46847a58b7663254b343e4d34fbd3963f7afce2aed245f567c"
+      "sha256": "587c1538fc07172960dc9f2f509a7814cda81fc4bf26681d64ccf94b352a9bc3"
     },
     {
       "bytes": 6086,

services/api/src/sixsentences_server/acquisition/upload.py

@@ -29,6 +29,7 @@
 import pypdf
 from PIL import Image
 from pypdf._codecs import adobe_glyphs
+from pypdf.errors import DependencyError
 from pypdf.generic import (
     ArrayObject,
     ContentStream,
@@ -104,7 +105,11 @@ def validate_pdf_structure(content: bytes) -> None:
         raise UploadError("the uploaded file is not a structurally valid PDF")
     try:
         reader = pypdf.PdfReader(BytesIO(content), strict=False)
-        if reader.is_encrypted:
+        # Publisher PDFs often carry only an owner password, which restricts
+        # printing or copying; the empty user password opens them in any reader.
+        # A PDF that needs a password to open is still refused, and no other
+        # password is ever tried.
+        if reader.is_encrypted and reader.decrypt("") == pypdf.PasswordType.NOT_DECRYPTED:
             raise UploadError("encrypted PDFs are not supported")
         page_count = len(reader.pages)
         if page_count < 1:
@@ -116,8 +121,16 @@ def validate_pdf_structure(content: bytes) -> None:
         for page in reader.pages:
             if str(page.get("/Type", "/Page")) != "/Page":
                 raise UploadError("the PDF contains a malformed page tree")
+        if reader.is_encrypted:
+            # The password check above needs no AES, but AES-encrypted content
+            # does. Decrypting the first page here refuses a file this server
+            # cannot read, instead of storing it without its text.
+            reader.pages[0].get_contents()
     except UploadError:
         raise
+    except DependencyError as exc:
+        # pypdf decrypts AES only through an optional crypto backend.
+        raise UploadError("encrypted PDFs are not supported") from exc
     except Exception as exc:
         raise UploadError("the uploaded file is not a structurally valid PDF") from exc
 

services/api/tests/test_documents.py

@@ -3,10 +3,14 @@
 
 import base64
 import json
+from io import BytesIO
 
+import pypdf
 import pytest
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
+from pypdf.errors import DependencyError
+from pypdf.generic import NameObject
 from sqlalchemy import func, select
 
 from sixsentences_server.acquisition.models import (
@@ -18,11 +22,13 @@
 from sixsentences_server.acquisition.pdf import extract_page_texts
 from sixsentences_server.acquisition.store import LocalDocumentStore
 from sixsentences_server.acquisition.upload import (
+    UploadError,
     _find_arxiv_id,
     _find_doi,
     _resolve_metadata,
     ingest_document,
     is_verified_work_id,
+    validate_pdf_structure,
 )
 from sixsentences_server.api.app import create_app
 from sixsentences_server.chat.service import answer_question
@@ -69,6 +75,21 @@ def _mini_pdf(text: str) -> bytes:
     return bytes(out)
 
 
+def _encrypted_pdf(text: str, *, user_password: str, extra_pages: int = 0) -> bytes:
+    """`_mini_pdf` with an owner password, encrypted with RC4 so no crypto backend is needed.
+
+    An empty `user_password` is how publishers restrict printing or copying: the
+    file still opens without a password. A non-empty one locks it.
+    """
+    writer = pypdf.PdfWriter(clone_from=pypdf.PdfReader(BytesIO(_mini_pdf(text))))
+    for _ in range(extra_pages):
+        writer.add_blank_page()
+    writer.encrypt(user_password=user_password, owner_password="owner", algorithm="RC4-128")
+    out = BytesIO()
+    writer.write(out)
+    return out.getvalue()
+
+
 class _StubOA:
     """OpenAlex stub: fixed work for id lookups, fixed hits for searches."""
 
@@ -903,6 +924,83 @@ def test_ingest_rejects_non_pdf(settings: Settings) -> None:
             )
 
 
+def test_ingest_accepts_a_permission_encrypted_pdf_and_extracts_its_text(
+    settings: Settings,
+) -> None:
+    pdf = _encrypted_pdf(
+        "Restricted Printing Study of Terraform Drift in Regulated Fleets", user_password=""
+    )
+    assert pypdf.PdfReader(BytesIO(pdf)).is_encrypted
+    init_db()
+    with db_session() as session:
+        org = get_default_org(session)
+        doc = ingest_document(
+            session,
+            org_id=org.id,
+            run_id=None,
+            content=pdf,
+            filename="restricted.pdf",
+            oa_client=_StubOA(),
+        )
+        assert doc.text_status == "parsed"
+        work = session.get(WorkRow, doc.work_id)
+        assert work is not None and "Terraform Drift" in work.title
+
+
+def test_ingest_still_refuses_a_pdf_that_needs_a_password(settings: Settings) -> None:
+    pdf = _encrypted_pdf("Locked article text", user_password="reader-secret")
+    init_db()
+    with db_session() as session:
+        org = get_default_org(session)
+        with pytest.raises(UploadError, match="^encrypted PDFs are not supported$"):
+            ingest_document(
+                session,
+                org_id=org.id,
+                run_id=None,
+                content=pdf,
+                filename="locked.pdf",
+                oa_client=_StubOA(),
+            )
+
+
+def test_permission_encrypted_pdf_still_meets_the_page_limit(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr("sixsentences_server.acquisition.upload._MAX_PDF_PAGES", 1)
+    pdf = _encrypted_pdf("Two page article", user_password="", extra_pages=1)
+
+    with pytest.raises(UploadError, match="1-page safety limit"):
+        validate_pdf_structure(pdf)
+
+
+def test_permission_encrypted_pdf_still_needs_a_resolvable_page_tree() -> None:
+    # The page tree counts one kid that is not a /Page, so walking the decrypted
+    # tree fails. The page-tree checks must run after decryption as before it.
+    writer = pypdf.PdfWriter(clone_from=pypdf.PdfReader(BytesIO(_mini_pdf("Odd page"))))
+    writer.pages[0][NameObject("/Type")] = NameObject("/Template")
+    writer.encrypt(user_password="", owner_password="owner", algorithm="RC4-128")
+    out = BytesIO()
+    writer.write(out)
+
+    with pytest.raises(UploadError, match="not a structurally valid PDF"):
+        validate_pdf_structure(out.getvalue())
+
+
+def test_encrypted_pdf_whose_content_cannot_be_decrypted_here_is_refused(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    # AES content needs an optional pypdf crypto backend; without one, the password
+    # check passes and decrypting the content raises DependencyError.
+    def no_backend(self: pypdf.PageObject) -> None:
+        raise DependencyError("cryptography>=3.1 is required for AES algorithm")
+
+    monkeypatch.setattr(pypdf.PageObject, "get_contents", no_backend)
+    pdf = _encrypted_pdf("AES article text", user_password="")
+
+    with pytest.raises(UploadError, match="^encrypted PDFs are not supported$"):
+        validate_pdf_structure(pdf)
+
+
 def _upload(client: TestClient, run_id: int, pdf: bytes, name: str = "paper.pdf") -> dict:
     resp = client.post(
         f"/runs/{run_id}/documents",