diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d74bda..d4dd79c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ equivalents for Python package metadata. ### Fixed +- Accept uploaded PDFs that carry only an owner password, as publisher PDFs that + restrict printing or copying usually do: they open with the empty user password + and their text is extracted. A PDF that needs a password to open is still + refused, no other password is tried, and page limits and page-tree checks still + apply. A file whose content this server cannot decrypt (AES without a pypdf + crypto backend) is refused as encrypted instead of as structurally invalid. - Distinguish voice-provider disconnects, failed handshakes and refused redirects from internal relay failures with fixed operator diagnostics. Public session-end reasons, settlement and the no-redirect credential boundary remain unchanged. diff --git a/services/api/COMMUNITY_EXPORT_MANIFEST.json b/services/api/COMMUNITY_EXPORT_MANIFEST.json index 6422762..4650097 100644 --- a/services/api/COMMUNITY_EXPORT_MANIFEST.json +++ b/services/api/COMMUNITY_EXPORT_MANIFEST.json @@ -223,10 +223,10 @@ "sha256": "6927d6272b210e1ee553ce1e549ab8aaadd77f7a08f2a7390b857139b7990f53" }, { - "bytes": 54579, + "bytes": 55423, "mode": "0644", "path": "src/sixsentences_server/acquisition/upload.py", - "sha256": "ee2b197e484725fa9f0e2dd7ee4b4be17cf5ba6ef85bb59ef09fee9d4ecdbb86" + "sha256": "e5d87f92ab5eab72b844dea2f74294533e5f5314f1b20026d53aa2407282f545" }, { "bytes": 7775, @@ -1777,10 +1777,10 @@ "sha256": "09cdfc2336300cc44daf4bee9d237d0b761483a9ade212ea82f841f2d8d644af" }, { - "bytes": 79982, + "bytes": 83794, "mode": "0644", "path": "tests/test_documents.py", - "sha256": "36b9d26944b06c46847a58b7663254b343e4d34fbd3963f7afce2aed245f567c" + "sha256": "587c1538fc07172960dc9f2f509a7814cda81fc4bf26681d64ccf94b352a9bc3" }, { "bytes": 6086, diff --git a/services/api/src/sixsentences_server/acquisition/upload.py b/services/api/src/sixsentences_server/acquisition/upload.py index 3ea8804..4d553ea 100644 --- a/services/api/src/sixsentences_server/acquisition/upload.py +++ b/services/api/src/sixsentences_server/acquisition/upload.py @@ -29,6 +29,7 @@ import pypdf from PIL import Image from pypdf._codecs import adobe_glyphs +from pypdf.errors import DependencyError from pypdf.generic import ( ArrayObject, ContentStream, @@ -104,7 +105,11 @@ def validate_pdf_structure(content: bytes) -> None: raise UploadError("the uploaded file is not a structurally valid PDF") try: reader = pypdf.PdfReader(BytesIO(content), strict=False) - if reader.is_encrypted: + # Publisher PDFs often carry only an owner password, which restricts + # printing or copying; the empty user password opens them in any reader. + # A PDF that needs a password to open is still refused, and no other + # password is ever tried. + if reader.is_encrypted and reader.decrypt("") == pypdf.PasswordType.NOT_DECRYPTED: raise UploadError("encrypted PDFs are not supported") page_count = len(reader.pages) if page_count < 1: @@ -116,8 +121,16 @@ def validate_pdf_structure(content: bytes) -> None: for page in reader.pages: if str(page.get("/Type", "/Page")) != "/Page": raise UploadError("the PDF contains a malformed page tree") + if reader.is_encrypted: + # The password check above needs no AES, but AES-encrypted content + # does. Decrypting the first page here refuses a file this server + # cannot read, instead of storing it without its text. + reader.pages[0].get_contents() except UploadError: raise + except DependencyError as exc: + # pypdf decrypts AES only through an optional crypto backend. + raise UploadError("encrypted PDFs are not supported") from exc except Exception as exc: raise UploadError("the uploaded file is not a structurally valid PDF") from exc diff --git a/services/api/tests/test_documents.py b/services/api/tests/test_documents.py index 7587cd0..dc84697 100644 --- a/services/api/tests/test_documents.py +++ b/services/api/tests/test_documents.py @@ -3,10 +3,14 @@ import base64 import json +from io import BytesIO +import pypdf import pytest from fastapi import FastAPI from fastapi.testclient import TestClient +from pypdf.errors import DependencyError +from pypdf.generic import NameObject from sqlalchemy import func, select from sixsentences_server.acquisition.models import ( @@ -18,11 +22,13 @@ from sixsentences_server.acquisition.pdf import extract_page_texts from sixsentences_server.acquisition.store import LocalDocumentStore from sixsentences_server.acquisition.upload import ( + UploadError, _find_arxiv_id, _find_doi, _resolve_metadata, ingest_document, is_verified_work_id, + validate_pdf_structure, ) from sixsentences_server.api.app import create_app from sixsentences_server.chat.service import answer_question @@ -69,6 +75,21 @@ def _mini_pdf(text: str) -> bytes: return bytes(out) +def _encrypted_pdf(text: str, *, user_password: str, extra_pages: int = 0) -> bytes: + """`_mini_pdf` with an owner password, encrypted with RC4 so no crypto backend is needed. + + An empty `user_password` is how publishers restrict printing or copying: the + file still opens without a password. A non-empty one locks it. + """ + writer = pypdf.PdfWriter(clone_from=pypdf.PdfReader(BytesIO(_mini_pdf(text)))) + for _ in range(extra_pages): + writer.add_blank_page() + writer.encrypt(user_password=user_password, owner_password="owner", algorithm="RC4-128") + out = BytesIO() + writer.write(out) + return out.getvalue() + + class _StubOA: """OpenAlex stub: fixed work for id lookups, fixed hits for searches.""" @@ -903,6 +924,83 @@ def test_ingest_rejects_non_pdf(settings: Settings) -> None: ) +def test_ingest_accepts_a_permission_encrypted_pdf_and_extracts_its_text( + settings: Settings, +) -> None: + pdf = _encrypted_pdf( + "Restricted Printing Study of Terraform Drift in Regulated Fleets", user_password="" + ) + assert pypdf.PdfReader(BytesIO(pdf)).is_encrypted + init_db() + with db_session() as session: + org = get_default_org(session) + doc = ingest_document( + session, + org_id=org.id, + run_id=None, + content=pdf, + filename="restricted.pdf", + oa_client=_StubOA(), + ) + assert doc.text_status == "parsed" + work = session.get(WorkRow, doc.work_id) + assert work is not None and "Terraform Drift" in work.title + + +def test_ingest_still_refuses_a_pdf_that_needs_a_password(settings: Settings) -> None: + pdf = _encrypted_pdf("Locked article text", user_password="reader-secret") + init_db() + with db_session() as session: + org = get_default_org(session) + with pytest.raises(UploadError, match="^encrypted PDFs are not supported$"): + ingest_document( + session, + org_id=org.id, + run_id=None, + content=pdf, + filename="locked.pdf", + oa_client=_StubOA(), + ) + + +def test_permission_encrypted_pdf_still_meets_the_page_limit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("sixsentences_server.acquisition.upload._MAX_PDF_PAGES", 1) + pdf = _encrypted_pdf("Two page article", user_password="", extra_pages=1) + + with pytest.raises(UploadError, match="1-page safety limit"): + validate_pdf_structure(pdf) + + +def test_permission_encrypted_pdf_still_needs_a_resolvable_page_tree() -> None: + # The page tree counts one kid that is not a /Page, so walking the decrypted + # tree fails. The page-tree checks must run after decryption as before it. + writer = pypdf.PdfWriter(clone_from=pypdf.PdfReader(BytesIO(_mini_pdf("Odd page")))) + writer.pages[0][NameObject("/Type")] = NameObject("/Template") + writer.encrypt(user_password="", owner_password="owner", algorithm="RC4-128") + out = BytesIO() + writer.write(out) + + with pytest.raises(UploadError, match="not a structurally valid PDF"): + validate_pdf_structure(out.getvalue()) + + +def test_encrypted_pdf_whose_content_cannot_be_decrypted_here_is_refused( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # AES content needs an optional pypdf crypto backend; without one, the password + # check passes and decrypting the content raises DependencyError. + def no_backend(self: pypdf.PageObject) -> None: + raise DependencyError("cryptography>=3.1 is required for AES algorithm") + + monkeypatch.setattr(pypdf.PageObject, "get_contents", no_backend) + pdf = _encrypted_pdf("AES article text", user_password="") + + with pytest.raises(UploadError, match="^encrypted PDFs are not supported$"): + validate_pdf_structure(pdf) + + def _upload(client: TestClient, run_id: int, pdf: bytes, name: str = "paper.pdf") -> dict: resp = client.post( f"/runs/{run_id}/documents",