fix(api): flag only retractions in the retraction check

CHANGELOG.md

@@ -10,6 +10,11 @@ equivalents for Python package metadata.
 
 ### Fixed
 
+- Flag only retractions in the community API's retraction check. The API's copy of the
+  Retraction Watch loader counted every row of the dataset, so a paper that was only
+  corrected, or carried an expression of concern or a reinstatement notice, was flagged
+  as retracted, ranked down, and reported by the living-review monitor. It now keeps
+  only `RetractionNature` "Retraction", as the engine's loader already did.
 - Report how old the cached Retraction Watch data is. `load_retractions()` returns the
   retracted DOIs with the time the cache was written, so a "no retractions found" can be
   told apart from one checked against a six-month-old copy: `describe()` gives

services/api/COMMUNITY_EXPORT_MANIFEST.json

@@ -379,10 +379,10 @@
       "sha256": "b9fb3da235e1fe8205aedb233f7a85d4c04f22a32138941e0787c2d88bae747d"
     },
     {
-      "bytes": 1489,
+      "bytes": 1799,
       "mode": "0644",
       "path": "src/sixsentences_server/connectors/retractions.py",
-      "sha256": "36f8ade2dbd40e832c04adb37f2a04b90f07fd9fb34d63ea195e91d33c18b8c7"
+      "sha256": "38b6e1c0aef70020bfa1ef793c490b4bfe69fb76038617319b1aea7bb8e2ea77"
     },
     {
       "bytes": 3988,
@@ -2220,6 +2220,12 @@
       "path": "tests/test_result_working_set.py",
       "sha256": "cb5a3174fe89faf6c18c4a8ab367d83ca9ddb81e077819c558b61f35fbb86f84"
     },
+    {
+      "bytes": 1122,
+      "mode": "0644",
+      "path": "tests/test_retractions_connector.py",
+      "sha256": "14969e9b70dd0c92e79f1564d456719f628b4e4da20e45b42ea80737cd812a3d"
+    },
     {
       "bytes": 1204,
       "mode": "0644",

services/api/src/sixsentences_server/connectors/retractions.py

@@ -28,14 +28,21 @@ def download_retractions(data_dir: Path, *, url: str = RETRACTION_WATCH_CSV_URL)
 
 
 def load_retracted_dois(data_dir: Path) -> set[str]:
-    """Return lowercased retracted DOIs; empty set if data not downloaded."""
+    """Return lowercased retracted DOIs; empty set if data not downloaded.
+
+    Retraction Watch also lists corrections, expressions of concern and
+    reinstatements; only rows whose ``RetractionNature`` is ``Retraction``
+    count, as in the engine's connector.
+    """
     path = data_dir / CACHE_FILE
     if not path.exists():
         return set()
     dois: set[str] = set()
-    with path.open(newline="", encoding="utf-8", errors="replace") as fh:
+    with path.open(newline="", encoding="utf-8-sig", errors="replace") as fh:
         reader = csv.DictReader(fh)
         for row in reader:
+            if (row.get("RetractionNature") or "").strip().casefold() != "retraction":
+                continue
             doi = (row.get("OriginalPaperDOI") or "").strip().lower()
             if doi and doi != "unavailable":
                 dois.add(doi.removeprefix("https://doi.org/"))

services/api/tests/test_retractions_connector.py

@@ -0,0 +1,32 @@
+"""The Retraction Watch loader flags retractions, not every notice in the dataset."""
+
+from pathlib import Path
+
+from sixsentences_server.connectors.retractions import CACHE_FILE, load_retracted_dois
+
+
+def test_only_retractions_are_flagged(tmp_path: Path) -> None:
+    (tmp_path / CACHE_FILE).write_text(
+        "Record ID,OriginalPaperDOI,RetractionNature\n"
+        "1,10.1/retracted,Retraction\n"
+        "2,10.1/corrected,Correction\n"
+        "3,10.1/concern,Expression of concern\n"
+        "4,10.1/reinstated,Reinstatement\n"
+        "5,https://doi.org/10.1/SPACED, retraction \n"
+        "6,unavailable,Retraction\n",
+        encoding="utf-8",
+    )
+
+    assert load_retracted_dois(tmp_path) == {"10.1/retracted", "10.1/spaced"}
+
+
+def test_a_byte_order_mark_does_not_hide_the_first_column(tmp_path: Path) -> None:
+    (tmp_path / CACHE_FILE).write_bytes(
+        b"\xef\xbb\xbfOriginalPaperDOI,RetractionNature\n10.1/BOM,Retraction\n"
+    )
+
+    assert load_retracted_dois(tmp_path) == {"10.1/bom"}
+
+
+def test_missing_data_yields_no_dois(tmp_path: Path) -> None:
+    assert load_retracted_dois(tmp_path) == set()