diff --git a/CHANGELOG.md b/CHANGELOG.md index 769dd4b..377c59b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ equivalents for Python package metadata. ### Fixed +- Flag only retractions in the community API's retraction check. The API's copy of the + Retraction Watch loader counted every row of the dataset, so a paper that was only + corrected, or carried an expression of concern or a reinstatement notice, was flagged + as retracted, ranked down, and reported by the living-review monitor. It now keeps + only `RetractionNature` "Retraction", as the engine's loader already did. - Report how old the cached Retraction Watch data is. `load_retractions()` returns the retracted DOIs with the time the cache was written, so a "no retractions found" can be told apart from one checked against a six-month-old copy: `describe()` gives diff --git a/services/api/COMMUNITY_EXPORT_MANIFEST.json b/services/api/COMMUNITY_EXPORT_MANIFEST.json index 4650097..e65d8c2 100644 --- a/services/api/COMMUNITY_EXPORT_MANIFEST.json +++ b/services/api/COMMUNITY_EXPORT_MANIFEST.json @@ -379,10 +379,10 @@ "sha256": "b9fb3da235e1fe8205aedb233f7a85d4c04f22a32138941e0787c2d88bae747d" }, { - "bytes": 1489, + "bytes": 1799, "mode": "0644", "path": "src/sixsentences_server/connectors/retractions.py", - "sha256": "36f8ade2dbd40e832c04adb37f2a04b90f07fd9fb34d63ea195e91d33c18b8c7" + "sha256": "38b6e1c0aef70020bfa1ef793c490b4bfe69fb76038617319b1aea7bb8e2ea77" }, { "bytes": 3988, @@ -2220,6 +2220,12 @@ "path": "tests/test_result_working_set.py", "sha256": "cb5a3174fe89faf6c18c4a8ab367d83ca9ddb81e077819c558b61f35fbb86f84" }, + { + "bytes": 1122, + "mode": "0644", + "path": "tests/test_retractions_connector.py", + "sha256": "14969e9b70dd0c92e79f1564d456719f628b4e4da20e45b42ea80737cd812a3d" + }, { "bytes": 1204, "mode": "0644", diff --git a/services/api/src/sixsentences_server/connectors/retractions.py b/services/api/src/sixsentences_server/connectors/retractions.py index ea49e8e..22fb1ca 100644 --- a/services/api/src/sixsentences_server/connectors/retractions.py +++ b/services/api/src/sixsentences_server/connectors/retractions.py @@ -28,14 +28,21 @@ def download_retractions(data_dir: Path, *, url: str = RETRACTION_WATCH_CSV_URL) def load_retracted_dois(data_dir: Path) -> set[str]: - """Return lowercased retracted DOIs; empty set if data not downloaded.""" + """Return lowercased retracted DOIs; empty set if data not downloaded. + + Retraction Watch also lists corrections, expressions of concern and + reinstatements; only rows whose ``RetractionNature`` is ``Retraction`` + count, as in the engine's connector. + """ path = data_dir / CACHE_FILE if not path.exists(): return set() dois: set[str] = set() - with path.open(newline="", encoding="utf-8", errors="replace") as fh: + with path.open(newline="", encoding="utf-8-sig", errors="replace") as fh: reader = csv.DictReader(fh) for row in reader: + if (row.get("RetractionNature") or "").strip().casefold() != "retraction": + continue doi = (row.get("OriginalPaperDOI") or "").strip().lower() if doi and doi != "unavailable": dois.add(doi.removeprefix("https://doi.org/")) diff --git a/services/api/tests/test_retractions_connector.py b/services/api/tests/test_retractions_connector.py new file mode 100644 index 0000000..0471d31 --- /dev/null +++ b/services/api/tests/test_retractions_connector.py @@ -0,0 +1,32 @@ +"""The Retraction Watch loader flags retractions, not every notice in the dataset.""" + +from pathlib import Path + +from sixsentences_server.connectors.retractions import CACHE_FILE, load_retracted_dois + + +def test_only_retractions_are_flagged(tmp_path: Path) -> None: + (tmp_path / CACHE_FILE).write_text( + "Record ID,OriginalPaperDOI,RetractionNature\n" + "1,10.1/retracted,Retraction\n" + "2,10.1/corrected,Correction\n" + "3,10.1/concern,Expression of concern\n" + "4,10.1/reinstated,Reinstatement\n" + "5,https://doi.org/10.1/SPACED, retraction \n" + "6,unavailable,Retraction\n", + encoding="utf-8", + ) + + assert load_retracted_dois(tmp_path) == {"10.1/retracted", "10.1/spaced"} + + +def test_a_byte_order_mark_does_not_hide_the_first_column(tmp_path: Path) -> None: + (tmp_path / CACHE_FILE).write_bytes( + b"\xef\xbb\xbfOriginalPaperDOI,RetractionNature\n10.1/BOM,Retraction\n" + ) + + assert load_retracted_dois(tmp_path) == {"10.1/bom"} + + +def test_missing_data_yields_no_dois(tmp_path: Path) -> None: + assert load_retracted_dois(tmp_path) == set()