From 752009cef1979ca7c152f63feffb74c7f7989799 Mon Sep 17 00:00:00 2001 From: Irfan Ahmad Date: Tue, 21 Jul 2026 21:00:03 +0500 Subject: [PATCH 1/2] feat: teach find_python_dependencies to scan pyproject.toml and uv.lock find_python_dependencies previously only understood pip-compile style requirements.txt files (via requirements-parser). Repos migrating from pip-compile to pyproject.toml + uv (openedx/public-engineering#506) have no flat requirements file left for it to scan, which forced openedx-platform to disable its Check Python Dependencies workflow entirely (openedx/openedx-platform#38915, tracked in openedx/repo-tools#725). iter_requirement_names() now detects the input file by name: - uv.lock: reads the fully-resolved [[package]] list, matching the same direct+transitive closure a pip-compile'd requirements.txt used to represent. - pyproject.toml: reads [project.dependencies], [project.optional-dependencies], and [dependency-groups], resolving {include-group = "..."} references. - anything else: falls back to the existing requirements.txt parsing, unchanged. Verified against openedx-platform's actual pyproject.toml (202 names) and uv.lock (445 names). Closes #725 --- .../find_python_dependencies.py | 87 +++++++++++-- pyproject.toml | 1 + tests/test_find_python_dependencies.py | 115 ++++++++++++++++++ uv.lock | 2 + 4 files changed, 194 insertions(+), 11 deletions(-) create mode 100644 tests/test_find_python_dependencies.py diff --git a/edx_repo_tools/find_dependencies/find_python_dependencies.py b/edx_repo_tools/find_dependencies/find_python_dependencies.py index 902f4e26..eb530e28 100644 --- a/edx_repo_tools/find_dependencies/find_python_dependencies.py +++ b/edx_repo_tools/find_dependencies/find_python_dependencies.py @@ -8,7 +8,9 @@ import os import requirements import sys +import tomllib from pathlib import Path +from packaging.requirements import Requirement import requests @@ -20,9 +22,9 @@ def request_package_info_url(package): if response.status_code == 200: data_dict = response.json() info = data_dict["info"] - return info["home_page"] + return info["home_page"] else: - print(f"Failed to retrieve data for package {package}. Status code:", response.status_code) + print(f"Failed to retrieve data for package {package}. Status code:", response.status_code) FIRST_PARTY_ORGS = ["openedx"] @@ -42,14 +44,78 @@ def urls_in_orgs(urls, orgs): if any(f"/{org}/" in url for org in orgs) ) + +def _dependency_group_names(group, all_groups, seen=None): + """ + Yield package names from a [dependency-groups] entry, resolving any + {include-group = "..."} references to the group they point at. + """ + if seen is None: + seen = set() + for item in group: + if isinstance(item, str): + yield Requirement(item).name + elif isinstance(item, dict) and "include-group" in item: + included = item["include-group"] + if included in seen: + continue + seen.add(included) + yield from _dependency_group_names(all_groups.get(included, []), all_groups, seen) + + +def _names_from_pyproject_toml(data): + """ + Yield package names declared in a pyproject.toml's [project.dependencies], + [project.optional-dependencies], and [dependency-groups]. + """ + project = data.get("project", {}) + for dep in project.get("dependencies", []): + yield Requirement(dep).name + for extra_deps in project.get("optional-dependencies", {}).values(): + for dep in extra_deps: + yield Requirement(dep).name + all_groups = data.get("dependency-groups", {}) + for group in all_groups.values(): + yield from _dependency_group_names(group, all_groups) + + +def _names_from_uv_lock(data): + """ + Yield package names from a uv.lock's fully-resolved [[package]] list. + This covers the same direct+transitive dependency closure that a + pip-compile'd requirements.txt used to represent. + """ + for package in data.get("package", []): + name = package.get("name") + if name: + yield name + + +def iter_requirement_names(path): + """ + Yield package names declared in `path`, which may be a pip-compile style + requirements.txt, a pyproject.toml, or a uv.lock. + """ + path = Path(path) + if path.name == "uv.lock": + yield from _names_from_uv_lock(tomllib.loads(path.read_text())) + elif path.name == "pyproject.toml": + yield from _names_from_pyproject_toml(tomllib.loads(path.read_text())) + else: + with open(path) as freq: + for req in requirements.parse(freq): + yield req.name + + @click.command() @click.option( '--req-file', 'directories', multiple=True, required=True, - help="The absolute file paths to locate Python dependencies" - "within a particular repository. You can provide this " - "option multiple times to include multiple requirement files.", + help="The absolute file paths to locate Python dependencies " + "within a particular repository. Accepts pip-compile style " + "requirements.txt files, pyproject.toml, or uv.lock. You can " + "provide this option multiple times to include multiple files.", ) @click.option( '--ignore', 'ignore_paths', @@ -60,16 +126,15 @@ def urls_in_orgs(urls, orgs): def main(directories=None, ignore_paths=None): """ - Analyze the requirements in input directory mentioned on the command line. + Analyze the requirements in input directory mentioned on the command line. """ home_page = set() for directory in directories: - with open(directory) as fbase: - for req in requirements.parse(fbase): - url = request_package_info_url(req.name) - if url is not None: - home_page.add(url) + for name in set(iter_requirement_names(directory)): + url = request_package_info_url(name) + if url is not None: + home_page.add(url) packages_urls = set(urls_in_orgs(home_page, SECOND_PARTY_ORGS)) diff --git a/pyproject.toml b/pyproject.toml index 4ed0366a..a2564529 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ conventional_commits = [ "pandas", ] find_dependencies = [ + "packaging", "requests", "requirements-parser", "rich", diff --git a/tests/test_find_python_dependencies.py b/tests/test_find_python_dependencies.py new file mode 100644 index 00000000..04e8fe06 --- /dev/null +++ b/tests/test_find_python_dependencies.py @@ -0,0 +1,115 @@ +from unittest.mock import patch + +from edx_repo_tools.find_dependencies.find_python_dependencies import ( + iter_requirement_names, + main, +) + + +def test_iter_requirement_names_requirements_txt(tmp_path): + req_file = tmp_path / "base.txt" + req_file.write_text( + "Django==4.2.1\n" + "git+https://github.com/mitodl/edx-sga.git@abc123#egg=edx-sga\n" + ) + assert sorted(iter_requirement_names(req_file)) == ["Django", "edx-sga"] + + +def test_iter_requirement_names_pyproject_toml(tmp_path): + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text( + """ + [project] + name = "sample" + version = "0.1" + dependencies = [ + "lxml[html_clean]", + "edx-sga @ git+https://github.com/mitodl/edx-sga.git@abc123", + ] + + [project.optional-dependencies] + extra = ["requests"] + + [dependency-groups] + test-base = ["pytest"] + test = [{include-group = "test-base"}, "responses"] + """ + ) + names = set(iter_requirement_names(pyproject)) + assert names == {"lxml", "edx-sga", "requests", "pytest", "responses"} + + +def test_iter_requirement_names_uv_lock(tmp_path): + uv_lock = tmp_path / "uv.lock" + uv_lock.write_text( + """ + version = 1 + + [[package]] + name = "lxml" + version = "5.3.2" + + [[package]] + name = "edx-sga" + version = "0.1" + """ + ) + assert sorted(iter_requirement_names(uv_lock)) == ["edx-sga", "lxml"] + + +def test_main_flags_second_party_dependency_from_uv_lock(tmp_path): + uv_lock = tmp_path / "uv.lock" + uv_lock.write_text( + """ + version = 1 + + [[package]] + name = "edx-sga" + version = "0.1" + + [[package]] + name = "django" + version = "4.2.1" + """ + ) + + def fake_request_package_info_url(package): + return { + "edx-sga": "https://github.com/mitodl/edx-sga", + "django": "https://github.com/django/django", + }.get(package) + + with patch( + "edx_repo_tools.find_dependencies.find_python_dependencies.request_package_info_url", + side_effect=fake_request_package_info_url, + ): + with patch( + "edx_repo_tools.find_dependencies.find_python_dependencies.exit" + ) as mock_exit: + main.callback(directories=[str(uv_lock)], ignore_paths=[]) + mock_exit.assert_called_once_with(1) + + +def test_main_respects_ignore_list_with_pyproject_toml(tmp_path): + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text( + """ + [project] + name = "sample" + version = "0.1" + dependencies = ["edx-sga"] + """ + ) + + with patch( + "edx_repo_tools.find_dependencies.find_python_dependencies.request_package_info_url", + return_value="https://github.com/mitodl/edx-sga", + ): + with patch( + "edx_repo_tools.find_dependencies.find_python_dependencies.exit" + ) as mock_exit: + main.callback( + directories=[str(pyproject)], + ignore_paths=["https://github.com/mitodl/edx-sga"], + ) + mock_exit.assert_not_called() diff --git a/uv.lock b/uv.lock index 423d9dd4..66832133 100644 --- a/uv.lock +++ b/uv.lock @@ -541,6 +541,7 @@ conventional-commits = [ { name = "pandas" }, ] find-dependencies = [ + { name = "packaging" }, { name = "requests" }, { name = "requirements-parser" }, { name = "rich" }, @@ -589,6 +590,7 @@ requires-dist = [ { name = "lockfile" }, { name = "matplotlib", marker = "extra == 'conventional-commits'" }, { name = "more-itertools" }, + { name = "packaging", marker = "extra == 'find-dependencies'" }, { name = "packaging", marker = "extra == 'pull-request-creator'" }, { name = "pandas", marker = "extra == 'conventional-commits'" }, { name = "path-py" }, From 2609cb9a2059ecfb820aba7e7f77c1d2af3f1371 Mon Sep 17 00:00:00 2001 From: Irfan Ahmad Date: Thu, 6 Aug 2026 15:23:37 +0500 Subject: [PATCH 2/2] fix: drop pyproject.toml scanning, keep uv.lock; bump to 4.1.0 Per review (feanil, PR #735): scanning pyproject.toml's [project.dependencies]/[dependency-groups] only surfaces direct dependencies, not the resolved transitive closure -- unlike uv.lock or a pip-compile'd requirements.txt, both of which represent a fully resolved dependency graph. Supporting pyproject.toml directly would be misleading (callers could assume this tool resolves dependencies when it doesn't) and isn't needed anyway, since repos migrating off pip-compile land on uv.lock as their fully-resolved file. Removed _names_from_pyproject_toml/_dependency_group_names and the pyproject.toml branch in iter_requirement_names(), the now-unused `packaging` dependency (only import was in the removed code) and its two uv.lock entries, and the two tests exercising pyproject.toml support. uv.lock and requirements.txt support is unchanged. Bumped __version__ to 4.1.0 (new capability: uv.lock support) per review request, ready to merge and release. --- edx_repo_tools/__init__.py | 2 +- .../find_python_dependencies.py | 46 +++-------------- pyproject.toml | 1 - tests/test_find_python_dependencies.py | 49 ------------------- uv.lock | 2 - 5 files changed, 8 insertions(+), 92 deletions(-) diff --git a/edx_repo_tools/__init__.py b/edx_repo_tools/__init__.py index ce1305bf..70397087 100644 --- a/edx_repo_tools/__init__.py +++ b/edx_repo_tools/__init__.py @@ -1 +1 @@ -__version__ = "4.0.0" +__version__ = "4.1.0" diff --git a/edx_repo_tools/find_dependencies/find_python_dependencies.py b/edx_repo_tools/find_dependencies/find_python_dependencies.py index eb530e28..8f596826 100644 --- a/edx_repo_tools/find_dependencies/find_python_dependencies.py +++ b/edx_repo_tools/find_dependencies/find_python_dependencies.py @@ -10,7 +10,6 @@ import sys import tomllib from pathlib import Path -from packaging.requirements import Requirement import requests @@ -45,40 +44,6 @@ def urls_in_orgs(urls, orgs): ) -def _dependency_group_names(group, all_groups, seen=None): - """ - Yield package names from a [dependency-groups] entry, resolving any - {include-group = "..."} references to the group they point at. - """ - if seen is None: - seen = set() - for item in group: - if isinstance(item, str): - yield Requirement(item).name - elif isinstance(item, dict) and "include-group" in item: - included = item["include-group"] - if included in seen: - continue - seen.add(included) - yield from _dependency_group_names(all_groups.get(included, []), all_groups, seen) - - -def _names_from_pyproject_toml(data): - """ - Yield package names declared in a pyproject.toml's [project.dependencies], - [project.optional-dependencies], and [dependency-groups]. - """ - project = data.get("project", {}) - for dep in project.get("dependencies", []): - yield Requirement(dep).name - for extra_deps in project.get("optional-dependencies", {}).values(): - for dep in extra_deps: - yield Requirement(dep).name - all_groups = data.get("dependency-groups", {}) - for group in all_groups.values(): - yield from _dependency_group_names(group, all_groups) - - def _names_from_uv_lock(data): """ Yield package names from a uv.lock's fully-resolved [[package]] list. @@ -94,13 +59,16 @@ def _names_from_uv_lock(data): def iter_requirement_names(path): """ Yield package names declared in `path`, which may be a pip-compile style - requirements.txt, a pyproject.toml, or a uv.lock. + requirements.txt or a uv.lock. Both represent a fully-resolved + direct+transitive dependency closure. A pyproject.toml is deliberately + not supported here: [project.dependencies]/[dependency-groups] only list + direct dependencies, so scanning it would silently miss transitive ones + and misleadingly suggest this tool resolves dependencies, which it does + not. """ path = Path(path) if path.name == "uv.lock": yield from _names_from_uv_lock(tomllib.loads(path.read_text())) - elif path.name == "pyproject.toml": - yield from _names_from_pyproject_toml(tomllib.loads(path.read_text())) else: with open(path) as freq: for req in requirements.parse(freq): @@ -114,7 +82,7 @@ def iter_requirement_names(path): required=True, help="The absolute file paths to locate Python dependencies " "within a particular repository. Accepts pip-compile style " - "requirements.txt files, pyproject.toml, or uv.lock. You can " + "requirements.txt files or uv.lock. You can " "provide this option multiple times to include multiple files.", ) @click.option( diff --git a/pyproject.toml b/pyproject.toml index a2564529..4ed0366a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,6 @@ conventional_commits = [ "pandas", ] find_dependencies = [ - "packaging", "requests", "requirements-parser", "rich", diff --git a/tests/test_find_python_dependencies.py b/tests/test_find_python_dependencies.py index 04e8fe06..49cd9528 100644 --- a/tests/test_find_python_dependencies.py +++ b/tests/test_find_python_dependencies.py @@ -15,30 +15,6 @@ def test_iter_requirement_names_requirements_txt(tmp_path): assert sorted(iter_requirement_names(req_file)) == ["Django", "edx-sga"] -def test_iter_requirement_names_pyproject_toml(tmp_path): - pyproject = tmp_path / "pyproject.toml" - pyproject.write_text( - """ - [project] - name = "sample" - version = "0.1" - dependencies = [ - "lxml[html_clean]", - "edx-sga @ git+https://github.com/mitodl/edx-sga.git@abc123", - ] - - [project.optional-dependencies] - extra = ["requests"] - - [dependency-groups] - test-base = ["pytest"] - test = [{include-group = "test-base"}, "responses"] - """ - ) - names = set(iter_requirement_names(pyproject)) - assert names == {"lxml", "edx-sga", "requests", "pytest", "responses"} - - def test_iter_requirement_names_uv_lock(tmp_path): uv_lock = tmp_path / "uv.lock" uv_lock.write_text( @@ -88,28 +64,3 @@ def fake_request_package_info_url(package): ) as mock_exit: main.callback(directories=[str(uv_lock)], ignore_paths=[]) mock_exit.assert_called_once_with(1) - - -def test_main_respects_ignore_list_with_pyproject_toml(tmp_path): - pyproject = tmp_path / "pyproject.toml" - pyproject.write_text( - """ - [project] - name = "sample" - version = "0.1" - dependencies = ["edx-sga"] - """ - ) - - with patch( - "edx_repo_tools.find_dependencies.find_python_dependencies.request_package_info_url", - return_value="https://github.com/mitodl/edx-sga", - ): - with patch( - "edx_repo_tools.find_dependencies.find_python_dependencies.exit" - ) as mock_exit: - main.callback( - directories=[str(pyproject)], - ignore_paths=["https://github.com/mitodl/edx-sga"], - ) - mock_exit.assert_not_called() diff --git a/uv.lock b/uv.lock index 66832133..423d9dd4 100644 --- a/uv.lock +++ b/uv.lock @@ -541,7 +541,6 @@ conventional-commits = [ { name = "pandas" }, ] find-dependencies = [ - { name = "packaging" }, { name = "requests" }, { name = "requirements-parser" }, { name = "rich" }, @@ -590,7 +589,6 @@ requires-dist = [ { name = "lockfile" }, { name = "matplotlib", marker = "extra == 'conventional-commits'" }, { name = "more-itertools" }, - { name = "packaging", marker = "extra == 'find-dependencies'" }, { name = "packaging", marker = "extra == 'pull-request-creator'" }, { name = "pandas", marker = "extra == 'conventional-commits'" }, { name = "path-py" },