Skip to content

fix: bridge pip-compile .txt and pyproject.toml/uv.lock for openedx Python builds #128

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
blarghmatey merged 4 commits into from
Jul 31, 2026
Merged

fix: bridge pip-compile .txt and pyproject.toml/uv.lock for openedx Python builds #128

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
235b4da
fix: source edx-platform requirements from uv.lock when present
blarghmatey Jul 23, 2026
da5269d
fix: install edx-platform deps via uv sync, add a dev target
blarghmatey Jul 23, 2026
8c1b9c9
fix: preserve codejail service deps through sandbox uv sync fallback
blarghmatey Jul 23, 2026
d05c893
refactor: make the uv.lock/pip-compile bridge default for every pytho…
blarghmatey Jul 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 44 additions & 10 deletions src/lehrer/core/codejail.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
"""Generic codejail service build for Open edX operators."""

import shlex

import dagger
from dagger import dag, function, object_type

Expand Down Expand Up @@ -95,6 +97,7 @@ async def _build(
"-y",
"--no-install-recommends",
"build-essential",
"curl",
"python3-virtualenv",
"python3-pip",
"git",
Expand Down Expand Up @@ -172,22 +175,53 @@ async def _build(
["pip", "install", "--no-cache-dir", "-r", "requirements/base.txt"]
).with_exec(["pip", "install", "--no-cache-dir", "gunicorn"])

# Install edx-platform sandbox requirements in virtualenv
# The URL pattern differs based on whether it's a release or master
sandbox_req_url = (
f"https://raw.githubusercontent.com/openedx/edx-platform/master/requirements/edx-sandbox/releases/{release_name}.txt"
if release_name != "master"
else "https://raw.githubusercontent.com/openedx/edx-platform/master/requirements/edx-sandbox/base.txt"
)

import shlex
# Install edx-platform sandbox requirements in virtualenv.
#
# Named releases haven't picked up the pip-compile -> uv migration
# and keep publishing a per-release export; read it directly as
# before. master's sandbox is now its own standalone uv project
# (requirements/edx-sandbox/{pyproject.toml,uv.lock}), with
# requirements/edx-sandbox/base.txt kept only as a temporary
# machine-generated compat export slated for removal once known
# consumers (including this one) stop reading it -- see
# openedx/public-engineering#552. Try the compat export first (fast,
# no uv needed) and fall back to a uv export of the sub-project once
# it's gone, so this keeps working across that removal without a
# lehrer change.
if release_name != "master":
sandbox_req_url = (
"https://raw.githubusercontent.com/openedx/edx-platform/master/"
f"requirements/edx-sandbox/releases/{release_name}.txt"
)
install_sandbox_reqs = (
f"pip install --no-cache-dir -r {shlex.quote(sandbox_req_url)}"
)
else:
sandbox_base_url = (
"https://raw.githubusercontent.com/openedx/edx-platform/master/"
"requirements/edx-sandbox"
)
install_sandbox_reqs = (
"set -eu && "
"if curl -fsSL -o /tmp/edx_sandbox.txt "
f"{shlex.quote(sandbox_base_url + '/base.txt')}; then :; else "
"mkdir -p /tmp/edx-sandbox-uv && cd /tmp/edx-sandbox-uv && "
"curl -fsSL -o pyproject.toml "
f"{shlex.quote(sandbox_base_url + '/pyproject.toml')} && "
f"curl -fsSL -o uv.lock {shlex.quote(sandbox_base_url + '/uv.lock')} && "
"pip install --quiet uv && "
"uv export --frozen --no-hashes --no-emit-project "
"-o /tmp/edx_sandbox.txt; "
Comment thread
blarghmatey marked this conversation as resolved.
Outdated
"fi && "
"pip install --no-cache-dir -r /tmp/edx_sandbox.txt"
)

container = container.with_exec(
[
"bash",
"-c",
f"source /sandbox/venv/bin/activate && "
f"pip install --no-cache-dir -r {shlex.quote(sandbox_req_url)} && "
f"{install_sandbox_reqs} && "
f"deactivate",
]
)
Expand Down
79 changes: 65 additions & 14 deletions src/lehrer/core/platform.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -457,6 +457,53 @@ def _derive_test_settings(service: str) -> str:
)


def _edx_base_requirements_script(*, base_dest: str, assets_dest: str) -> str:
"""Shell script that sources edx-platform's base + assets requirements.

openedx-platform is mid-migration from pip-compile (``requirements/edx/
{base,assets}.txt``) to ``pyproject.toml``/``uv.lock`` as the source of
truth; the ``.txt`` compat exports are slated for removal once known
consumers move off them (openedx/public-engineering#552). Named releases
haven't picked up the migration and won't grow a ``uv.lock`` until they
do, so branching on ``uv.lock``'s presence -- rather than hardcoding
``release_name == "master"`` -- keeps both tracks working, and picks up
master's migration automatically without a lehrer change once it lands.

Must run with cwd (or an explicit ``cd``) at the edx-platform checkout.
"""
return (
"set -eu\n"
"cd /openedx/edx-platform\n"
"if [ -f uv.lock ]; then\n"
" uv export --frozen --no-hashes --no-emit-project"
f" --no-default-groups -o {base_dest}\n"
" uv export --frozen --no-hashes --no-emit-project"
f" --no-default-groups --only-group assets -o {assets_dest}\n"
"else\n"
f" cp requirements/edx/base.txt {base_dest}\n"
f" cp requirements/edx/assets.txt {assets_dest}\n"
"fi\n"
)


def _edx_testing_requirements_script(dest: str) -> str:
"""Shell script that sources edx-platform's test-suite requirements.

Same uv.lock-presence branch as ``_edx_base_requirements_script`` — see
its docstring and openedx/public-engineering#552.
"""
return (
"set -eu\n"
"cd /openedx/edx-platform\n"
"if [ -f uv.lock ]; then\n"
" uv export --frozen --no-hashes --no-emit-project"
f" --no-default-groups --only-group testing -o {dest}\n"
"else\n"
f" cp requirements/edx/testing.txt {dest}\n"
"fi\n"
)


@object_type
class OpenedxPlatform:
"""Generic edx-platform build pipeline.
Expand Down Expand Up @@ -723,19 +770,16 @@ def install_deps(
.with_mounted_directory(
"/root/pip_package_overrides", pip_package_overrides
)
# Copy base requirements from edx-platform
# Source base + assets requirements from edx-platform (uv.lock
# export or legacy pip-compile .txt, whichever the checkout has).
.with_exec(
[
"sh",
"-c",
"cp /openedx/edx-platform/requirements/edx/base.txt /root/pip_package_lists/edx_base.txt",
]
)
.with_exec(
[
"sh",
"-c",
"cp /openedx/edx-platform/requirements/edx/assets.txt /root/pip_package_lists/edx_assets.txt",
_edx_base_requirements_script(
base_dest="/root/pip_package_lists/edx_base.txt",
assets_dest="/root/pip_package_lists/edx_assets.txt",
),
]
)
# Install base Python dependencies using uv (much faster than pip)
Expand Down Expand Up @@ -1585,10 +1629,10 @@ def _python_only_env(
[
"sh",
"-c",
"cp /openedx/edx-platform/requirements/edx/base.txt"
" /root/pip_package_lists/edx_base.txt"
" && cp /openedx/edx-platform/requirements/edx/assets.txt"
" /root/pip_package_lists/edx_assets.txt",
_edx_base_requirements_script(
base_dest="/root/pip_package_lists/edx_base.txt",
assets_dest="/root/pip_package_lists/edx_assets.txt",
),
]
)
)
Expand Down Expand Up @@ -2294,7 +2338,14 @@ async def test( # noqa: PLR0913
container = (
container.with_workdir("/openedx/edx-platform")
.with_exec(["uv", "pip", "install", "-e", "."])
.with_exec(["uv", "pip", "install", "-r", "requirements/edx/testing.txt"])
.with_exec(
[
"sh",
"-c",
_edx_testing_requirements_script("/tmp/edx_testing.txt")
+ "uv pip install -r /tmp/edx_testing.txt\n",
]
)
)

# When folding the plugins' own suites into this run, derive the plugin
Expand Down