Skip to content

fix: bridge pip-compile .txt and pyproject.toml/uv.lock for openedx Python builds #128

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
blarghmatey merged 4 commits into from
Jul 31, 2026
Merged

fix: bridge pip-compile .txt and pyproject.toml/uv.lock for openedx Python builds #128

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
235b4da
fix: source edx-platform requirements from uv.lock when present
blarghmatey Jul 23, 2026
da5269d
fix: install edx-platform deps via uv sync, add a dev target
blarghmatey Jul 23, 2026
8c1b9c9
fix: preserve codejail service deps through sandbox uv sync fallback
blarghmatey Jul 23, 2026
d05c893
refactor: make the uv.lock/pip-compile bridge default for every pytho…
blarghmatey Jul 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 44 additions & 10 deletions src/lehrer/core/codejail.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
"""Generic codejail service build for Open edX operators."""

import shlex

import dagger
from dagger import dag, function, object_type

Expand Down Expand Up @@ -95,6 +97,7 @@ async def _build(
"-y",
"--no-install-recommends",
"build-essential",
"curl",
"python3-virtualenv",
"python3-pip",
"git",
Expand Down Expand Up @@ -172,22 +175,53 @@ async def _build(
["pip", "install", "--no-cache-dir", "-r", "requirements/base.txt"]
).with_exec(["pip", "install", "--no-cache-dir", "gunicorn"])

# Install edx-platform sandbox requirements in virtualenv
# The URL pattern differs based on whether it's a release or master
sandbox_req_url = (
f"https://raw.githubusercontent.com/openedx/edx-platform/master/requirements/edx-sandbox/releases/{release_name}.txt"
if release_name != "master"
else "https://raw.githubusercontent.com/openedx/edx-platform/master/requirements/edx-sandbox/base.txt"
)

import shlex
# Install edx-platform sandbox requirements in virtualenv.
#
# Named releases haven't picked up the pip-compile -> uv migration
# and keep publishing a per-release export; read it directly as
# before. master's sandbox is now its own standalone uv project
# (requirements/edx-sandbox/{pyproject.toml,uv.lock}), with
# requirements/edx-sandbox/base.txt kept only as a temporary
# machine-generated compat export slated for removal once known
# consumers (including this one) stop reading it -- see
# openedx/public-engineering#552. Try the compat export first (fast,
# no uv needed) and fall back to a uv sync of the sub-project once
# it's gone, so this keeps working across that removal without a
# lehrer change.
if release_name != "master":
sandbox_req_url = (
"https://raw.githubusercontent.com/openedx/edx-platform/master/"
f"requirements/edx-sandbox/releases/{release_name}.txt"
)
install_sandbox_reqs = (
f"pip install --no-cache-dir -r {shlex.quote(sandbox_req_url)}"
)
else:
sandbox_base_url = (
"https://raw.githubusercontent.com/openedx/edx-platform/master/"
"requirements/edx-sandbox"
)
install_sandbox_reqs = (
"set -eu && "
"if curl -fsSL -o /tmp/edx_sandbox.txt "
f"{shlex.quote(sandbox_base_url + '/base.txt')}; then "
"pip install --no-cache-dir -r /tmp/edx_sandbox.txt; "
"else "
"mkdir -p /tmp/edx-sandbox-uv && cd /tmp/edx-sandbox-uv && "
"curl -fsSL -o pyproject.toml "
f"{shlex.quote(sandbox_base_url + '/pyproject.toml')} && "
f"curl -fsSL -o uv.lock {shlex.quote(sandbox_base_url + '/uv.lock')} && "
"pip install --quiet uv && "
"uv sync --locked --active --no-install-project; "
Comment thread
blarghmatey marked this conversation as resolved.
Outdated
"fi"
)

container = container.with_exec(
[
"bash",
"-c",
f"source /sandbox/venv/bin/activate && "
f"pip install --no-cache-dir -r {shlex.quote(sandbox_req_url)} && "
f"{install_sandbox_reqs} && "
f"deactivate",
]
)
Expand Down
138 changes: 101 additions & 37 deletions src/lehrer/core/platform.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -457,6 +457,78 @@ def _derive_test_settings(service: str) -> str:
)


def _edx_base_deps_script(*, include_dev: bool = False) -> str:
Comment thread
blarghmatey marked this conversation as resolved.
"""Shell script that installs edx-platform's base + assets (+ dev) deps.

openedx-platform is mid-migration from pip-compile (``requirements/edx/
{base,assets,development}.txt``) to ``pyproject.toml``/``uv.lock`` as the
source of truth; the ``.txt`` compat exports are slated for removal once
known consumers move off them (openedx/public-engineering#552). Named
releases haven't picked up the migration and won't grow a ``uv.lock``
until they do, so branching on ``uv.lock``'s presence -- rather than
hardcoding ``release_name == "master"`` -- keeps both tracks working, and
picks up master's migration automatically without a lehrer change once it
lands.

``--inexact`` on the ``uv sync`` path is load-bearing: callers layer their
own editable installs (e.g. a local django-aqueduct checkout) and the
deployment's own pinned package list on top of this, before or after —
a plain ``uv sync`` prunes anything not in edx-platform's own lock
resolution, which would strip those back out.

``include_dev`` pulls in openedx-platform's ``dev`` dependency group
(mypy, tox, django-debug-toolbar, ...) — or, pre-migration,
``requirements/edx/development.txt`` — for local-dev builds; production
builds leave it out (the default).

Must run with cwd (or an explicit ``cd``) at the edx-platform checkout,
and with ``VIRTUAL_ENV`` set to the target environment.
"""
sync_groups = "--group assets" + (" --group dev" if include_dev else "")
legacy_cp = (
"cp requirements/edx/base.txt /tmp/edx_base.txt\n"
" cp requirements/edx/assets.txt /tmp/edx_assets.txt\n"
)
legacy_files = "/tmp/edx_base.txt /tmp/edx_assets.txt"
if include_dev:
legacy_cp += " cp requirements/edx/development.txt /tmp/edx_development.txt\n"
legacy_files += " /tmp/edx_development.txt"
legacy_install = " ".join(f"-r {path}" for path in legacy_files.split())
return (
"set -eu\n"
"cd /openedx/edx-platform\n"
"if [ -f uv.lock ]; then\n"
" uv sync --locked --active --no-install-project --inexact"
f" --no-default-groups {sync_groups}\n"
"else\n"
f" {legacy_cp}"
f" uv pip install {legacy_install}\n"
"fi\n"
)


def _edx_testing_deps_script() -> str:
"""Shell script that installs edx-platform's test-suite deps.

Same ``uv.lock``-presence branch and ``--inexact`` rationale as
``_edx_base_deps_script`` — see its docstring and
openedx/public-engineering#552. Runs after ``install_deps`` has already
layered the deployment's own packages into the environment, so pruning
those back out on sync is not an option.
"""
return (
"set -eu\n"
"cd /openedx/edx-platform\n"
"if [ -f uv.lock ]; then\n"
" uv sync --locked --active --no-install-project --inexact"
" --no-default-groups --group testing\n"
"else\n"
" cp requirements/edx/testing.txt /tmp/edx_testing.txt\n"
" uv pip install -r /tmp/edx_testing.txt\n"
"fi\n"
)


@object_type
class OpenedxPlatform:
"""Generic edx-platform build pipeline.
Expand Down Expand Up @@ -684,6 +756,7 @@ def install_deps(
packages_to_remove: list[str] | None = None,
extra_npm_packages: list[str] | None = None,
install_node: bool = True,
include_dev_dependencies: bool = False, # noqa: FBT001, FBT002
) -> dagger.Container:
"""Install Python and Node.js dependencies using uv

Expand All @@ -707,6 +780,11 @@ def install_deps(
``False`` for Python-only consumers (e.g. plugin import checks,
settings regeneration) that never build webpack assets — the
Python environment above is complete without it.
include_dev_dependencies: Also install edx-platform's ``dev``
dependency group (mypy, tox, django-debug-toolbar, ...) — or,
pre-migration, ``requirements/edx/development.txt``. Default
``False`` for production builds; set ``True`` for local-dev
images.

Returns:
Container with all dependencies installed
Expand All @@ -723,33 +801,23 @@ def install_deps(
.with_mounted_directory(
"/root/pip_package_overrides", pip_package_overrides
)
# Copy base requirements from edx-platform
.with_exec(
[
"sh",
"-c",
"cp /openedx/edx-platform/requirements/edx/base.txt /root/pip_package_lists/edx_base.txt",
]
)
# Install base + assets (+ dev) deps from edx-platform (uv sync
# against uv.lock, or the legacy pip-compile .txt, whichever the
# checkout has), then layer the deployment's own pinned package
# list on top.
.with_exec(
[
"sh",
"-c",
"cp /openedx/edx-platform/requirements/edx/assets.txt /root/pip_package_lists/edx_assets.txt",
_edx_base_deps_script(include_dev=include_dev_dependencies),
]
)
# Install base Python dependencies using uv (much faster than pip)
# uv automatically uses the VIRTUAL_ENV set in apt_base
.with_exec(
[
"uv",
"pip",
"install",
"-r",
"/root/pip_package_lists/edx_base.txt",
"-r",
"/root/pip_package_lists/edx_assets.txt",
"-r",
f"/root/pip_package_lists/{release_name}/{deployment_name}.txt",
]
)
Expand Down Expand Up @@ -1574,40 +1642,29 @@ def _python_only_env(
edx_platform_git_repo=cell.platform_repo,
edx_platform_git_branch=cell.platform_branch,
)
container = (
container.with_mounted_directory(
"/root/pip_package_lists", cell.pip_package_lists
)
.with_mounted_directory(
"/root/pip_package_overrides", cell.pip_package_overrides
)
.with_exec(
[
"sh",
"-c",
"cp /openedx/edx-platform/requirements/edx/base.txt"
" /root/pip_package_lists/edx_base.txt"
" && cp /openedx/edx-platform/requirements/edx/assets.txt"
" /root/pip_package_lists/edx_assets.txt",
]
)
container = container.with_mounted_directory(
"/root/pip_package_lists", cell.pip_package_lists
).with_mounted_directory(
"/root/pip_package_overrides", cell.pip_package_overrides
)

# Installed *before* the base deps sync/install below so its version
# satisfies the pinned django-aqueduct== constraint and uv skips the
# PyPI fetch. The base step's --inexact flag is what keeps this from
# being pruned back out.
if aqueduct_source is not None:
container = container.with_mounted_directory(
"/root/django-aqueduct", aqueduct_source
).with_exec(["uv", "pip", "install", "-e", "/root/django-aqueduct"])

container = container.with_exec(
["sh", "-c", _edx_base_deps_script()]
).with_exec(
[
"uv",
"pip",
"install",
"-r",
"/root/pip_package_lists/edx_base.txt",
"-r",
"/root/pip_package_lists/edx_assets.txt",
"-r",
f"/root/pip_package_lists/{release_name}/{deployment_name}.txt",
]
)
Expand Down Expand Up @@ -1668,6 +1725,7 @@ async def build_platform(
extra_npm_packages: list[str] | None = None,
verify_boot: bool = True, # noqa: FBT001, FBT002
strict_translations: bool = False, # noqa: FBT001, FBT002
include_dev_dependencies: bool = False, # noqa: FBT001, FBT002
) -> dagger.Container:
"""Build a complete openedx-platform image

Expand Down Expand Up @@ -1717,6 +1775,11 @@ async def build_platform(
pull/compile step fails, instead of warning (default: False —
a missing plugin translation is normal, so this is opt-in until
the per-step baseline is known).
include_dev_dependencies: Also install edx-platform's ``dev``
dependency group (mypy, tox, django-debug-toolbar, ...) — or,
pre-migration, ``requirements/edx/development.txt``. Default
``False`` for production builds; set ``True`` to build a
local-dev image.

Returns:
Container ready to be deployed
Expand Down Expand Up @@ -1828,6 +1891,7 @@ async def build_platform(
node_version=node_version,
packages_to_remove=packages_to_remove,
extra_npm_packages=extra_npm_packages,
include_dev_dependencies=include_dev_dependencies,
)

# ── Clean base ────────────────────────────────────────────────────────
Expand Down Expand Up @@ -2294,7 +2358,7 @@ async def test( # noqa: PLR0913
container = (
container.with_workdir("/openedx/edx-platform")
.with_exec(["uv", "pip", "install", "-e", "."])
.with_exec(["uv", "pip", "install", "-r", "requirements/edx/testing.txt"])
.with_exec(["sh", "-c", _edx_testing_deps_script()])
)

# When folding the plugins' own suites into this run, derive the plugin
Expand Down