fix: bridge pip-compile .txt and pyproject.toml/uv.lock for openedx Python builds

src/lehrer/core/codejail.py

@@ -1,8 +1,12 @@
 """Generic codejail service build for Open edX operators."""
 
+import shlex
+
 import dagger
 from dagger import dag, function, object_type
 
+from lehrer.core.pip_compile_bridge import python_deps_install_script
+
 
 @object_type
 class OpenedxCodejail:
@@ -95,6 +99,7 @@ async def _build(
                     "-y",
                     "--no-install-recommends",
                     "build-essential",
+                    "curl",
                     "python3-virtualenv",
                     "python3-pip",
                     "git",
@@ -144,10 +149,12 @@ async def _build(
             .with_exec(["chown", "-R", "sandbox:sandbox", "/sandbox/venv"])
         )
 
-        # Update PATH to use virtualenv
+        # Update PATH to use virtualenv. VIRTUAL_ENV is also needed by any
+        # `uv sync --active` call below (python_deps_install_script's
+        # uv.lock path) -- it targets that env var, not just PATH.
         container = container.with_env_variable(
             "PATH", "/sandbox/venv/bin:/usr/local/bin:/usr/bin:/bin"
-        )
+        ).with_env_variable("VIRTUAL_ENV", "/sandbox/venv")
 
         # Clone codejail service
         container = container.with_workdir("/codejail").with_exec(
@@ -167,27 +174,74 @@ async def _build(
         sudoers_file = codejail_config.file("01-sandbox")
         container = container.with_file("/etc/sudoers.d/01-sandbox", sudoers_file)
 
-        # Install dependencies
+        # Install dependencies. codejailservice has no uv.lock today (plain
+        # pip-compile requirements/), but sourcing it through the same
+        # uv.lock-vs-legacy bridge as every other Python build here means
+        # this keeps working with no lehrer change if that ever changes --
+        # see python_deps_install_script.
         container = container.with_exec(
-            ["pip", "install", "--no-cache-dir", "-r", "requirements/base.txt"]
+            [
+                "sh",
+                "-c",
+                python_deps_install_script(
+                    workdir="/codejail",
+                    legacy_requirements=["requirements/base.txt"],
+                    ensure_uv=True,
+                ),
+            ]
         ).with_exec(["pip", "install", "--no-cache-dir", "gunicorn"])
 
-        # Install edx-platform sandbox requirements in virtualenv
-        # The URL pattern differs based on whether it's a release or master
-        sandbox_req_url = (
-            f"https://raw.githubusercontent.com/openedx/edx-platform/master/requirements/edx-sandbox/releases/{release_name}.txt"
-            if release_name != "master"
-            else "https://raw.githubusercontent.com/openedx/edx-platform/master/requirements/edx-sandbox/base.txt"
-        )
-
-        import shlex
+        # Install edx-platform sandbox requirements in virtualenv.
+        #
+        # Named releases haven't picked up the pip-compile -> uv migration
+        # and keep publishing a per-release export; read it directly as
+        # before. master's sandbox is now its own standalone uv project
+        # (requirements/edx-sandbox/{pyproject.toml,uv.lock}), with
+        # requirements/edx-sandbox/base.txt kept only as a temporary
+        # machine-generated compat export slated for removal once known
+        # consumers (including this one) stop reading it -- see
+        # openedx/public-engineering#552. Try the compat export first (fast,
+        # no uv needed) and fall back to a uv sync of the sub-project once
+        # it's gone, so this keeps working across that removal without a
+        # lehrer change. --inexact on that sync is required: codejailservice's
+        # own requirements/base.txt and gunicorn (installed above, into this
+        # same /sandbox/venv) aren't part of the sandbox project's uv.lock, and
+        # a plain sync would prune them back out, leaving the image without
+        # its gunicorn entrypoint.
+        if release_name != "master":
+            sandbox_req_url = (
+                "https://raw.githubusercontent.com/openedx/edx-platform/master/"
+                f"requirements/edx-sandbox/releases/{release_name}.txt"
+            )
+            install_sandbox_reqs = (
+                f"pip install --no-cache-dir -r {shlex.quote(sandbox_req_url)}"
+            )
+        else:
+            sandbox_base_url = (
+                "https://raw.githubusercontent.com/openedx/edx-platform/master/"
+                "requirements/edx-sandbox"
+            )
+            install_sandbox_reqs = (
+                "set -eu && "
+                "if curl -fsSL -o /tmp/edx_sandbox.txt "
+                f"{shlex.quote(sandbox_base_url + '/base.txt')}; then "
+                "pip install --no-cache-dir -r /tmp/edx_sandbox.txt; "
+                "else "
+                "mkdir -p /tmp/edx-sandbox-uv && cd /tmp/edx-sandbox-uv && "
+                "curl -fsSL -o pyproject.toml "
+                f"{shlex.quote(sandbox_base_url + '/pyproject.toml')} && "
+                f"curl -fsSL -o uv.lock {shlex.quote(sandbox_base_url + '/uv.lock')} && "
+                "pip install --quiet uv && "
+                "uv sync --locked --active --no-install-project --inexact; "
+                "fi"
+            )
 
         container = container.with_exec(
             [
                 "bash",
                 "-c",
                 f"source /sandbox/venv/bin/activate && "
-                f"pip install --no-cache-dir -r {shlex.quote(sandbox_req_url)} && "
+                f"{install_sandbox_reqs} && "
                 f"deactivate",
             ]
         )

src/lehrer/core/notes.py

@@ -3,6 +3,8 @@
 import dagger
 from dagger import dag, function, object_type
 
+from lehrer.core.pip_compile_bridge import python_deps_install_script
+
 
 @object_type
 class OpenedxNotes:
@@ -106,9 +108,18 @@ async def _build(
             ]
         )
 
-        # Set working directory and PATH
-        container = container.with_workdir("/app/edx-notes-api").with_env_variable(
-            "PATH", "/app/.local/bin:/usr/local/bin:/usr/bin:/bin"
+        # Set working directory and PATH. /opt/notes-venv holds the app's own
+        # deps -- a real venv (not just system site-packages) is what
+        # `uv sync --active` needs a target for, should edx-notes-api's own
+        # requirements ever migrate to uv.lock. See python_deps_install_script.
+        container = (
+            container.with_workdir("/app/edx-notes-api")
+            .with_exec(["python3", "-m", "venv", "/opt/notes-venv"])
+            .with_env_variable(
+                "PATH",
+                "/opt/notes-venv/bin:/app/.local/bin:/usr/local/bin:/usr/bin:/bin",
+            )
+            .with_env_variable("VIRTUAL_ENV", "/opt/notes-venv")
         )
 
         # Get edx-notes-api source from local directory or Git
@@ -130,12 +141,22 @@ async def _build(
         else:
             raise ValueError("Must provide either notes_code or notes_repo")
 
-        # Install Python dependencies as root (system site-packages), then fix ownership
+        # Install Python dependencies (uv sync against a uv.lock, or the
+        # legacy pip-compile requirements/base.txt, whichever the checkout
+        # has -- see python_deps_install_script), then fix ownership.
         container = (
             container.with_exec(
-                ["pip", "install", "--no-cache-dir", "-r", "requirements/base.txt"]
+                [
+                    "sh",
+                    "-c",
+                    python_deps_install_script(
+                        workdir="/app/edx-notes-api",
+                        legacy_requirements=["requirements/base.txt"],
+                        ensure_uv=True,
+                    ),
+                ]
             )
-            .with_exec(["chown", "-R", "app:app", "/app"])
+            .with_exec(["chown", "-R", "app:app", "/app", "/opt/notes-venv"])
             .with_user("1000")
         )
 

src/lehrer/core/pip_compile_bridge.py

@@ -0,0 +1,90 @@
+"""Shared pip-compile -> uv.lock bridge for Python service builds.
+
+Several Open edX services are (or may eventually be) migrating their Python
+dependency management from pip-compile-generated ``requirements/*.txt``
+files to ``pyproject.toml`` + ``uv.lock`` as the source of truth --
+openedx-platform's own migration is tracked in
+openedx/public-engineering#552. Every Python build in lehrer sources its
+dependencies through :func:`python_deps_install_script`, which detects the
+track from the checkout itself (``uv.lock`` present or not) rather than
+hardcoding which release or service has migrated. That keeps a build
+working across a service's migration landing without a lehrer change --
+and costs nothing for a checkout that never migrates: the branch just
+always takes the legacy arm.
+"""
+
+import shlex
+
+
+def python_deps_install_script(
+    *,
+    workdir: str,
+    legacy_requirements: list[str],
+    sync_groups: list[str] | None = None,
+    legacy_installer: list[str] | None = None,
+    ensure_uv: bool = False,
+) -> str:
+    """Shell script installing a checkout's Python deps.
+
+    Runs ``uv sync --locked`` against a committed ``uv.lock`` when present,
+    otherwise falls back to installing the legacy pip-compile
+    ``requirements/*.txt`` files.
+
+    ``--inexact`` is always passed on the sync path: callers commonly layer
+    other installs into the same environment around this call -- a
+    deployment's own pinned package list, another service's base
+    requirements, an editable framework checkout installed just before it.
+    A plain (exact) sync prunes anything not in the checkout's own lock
+    resolution, which would silently strip those back out.
+
+    Args:
+        workdir: Directory containing the checkout (``pyproject.toml``/
+            ``uv.lock`` or the legacy ``requirements/`` tree).
+        legacy_requirements: Requirement file paths (relative to
+            ``workdir``) to install when no ``uv.lock`` is present.
+        sync_groups: Dependency groups to select when ``uv.lock`` IS
+            present, via ``--no-default-groups --group <name>`` for each.
+            Omit for just the project's own default dependencies.
+        legacy_installer: Command prefix for the legacy branch. Default
+            ``["pip", "install", "--no-cache-dir"]``. Pass
+            ``["uv", "pip", "install"]`` for a container where uv is
+            already the installer of record.
+        ensure_uv: Install uv via pip before the sync branch runs. Default
+            ``False`` for containers (e.g. the edx-platform build) that
+            already have uv on PATH; set ``True`` for containers (e.g.
+            codejail, notes) that only have plain pip.
+
+    Must run with ``VIRTUAL_ENV`` set to the target environment -- the sync
+    branch uses ``--active``.
+    """
+    installer = list(legacy_installer or ["pip", "install", "--no-cache-dir"])
+    legacy_args: list[str] = []
+    for req in legacy_requirements:
+        legacy_args += ["-r", req]
+    legacy_cmd = " ".join(shlex.quote(part) for part in [*installer, *legacy_args])
+
+    sync_parts = [
+        "uv",
+        "sync",
+        "--locked",
+        "--active",
+        "--no-install-project",
+        "--inexact",
+    ]
+    if sync_groups:
+        sync_parts.append("--no-default-groups")
+        for group in sync_groups:
+            sync_parts += ["--group", group]
+    sync_cmd = " ".join(shlex.quote(part) for part in sync_parts)
+    if ensure_uv:
+        sync_cmd = f"pip install --quiet uv && {sync_cmd}"
+
+    return (
+        "set -eu\n"
+        f"cd {shlex.quote(workdir)}\n"
+        "if [ -f uv.lock ]; then\n"
+        f"  {sync_cmd}\n"
+        "else\n"
+        f"  {legacy_cmd}\n"
+        "fi\n"
+    )