{Packaging} Migrate Azure CLI packages to PEP 621 pyproject.toml

.azure-pipelines/templates/automation_test.yml

@@ -27,6 +27,12 @@ parameters:
   displayName: Job name
   type: string
   default: ''
+- name: AzdevPackage
+  type: string
+  default: 'azdev @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e'
+- name: DiffToolPackage
+  type: string
+  default: 'azure-cli-diff-tool @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e#subdirectory=azure-cli-diff-tool'
 
 steps:
   - task: UsePythonVersion@0
@@ -36,6 +42,8 @@ steps:
   - template: ./azdev_setup.yml
     parameters:
       EnableCompactAAZ: true
+      AzdevPackage: ${{ parameters.AzdevPackage }}
+      DiffToolPackage: ${{ parameters.DiffToolPackage }}
   - bash: |
       set -ev
 

.azure-pipelines/templates/azdev_setup.yml

@@ -8,6 +8,14 @@ parameters:
   - name: EnableCompactAAZ
     type: boolean
     default: false
+  # Temporary immutable candidates until the validated packages are released.
+  # Keep both requirements on the same commit; overrides may also be wheel paths.
+  - name: AzdevPackage
+    type: string
+    default: 'azdev @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e'
+  - name: DiffToolPackage
+    type: string
+    default: 'azure-cli-diff-tool @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e#subdirectory=azure-cli-diff-tool'
 steps:
   - bash: |
       set -ev
@@ -16,8 +24,12 @@ steps:
       chmod +x env/bin/activate
       . env/bin/activate
 
-      python -m pip install -U pip "setuptools<81"
-      pip install --upgrade "azdev==0.2.13"
+      python -m pip install -U pip "setuptools>=78.1.1"
+      packages=("$AZDEV_PACKAGE")
+      if [ -n "$DIFF_TOOL_PACKAGE" ]; then
+        packages+=("$DIFF_TOOL_PACKAGE")
+      fi
+      python -m pip install --upgrade "${packages[@]}"
       azdev --version
 
       if [ -z "$CLI_EXT_REPO_PATH" ]; then
@@ -37,5 +49,7 @@ steps:
       az --version
     displayName: 'azdev setup'
     env:
+      AZDEV_PACKAGE: ${{ parameters.AzdevPackage }}
+      DIFF_TOOL_PACKAGE: ${{ parameters.DiffToolPackage }}
       CLI_REPO_PATH: ${{ parameters.CLIRepoPath }}
       CLI_EXT_REPO_PATH: ${{ parameters.CLIExtensionRepoPath }}

.github/CODEOWNERS

@@ -22,8 +22,8 @@
 
 /src/azure-cli/ @Azure/act-codegen-extensibility-squad @Azure/act-quality-productivity-squad @Azure/act-platform-engineering-squad
 
-/src/azure-cli-core/setup.py @Azure/act-codegen-extensibility-squad @Azure/act-quality-productivity-squad @Azure/act-platform-engineering-squad @Azure/act-identity-squad
-/src/azure-cli/setup.py @Azure/act-identity-squad @Azure/act-codegen-extensibility-squad
+/src/azure-cli-core/pyproject.toml @Azure/act-codegen-extensibility-squad @Azure/act-quality-productivity-squad @Azure/act-platform-engineering-squad @Azure/act-identity-squad
+/src/azure-cli/pyproject.toml @Azure/act-identity-squad @Azure/act-codegen-extensibility-squad
 /src/azure-cli/requirements.*.txt @Azure/act-codegen-extensibility-squad @Azure/act-quality-productivity-squad @Azure/act-platform-engineering-squad @Azure/act-identity-squad
 
 /src/azure-cli/azure/cli/command_modules/acr/ @Azure/act-observability-squad

.github/actions/env-setup/action.yml

@@ -23,13 +23,17 @@ runs:
         python-version: "3.12"
     - name: Install azdev
       shell: bash
+      # Temporary immutable candidates; neither package needs a PyPI release.
+      env:
+        AZDEV_PACKAGE: 'azdev @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e'
+        DIFF_TOOL_PACKAGE: 'azure-cli-diff-tool @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e#subdirectory=azure-cli-diff-tool'
       run: | 
         python -m pip install --upgrade pip
         set -ev
         python -m venv env
         chmod +x env/bin/activate
         source ./env/bin/activate
-        pip install azdev
+        python -m pip install "$AZDEV_PACKAGE" "$DIFF_TOOL_PACKAGE"
         azdev --version
         cd ../
         azdev setup -c azure-cli --debug

azure-cli.pyproj

@@ -75,7 +75,7 @@
     <Compile Include="azure-cli-core\azure\cli\core\__init__.py" />
     <Compile Include="azure-cli-core\azure\cli\__init__.py" />
     <Compile Include="azure-cli-core\azure\__init__.py" />
-    <Compile Include="azure-cli-core\setup.py" />
+    <Content Include="azure-cli-core\pyproject.toml" />
     <Compile Include="azure-cli-testsdk\azure\cli\testsdk\base.py" />
     <Compile Include="azure-cli-testsdk\azure\cli\testsdk\checkers.py" />
     <Compile Include="azure-cli-testsdk\azure\cli\testsdk\decorators.py" />
@@ -84,11 +84,11 @@
     <Compile Include="azure-cli-testsdk\azure\cli\testsdk\preparers.py" />
     <Compile Include="azure-cli-testsdk\azure\cli\testsdk\utilities.py" />
     <Compile Include="azure-cli-testsdk\azure\cli\testsdk\__init__.py" />
-    <Compile Include="azure-cli-testsdk\setup.py" />
+    <Content Include="azure-cli-testsdk\pyproject.toml" />
     <Compile Include="azure-cli\azure\cli\__init__.py" />
     <Compile Include="azure-cli\azure\cli\__main__.py" />
     <Compile Include="azure-cli\azure_bdist_wheel.py" />
-    <Compile Include="azure-cli\setup.py" />
+    <Content Include="azure-cli\pyproject.toml" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\commands.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\credential.py" />
     <Compile Include="command_modules\azure-cli-acr\azure\cli\command_modules\acr\custom.py" />

azure-cli2017.pyproj

@@ -87,7 +87,7 @@
     <Compile Include="azure-cli-core\azure\cli\core\_session.py" />
     <Compile Include="azure-cli-core\azure\cli\core\telemetry.py" />
     <Compile Include="azure-cli-core\azure\cli\core\__init__.py" />
-    <Compile Include="azure-cli-core\setup.py" />
+    <Content Include="azure-cli-core\pyproject.toml" />
     <Compile Include="azure-cli-telemetry\azure\cli\telemetry\components\records_collection.py" />
     <Compile Include="azure-cli-telemetry\azure\cli\telemetry\components\records_collection.py" />
     <Compile Include="azure-cli-telemetry\azure\cli\telemetry\components\telemetry_client.py" />
@@ -118,8 +118,7 @@
     <Compile Include="azure-cli-telemetry\azure\cli\__init__.py" />
     <Compile Include="azure-cli-telemetry\azure\__init__.py" />
     <Compile Include="azure-cli-telemetry\azure\__init__.py" />
-    <Compile Include="azure-cli-telemetry\setup.py" />
-    <Compile Include="azure-cli-telemetry\setup.py" />
+    <Content Include="azure-cli-telemetry\pyproject.toml" />
     <Compile Include="azure-cli-testsdk\azure\cli\testsdk\base.py" />
     <Compile Include="azure-cli-testsdk\azure\cli\testsdk\checkers.py" />
     <Compile Include="azure-cli-testsdk\azure\cli\testsdk\decorators.py" />
@@ -129,7 +128,7 @@
     <Compile Include="azure-cli-testsdk\azure\cli\testsdk\reverse_dependency.py" />
     <Compile Include="azure-cli-testsdk\azure\cli\testsdk\utilities.py" />
     <Compile Include="azure-cli-testsdk\azure\cli\testsdk\__init__.py" />
-    <Compile Include="azure-cli-testsdk\setup.py" />
+    <Content Include="azure-cli-testsdk\pyproject.toml" />
     <Compile Include="azure-cli\azure\cli\command_modules\acr\build.py" />
     <Compile Include="azure-cli\azure\cli\command_modules\acr\check_health.py" />
     <Compile Include="azure-cli\azure\cli\command_modules\acr\commands.py" />
@@ -963,7 +962,7 @@
     <Compile Include="azure-cli\azure\cli\__init__.py" />
     <Compile Include="azure-cli\azure\cli\__main__.py" />
     <Compile Include="azure-cli\azure\__init__.py" />
-    <Compile Include="azure-cli\setup.py" />
+    <Content Include="azure-cli\pyproject.toml" />
   </ItemGroup>
   <ItemGroup>
     <Folder Include="azure-cli-core\" />

azure-pipelines.yml

@@ -541,6 +541,28 @@ jobs:
       pythonVersion: '$(python.version)'
       module: 'azure-cli-core'
 
+- job: TestPackagingMetadata
+  displayName: Test Packaging Metadata
+  timeoutInMinutes: 10
+  pool:
+    name: ${{ variables.ubuntu_pool }}
+  strategy:
+    matrix:
+      Python310:
+        python.version: '3.10'
+      Python314:
+        python.version: '3.14'
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '$(python.version)'
+  - bash: |
+      set -e
+      python -m pip install docutils requests packaging tomli
+      python -B -m unittest discover -s tools/automation/tests -t tools -p test_packaging_migration.py -v
+      python -B scripts/ci/check_package_versions.py
+    displayName: Validate source metadata and packaging automation
+
 - job: TestTelemetry
   displayName: Unit Test for Telemetry
   timeoutInMinutes: 10
@@ -577,7 +599,7 @@ jobs:
     displayName: 'Use Python $(python.version)'
     inputs:
       versionSpec: '$(python.version)'
-  - bash: pip install --upgrade pip wheel build "setuptools<81"
+  - bash: pip install --upgrade pip wheel build "setuptools>=78.1.1"
     displayName: 'Install pip and wheel'
   - bash: ./scripts/ci/test_profile_integration.sh
     displayName: 'Run Integration Test against Profiles'
@@ -598,7 +620,7 @@ jobs:
     displayName: 'Use Python $(python.version)'
     inputs:
       versionSpec: '$(python.version)'
-  - bash: pip install --upgrade pip wheel build "setuptools<81"
+  - bash: pip install --upgrade pip wheel build "setuptools>=78.1.1"
     displayName: 'Install pip and wheel setuptools'
   - bash: ./scripts/ci/test_extensions.sh
     displayName: 'Load extensions'

requirements.txt

@@ -1,6 +1,4 @@
 # basic
-# Cap setuptools<81: newer setuptools breaks the CLI's setup.py-based builds
-# (81 removes setup.py --dry-run and changes distutils command signatures;
-# 82 removes pkg_resources). Pin to the last 80.x line.
-setuptools>=65.5.1,<81
+# Security floor for setuptools; CLI packages use the PEP 517 build backend.
+setuptools>=78.1.1
 pip>=9.0.1

scripts/ci/build.sh

@@ -55,13 +55,15 @@ title 'Build Azure CLI and its command modules'
 # provisioning the frontend is easy to miss. Install it here if it is absent rather
 # than failing partway through the build.
 python -c 'import build' 2>/dev/null || python -m pip install --disable-pip-version-check -q build
+python -c 'import tomllib' 2>/dev/null || python -m pip install --disable-pip-version-check -q 'tomli>=2.0.1'
+python "$script_dir/check_package_versions.py"
 
-for setup_file in $(find src -name 'setup.py'); do
-    pushd $(dirname ${setup_file}) >/dev/null
+for pyproject_file in $(find src -name 'pyproject.toml'); do
+    pushd $(dirname ${pyproject_file}) >/dev/null
     echo "Building module at $(pwd) ..."
     # --no-isolation builds against the environment prepared by the caller rather than
-    # provisioning a fresh one, which keeps the caller's setuptools pin in force and
-    # avoids requiring outbound network access inside the packaging test containers.
+    # provisioning a fresh one. The caller must provide the build requirements;
+    # this avoids fetching them inside the packaging test containers.
     python -m build --wheel --no-isolation --outdir $output_dir
     python -m build --sdist --no-isolation --outdir $sdist_dir
     popd >/dev/null

scripts/ci/check_package_versions.py

@@ -0,0 +1,32 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+"""Validate source versions before building release or nightly distributions."""
+
+from pathlib import Path
+import sys
+
+ROOT = Path(__file__).resolve().parents[2]
+PACKAGES = ('azure-cli', 'azure-cli-core', 'azure-cli-telemetry', 'azure-cli-testsdk')
+sys.path.insert(0, str(ROOT / 'tools'))
+
+from automation.utilities.packaging import get_package_version, tomllib  # noqa: E402
+
+
+def main():
+    versions = {}
+    for name in PACKAGES:
+        versions[name] = get_package_version(ROOT / 'src' / name)
+    with (ROOT / 'src/azure-cli/pyproject.toml').open('rb') as stream:
+        dependencies = tomllib.load(stream)['project']['dependencies']
+    expected = 'azure-cli-core==' + versions['azure-cli-core']
+    if expected not in dependencies or versions['azure-cli'] != versions['azure-cli-core']:
+        raise ValueError('CLI and core versions must match, including the dependency: ' + expected)
+    for name, version in versions.items():
+        print('{}: {}'.format(name, version))
+
+
+if __name__ == '__main__':
+    main()

scripts/ci/version.sh

@@ -32,11 +32,13 @@ for each in $(find src -name __init__.py); do
     fi
 done
 
-for each in $(find src -name setup.py); do
+for each in $(find src -name pyproject.toml); do
     if [ "$platform" == "Darwin" ]; then
-        sed -i "" "s/^VERSION = [\"']\(.*\)[\"']/VERSION = \"\1.$version\"/" $each
+        sed -i "" "s/^version = [\"']\(.*\)[\"']/version = \"\1.$version\"/" "$each"
+        sed -i "" "s/azure-cli-core==\([^\"']*\)/azure-cli-core==\1.$version/" "$each"
     else
-        sed -i "s/^VERSION = [\"']\(.*\)[\"']/VERSION = \"\1.$version\"/" $each
+        sed -i "s/^version = [\"']\(.*\)[\"']/version = \"\1.$version\"/" "$each"
+        sed -i "s/azure-cli-core==\([^\"']*\)/azure-cli-core==\1.$version/" "$each"
     fi
 done
 

scripts/install_full.sh

@@ -16,7 +16,7 @@ REPO_ROOT="$(dirname ${BASH_SOURCE[0]})/.."
 
 pushd ${REPO_ROOT} > /dev/null
 
-find src/ -name setup.py -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip install --no-deps
+find src/ -name pyproject.toml -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip install --no-deps
 pip install -r ./src/azure-cli/requirements.$(python ./scripts/get-python-version.py).$(uname).txt
 
 popd > /dev/null

scripts/live_test/CLITest.yml

@@ -13,6 +13,11 @@ trigger:
 
 variables:
 - template: ${{ variables.Pipeline.Workspace }}/.azure-pipelines/templates/variables.yml
+# Temporary immutable candidates, matching the shared CI setup.
+- name: AzdevPackage
+  value: 'azdev @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e'
+- name: DiffToolPackage
+  value: 'azure-cli-diff-tool @ git+https://github.com/aryasadeghi1/azure-cli-dev-tools.git@c429216bfdae9b196dcfa6689f4d3f15a7b7054e#subdirectory=azure-cli-diff-tool'
 
 parameters:
 - name: maxParallel
@@ -558,7 +563,7 @@ stages:
             source env/bin/activate
 
             # TODO: support pytest-html >= 4.0.0
-            pip install azdev pytest-json-report pytest-html==3.2.0 pytest-rerunfailures
+            python -m pip install "$(AzdevPackage)" "$(DiffToolPackage)" pytest-json-report pytest-html==3.2.0 pytest-rerunfailures
 
             azdev setup -c azure-cli -r azure-cli-extensions
 
@@ -1136,7 +1141,7 @@ stages:
             export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
 
             # TODO: support pytest-html >= 4.0.0
-            pip install azdev pytest-json-report pytest-html==3.2.0 pytest-rerunfailures
+            python -m pip install "$(AzdevPackage)" "$(DiffToolPackage)" pytest-json-report pytest-html==3.2.0 pytest-rerunfailures
 
             azdev setup -c azure-cli -r azure-cli-extensions
 
@@ -1712,7 +1717,7 @@ stages:
             source env/Scripts/activate
 
             # TODO: support pytest-html >= 4.0.0
-            pip install azdev pytest-json-report pytest-html==3.2.0 pytest-rerunfailures
+            python -m pip install "$(AzdevPackage)" "$(DiffToolPackage)" pytest-json-report pytest-html==3.2.0 pytest-rerunfailures
 
             azdev setup -c azure-cli -r azure-cli-extensions
 

scripts/regression_test/regression_test.yml

@@ -46,8 +46,8 @@ jobs:
             echo "'PACKAGE' and 'TARGET_PACKAGE_VERSION' are required"
             exit 1
           fi
-          sed -i "s/'$(PACKAGE)==.*'/'$(PACKAGE)==$(TARGET_PACKAGE_VERSION)'/g" ./src/azure-cli/setup.py
-          sed -i "s/'$(PACKAGE)~=.*'/'$(PACKAGE)~=$(TARGET_PACKAGE_VERSION)'/g" ./src/azure-cli/setup.py
+          sed -i "s/'$(PACKAGE)==.*'/'$(PACKAGE)==$(TARGET_PACKAGE_VERSION)'/g" ./src/azure-cli/pyproject.toml
+          sed -i "s/'$(PACKAGE)~=.*'/'$(PACKAGE)~=$(TARGET_PACKAGE_VERSION)'/g" ./src/azure-cli/pyproject.toml
           sed -i "s/$(PACKAGE)==.*/$(PACKAGE)==$(TARGET_PACKAGE_VERSION)/g" ./src/azure-cli/requirements.py3.windows.txt
           sed -i "s/$(PACKAGE)==.*/$(PACKAGE)==$(TARGET_PACKAGE_VERSION)/g" ./src/azure-cli/requirements.py3.Linux.txt
           sed -i "s/$(PACKAGE)==.*/$(PACKAGE)==$(TARGET_PACKAGE_VERSION)/g" ./src/azure-cli/requirements.py3.Darwin.txt

scripts/release/debian/build.sh

@@ -37,12 +37,12 @@ $PYTHON_SRC_DIR/*/configure --srcdir $PYTHON_SRC_DIR/* --prefix $WORKDIR/python_
 make
 make install
 
-# Cap setuptools<81: 81 removes setup.py --dry-run and changes distutils command signatures (82 removes pkg_resources); the CLI build relies on setup.py.
-$WORKDIR/python_env/bin/python3 -m pip install --upgrade pip "setuptools<81"
+# Keep setuptools at or above the security floor.
+$WORKDIR/python_env/bin/python3 -m pip install --upgrade pip "setuptools>=78.1.1"
 
 export PATH=$PATH:$WORKDIR/python_env/bin
 
-find ${WORKDIR}/src/ -name setup.py -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip3 install --no-deps
+find ${WORKDIR}/src/ -name pyproject.toml -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip3 install --no-deps
 pip3 install -r ${WORKDIR}/src/azure-cli/requirements.py3.$(uname).txt
 $WORKDIR/python_env/bin/python3 ${WORKDIR}/scripts/trim_sdk.py
 

scripts/release/homebrew/docker/requirements.txt

@@ -1,7 +1,7 @@
-# Pin setuptools<81 to stay aligned with the CLI-wide cap: homebrew-pypi-poet
-# (abandoned since 2018) uses pkg_resources, which was removed in setuptools 82+,
-# and 81 changes setup.py/distutils behavior. See https://github.com/Azure/azure-cli/issues/32800
-setuptools<81
+# homebrew-pypi-poet uses pkg_resources, removed in setuptools 82.
+# Keep this cap local to formula-generation tooling, with the security floor.
+# See https://github.com/Azure/azure-cli/issues/32800
+setuptools>=78.1.1,<82
 homebrew-pypi-poet~=0.10.0
 jinja2~=3.1.6
 requests>=2.20.0

scripts/release/homebrew/docker/run.sh

@@ -13,7 +13,7 @@ fi
 pip install wheel
 pip install -U pip
 pip install -r $root/requirements.txt
-find /mnt/src/ -name setup.py -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip install --no-deps
+find /mnt/src/ -name pyproject.toml -type f | xargs -I {} dirname {} | grep -v azure-cli-testsdk | xargs pip install --no-deps
 pip install -r /mnt/src/azure-cli/requirements.py3.Darwin.txt
 
 pip list