diff --git a/.flake8 b/.flake8 deleted file mode 100644 index e337d7ab..00000000 --- a/.flake8 +++ /dev/null @@ -1,4 +0,0 @@ -[flake8] - max-line-length = 120 - exclude = unittests/recipes/with-repo/repo/packages/, external/ - extend-ignore = E203 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index b00e51a2..07b72ac3 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -1,25 +1,16 @@ name: lint -on: - pull_request: - branches: [main] +on: [push, pull_request] jobs: lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: Install Tools + - name: Install uv run: | - python -m pip install --upgrade pip - python -m pip install black flake8 mypy - - name: Black + curl -LsSf https://astral.sh/uv/install.sh | sh + - name: ruff run: | - black --check --verbose stackinator unittests - - name: flake8 - run: | - flake8 --count --show-source --statistics . + uvx ruff format --check + uvx ruff check diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 7b15404c..8bd38da5 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -3,20 +3,13 @@ name: Stackinator CI on: [push, pull_request] jobs: - unittestpy36: - runs-on: ubuntu-20.04 - strategy: - matrix: - python-version: ['3.6'] + unittestpy: + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Bootstrap + - name: Install uv run: | - ./bootstrap.sh + curl -LsSf https://astral.sh/uv/install.sh | sh - name: Generic Unittests run: | ./test_stackinator.py diff --git a/docs/readme.md b/docs/readme.md index 0069b81c..39da9ba5 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -1,15 +1,11 @@ The documentation for Stackinator is built from the markdown files in this path using MkDocs and MkDocs-material. You can view the latest documentation online at [github.io](https://eth-cscs.github.io/stackinator/) -To build a copy locally, first install `mkdocs-material`, e.g.: +To view work in progress docs, run the serve script and follow the link it provides to view a local copy of the docs in your browser. ```bash -python3 -m venv docs-env -source docs-env/bin/activate -pip install mkdocs-material +./serve ``` -Then in the root of this project, build the docs and view them with your favourite browser: -```bash -mkdocs build -firefox site/index.html -``` +> [!IMPORTANT] +> to run the serve script, you need to first install [uv](https://docs.astral.sh/uv/getting-started/installation/). + diff --git a/docs/recipes.md b/docs/recipes.md index db2ca13e..47c6e1bd 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -25,6 +25,7 @@ spack: commit: releases/v0.20 modules: true description: "HPC development tools for building MPI applications with the GNU compiler toolchain" +version: 2 ``` * `name`: a plain text name for the environment @@ -32,6 +33,24 @@ description: "HPC development tools for building MPI applications with the GNU c * `spack`: which spack repository to use for installation. * `modules`: _optional_ enable/diasble module file generation (default `true`). * `description`: _optional_ a string that describes the environment (default empty). +* `version`: _default = 1_ the version of the uenv recipe (see below) + +!!! note "uenv recipe versions" + Stackinator 6 introduces breaking changes to the uenv recipe format, introduced to support Spack v1.0. + + We have started versioning uenv recipes: + + * **version 1**: original uenv recipes for Spack v0.23 and earlier, supported by Stackinator version 5. + * **version 2**: uenv recipes for Spack v1.0 and later, supported by Stackinator version 6. + + The default version is 1, so that old recipes that do not set a version are supported. + + !!! warning "You must set version 2 explicitly to use Spack v1.0" + + !!! warning "Version 1 recipes must be configured using Stackinator v5" + Version 5 of Stackinator is maintained in the `releases/v5` branch of stackinator. + + You must also use the `releases/v5` branch of [Alps cluster config](https://github.com/eth-cscs/alps-cluster-config). ## Compilers diff --git a/lint b/lint new file mode 100755 index 00000000..c2db4fb2 --- /dev/null +++ b/lint @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +uvx ruff format +uvx ruff check --fix diff --git a/pyproject.toml b/pyproject.toml index 315b4437..ac4f1a33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,13 +2,11 @@ requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" -[tool.black] -line-length = 120 - -[tool.isort] -profile = "black" -skip = ["external/"] - [tool.ruff] line-length = 120 extend-exclude = ["external/", "unittests/recipes"] +exclude = ["unittests/recipes/with-repo/repo/packages"] + +[tool.ruff.lint] +select = ["E", "F"] +ignore = ["E203"] diff --git a/stackinator/builder.py b/stackinator/builder.py index 61d0e3ad..66d50a5d 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -183,7 +183,7 @@ def generate(self, recipe): # Clone the spack repository if it has not already been checked out if not (spack_path / ".git").is_dir(): - self._logger.info(f'spack: clone repository {spack["repo"]}') + self._logger.info(f"spack: clone repository {spack['repo']}") # clone the repository capture = subprocess.run( @@ -195,12 +195,12 @@ def generate(self, recipe): self._logger.debug(capture.stdout.decode("utf-8")) if capture.returncode != 0: - self._logger.error(f'error cloning the repository {spack["repo"]}') + self._logger.error(f"error cloning the repository {spack['repo']}") capture.check_returncode() # Fetch the specific branch if spack["commit"]: - self._logger.info(f'spack: fetch branch/commit {spack["commit"]}') + self._logger.info(f"spack: fetch branch/commit {spack['commit']}") capture = subprocess.run( ["git", "-C", spack_path, "fetch", "origin", spack["commit"]], shell=False, @@ -210,12 +210,12 @@ def generate(self, recipe): self._logger.debug(capture.stdout.decode("utf-8")) if capture.returncode != 0: - self._logger.debug(f'unable to change to the fetch {spack["commit"]}') + self._logger.debug(f"unable to change to the fetch {spack['commit']}") capture.check_returncode() # Check out a branch or commit if one was specified if spack["commit"]: - self._logger.info(f'spack: checkout branch/commit {spack["commit"]}') + self._logger.info(f"spack: checkout branch/commit {spack['commit']}") capture = subprocess.run( ["git", "-C", spack_path, "checkout", spack["commit"]], shell=False, @@ -225,7 +225,7 @@ def generate(self, recipe): self._logger.debug(capture.stdout.decode("utf-8")) if capture.returncode != 0: - self._logger.debug(f'unable to change to the requested commit {spack["commit"]}') + self._logger.debug(f"unable to change to the requested commit {spack['commit']}") capture.check_returncode() # get the spack commit diff --git a/stackinator/etc/add-compiler-links.py b/stackinator/etc/add-compiler-links.py index 6d76b002..3975d7fe 100755 --- a/stackinator/etc/add-compiler-links.py +++ b/stackinator/etc/add-compiler-links.py @@ -69,7 +69,7 @@ def has_prefix(path, prefix): for c in compilers: local_paths = set([os.path.dirname(v) for k, v in c["paths"].items() if v is not None]) paths += local_paths - print(f'adding compiler {c["spec"]} -> {[p for p in local_paths]}') + print(f"adding compiler {c['spec']} -> {[p for p in local_paths]}") # find unique paths and concatenate them pathstring = ":".join(set(paths)) diff --git a/stackinator/etc/envvars.py b/stackinator/etc/envvars.py index 63ca1a01..b70263b0 100755 --- a/stackinator/etc/envvars.py +++ b/stackinator/etc/envvars.py @@ -161,7 +161,7 @@ def __repr__(self): return f'envvars.ListEnvVar("{self.name}", {self._updates})' def __str__(self): - return f"(\"{self.name}\": [{','.join([str(u) for u in self._updates])}])" + return f'("{self.name}": [{",".join([str(u) for u in self._updates])}])' class ScalarEnvVar(EnvVar): @@ -453,7 +453,7 @@ def view_impl(args): for c in compilers: local_paths = set([os.path.dirname(v) for _, v in c["paths"].items() if v is not None]) compiler_paths += local_paths - print(f'adding compiler {c["spec"]} -> {[p for p in local_paths]}') + print(f"adding compiler {c['spec']} -> {[p for p in local_paths]}") envvars.set_list("PATH", compiler_paths, EnvVarOp.PREPEND) diff --git a/stackinator/main.py b/stackinator/main.py index 4a6de668..73ea92c1 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -14,7 +14,7 @@ def generate_logfile_name(name=""): idstr = f"{time.localtime()}{os.getpid}{platform.uname()}" - return f'log{name}_{hashlib.md5(idstr.encode("utf-8")).hexdigest()}' + return f"log{name}_{hashlib.md5(idstr.encode('utf-8')).hexdigest()}" def configure_logging(logfile): @@ -45,7 +45,7 @@ def log_header(args): def make_argparser(): - parser = argparse.ArgumentParser(description=("Generate a build configuration for a spack stack from " "a recipe.")) + parser = argparse.ArgumentParser(description=("Generate a build configuration for a spack stack from a recipe.")) parser.add_argument("--version", action="version", version=f"stackinator version {VERSION}") parser.add_argument("-b", "--build", required=True, type=str) parser.add_argument("--no-bwrap", action="store_true", required=False) @@ -76,7 +76,7 @@ def main(): builder.generate(recipe) - root_logger.info("\nConfiguration finished, run the following to build the " "environment:\n") + root_logger.info("\nConfiguration finished, run the following to build the environment:\n") root_logger.info(f"cd {builder.path}") root_logger.info( 'env --ignore-environment http_proxy="$http_proxy" https_proxy="$https_proxy" no_proxy="$no_proxy"' diff --git a/stackinator/recipe.py b/stackinator/recipe.py index a7f90802..11ad31ca 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -51,7 +51,29 @@ def __init__(self, args): # required config.yaml file self.config = self.path / "config.yaml" - # set the recipe-defined mount point + # check the version of the recipe + if self.config["version"] != 2: + rversion = self.config["version"] + if rversion == 1: + self._logger.error( + "\nThe recipe is an old version 1 recipe for Spack v0.23 and earlier.\n" + "This version of Stackinator supports Spack 1.0, and has deprecated support for Spack v0.23.\n" + "Use version 5 of stackinator, which can be accessed via the releases/v5 branch:\n" + " git switch releases/v5\n\n" + "If this recipe is to be used with Spack 1.0, then please add the field 'version: 2' to\n" + "config.yaml in your recipe.\n\n" + "For more information: https://eth-cscs.github.io/stackinator/recipes/#configuration\n" + ) + raise RuntimeError("incompatible uenv recipe version") + else: + self._logger.error( + f"\nThe config.yaml file sets an unknown recipe version={rversion}.\n" + "This version of Stackinator supports version 2 recipes.\n\n" + "For more information: https://eth-cscs.github.io/stackinator/recipes/#configuration\n" + ) + raise RuntimeError("incompatible uenv recipe version") + + # override the mount point if defined as a CLI argument if args.mount: self.config["store"] = args.mount diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index a268d28a..df63fa70 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -56,6 +56,10 @@ {"type" : "null"} ], "default": null + }, + "version" : { + "type": "number", + "default": 1 } } }