Merge pull request #1938 from h-mayorquin/fix_CI

Fall back to the cached test data when gin cannot be reached in CI

Author: h-mayorquin

.github/workflows/io-test.yml

@@ -42,10 +42,11 @@ jobs:
         # the key depend on the last commit repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git
         id: ephy_testing_data
         run: |
+          # if gin cannot be reached the key matches no cache and the restore-keys below restore the most recent one
           set -o pipefail
           if ! hash="$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" || [ -z "$hash" ]; then
-            echo "::error::could not resolve ephy_testing_data HEAD on gin; refusing to build a degenerate cache key"
-            exit 1
+            echo "::warning::could not resolve ephy_testing_data HEAD on gin; restoring the most recent dataset cache"
+            hash=""
           fi
           echo "dataset_hash=$hash" >> "$GITHUB_OUTPUT"
 

.github/workflows/plexon2-testing.yml

@@ -33,10 +33,11 @@ jobs:
         # the key depend on the last commit repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git
         id: ephy_testing_data
         run: |
+          # if gin cannot be reached the key matches no cache and the restore-keys below restore the most recent one
           set -o pipefail
           if ! hash="$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" || [ -z "$hash" ]; then
-            echo "::error::could not resolve ephy_testing_data HEAD on gin; refusing to build a degenerate cache key"
-            exit 1
+            echo "::warning::could not resolve ephy_testing_data HEAD on gin; restoring the most recent dataset cache"
+            hash=""
           fi
           echo "dataset_hash=$hash" >> "$GITHUB_OUTPUT"
 

neo/utils/datasets.py

@@ -3,6 +3,7 @@
 """
 
 import os
+import warnings
 from pathlib import Path
 
 default_testing_repo = "https://gin.g-node.org/NeuralEnsemble/ephy_testing_data"
@@ -14,6 +15,9 @@
     # set in home
     local_testing_data_folder = Path.home() / "ephy_testing_data"
 
+# local copies already updated in this process, so each one asks its remote at most once
+_updated_folders = set()
+
 
 def get_local_testing_data_folder():
     global local_testing_data_folder
@@ -64,7 +68,12 @@ def download_dataset(repo=default_testing_repo, remote_path=None, local_folder=N
         # make sure git repo is in clean state
         repo = dataset.repo
         repo.call_git(["checkout", "--force", "master"])
-        dataset.update(merge=True)
+        if local_folder not in _updated_folders:
+            _updated_folders.add(local_folder)
+            # the remote can be unreachable (gin answers 403 to CI runners at times), so keep the local copy then
+            results = dataset.update(merge=True, on_failure="ignore")
+            if any(result["status"] in ("impossible", "error") for result in results):
+                warnings.warn(f"Could not update {local_folder} from its remote, using the local copy as it is")
     else:
         dataset = datalad.api.install(path=local_folder, source=repo)