filer: switch workspace upload from import-file to /workspace/import

.nextchanges/bundles/workspace-import-migration.md

@@ -0,0 +1 @@
+Bundle file uploads now use the multipart `POST /api/2.0/workspace/import` endpoint instead of the deprecated `POST /api/2.0/workspace-files/import-file`. The new endpoint has a documented rate limit of 30 requests per second per workspace ([API rate limits](https://docs.databricks.com/aws/en/resources/limits)), higher than the limit that applied to the previous endpoint, and is ~1.5–2× faster for typical bundle deployments.

acceptance/bin/fault.py

@@ -1,54 +1,62 @@
 #!/usr/bin/env python3
 """Set up a fault rule on the testserver for the current test token.
 
-Usage: fault.py [--after-handler] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]
-
-  --after-handler  run the handler first; keep its effect and replace only the response
-  PATTERN     HTTP method and path, supports trailing * wildcard,
-              e.g. "PUT /api/2.0/permissions/pipelines/*"
-  STATUS_CODE HTTP status code to return, e.g. 504
-  OFFSET      number of requests to let through before fault starts
-  TIMES       number of times to return the fault response
-  ERROR_CODE  optional error_code for the response body, e.g.
-              MAX_CHILD_NODE_SIZE_EXCEEDED (defaults to INJECTED)
-
 The rule is scoped to the current DATABRICKS_TOKEN so it only affects
 the test that registers it, even when tests share a server.
 """
 
+import argparse
 import json
 import os
 import sys
 import urllib.request
 
+parser = argparse.ArgumentParser(description="Set up a fault rule on the testserver for the current test token.")
+parser.add_argument(
+    "pattern", help='HTTP method and path, supports trailing * wildcard, e.g. "PUT /api/2.0/permissions/pipelines/*"'
+)
+parser.add_argument("status_code", type=int, help="HTTP status code to return, e.g. 504")
+parser.add_argument("offset", type=int, help="number of requests to let through before fault starts")
+parser.add_argument("times", type=int, help="number of times to return the fault response")
+parser.add_argument(
+    "error_code",
+    nargs="?",
+    default="INJECTED",
+    help="error_code for the response body, e.g. MAX_CHILD_NODE_SIZE_EXCEEDED",
+)
+parser.add_argument(
+    "--after-handler",
+    action="store_true",
+    help="run the handler first; keep its effect and replace only the response",
+)
+parser.add_argument(
+    "--body-contains",
+    default="",
+    metavar="SUBSTR",
+    help="only fire when the request body contains SUBSTR. Needed to target a single file's "
+    "/workspace/import upload, since every upload shares the same method+path and only "
+    'differs by the multipart "path" form field.',
+)
+args = parser.parse_args()
+
 host = os.environ.get("DATABRICKS_HOST", "")
 token = os.environ.get("DATABRICKS_TOKEN", "")
 
 if not host:
     print("DATABRICKS_HOST not set", file=sys.stderr)
     sys.exit(1)
 
-args = sys.argv[1:]
-after_handler = "--after-handler" in args
-if after_handler:
-    args.remove("--after-handler")
-
-if len(args) not in (4, 5):
-    print(f"usage: {sys.argv[0]} [--after-handler] PATTERN STATUS_CODE OFFSET TIMES [ERROR_CODE]", file=sys.stderr)
-    sys.exit(1)
-
-pattern, status_code, offset, times = args[0], int(args[1]), int(args[2]), int(args[3])
-error_code = args[4] if len(args) == 5 else "INJECTED"
-body = json.dumps({"error_code": error_code, "message": "Fault injected by test."})
+body = json.dumps({"error_code": args.error_code, "message": "Fault injected by test."})
 
 data = json.dumps(
     {
-        "pattern": pattern,
-        "status_code": status_code,
+        "pattern": args.pattern,
+        "status_code": args.status_code,
         "body": body,
-        "offset": offset,
-        "times": times,
-        "after_handler": after_handler,
+        "offset": args.offset,
+        "times": args.times,
+        "after_handler": args.after_handler,
+        "body_contains": args.body_contains,
     }
 ).encode()
 

acceptance/bin/update_file.py

@@ -20,9 +20,14 @@
 # call update_file.py on that file.
 assert filename != "output.txt"
 
-data = open(filename).read()
+# Read raw and normalize CRLF to LF so the search string (always LF in the
+# script) matches on Windows, where the file is checked out with CRLF. Write
+# with newline="" so Python does not turn the \n back into \r\n: the acceptance
+# harness treats every file as LF, and a stray \r would change any uploaded
+# bytes or content hash a test later asserts.
+data = open(filename, newline="").read().replace("\r\n", "\n")
 newdata = data.replace(old, new)
 if newdata == data:
     sys.exit(f"{old=} not found in {filename=}\n{data}")
-with open(filename, "w") as fobj:
+with open(filename, "w", newline="") as fobj:
     fobj.write(newdata)

acceptance/bundle/ai_runtime_task/local_code_source/.gitattributes

@@ -0,0 +1,9 @@
+# These files are packaged verbatim into the .air_snapshots tarballs whose
+# uploaded sizes the test asserts, so their line endings must stay \n on every
+# OS — a Windows \r would change every recorded payload size.
+src/.gitignore text eol=lf
+src/command.sh text eol=lf
+src/data/model.bin text eol=lf
+src/train.py text eol=lf
+src2/command.sh text eol=lf
+src2/train.py text eol=lf

acceptance/bundle/ai_runtime_task/local_code_source/output.txt

@@ -21,19 +21,29 @@ src/train.py
 
 === both code_source_paths point into the bundle .air_snapshots, command_paths rewritten, deps on environments spec
 
->>> print_requests.py --sort --del-field raw_body //.air_snapshots/ //jobs/create
+>>> print_requests.py --sort //workspace/import //jobs/create
 {
   "method": "POST",
-  "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz",
-  "q": {
-    "overwrite": "true"
+  "path": "/api/2.0/workspace/import",
+  "body": {
+    "multipart_form": {
+      "content": "[binary content 189 bytes]",
+      "format": "AUTO",
+      "overwrite": "true",
+      "path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz"
+    }
   }
 }
 {
   "method": "POST",
-  "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz",
-  "q": {
-    "overwrite": "true"
+  "path": "/api/2.0/workspace/import",
+  "body": {
+    "multipart_form": {
+      "content": "[binary content 273 bytes]",
+      "format": "AUTO",
+      "overwrite": "true",
+      "path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz"
+    }
   }
 }
 {
@@ -114,12 +124,17 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> print_requests.py --sort --del-field raw_body //.air_snapshots/
+>>> print_requests.py --sort //workspace/import
 {
   "method": "POST",
-  "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz",
-  "q": {
-    "overwrite": "true"
+  "path": "/api/2.0/workspace/import",
+  "body": {
+    "multipart_form": {
+      "content": "[binary content 276 bytes]",
+      "format": "AUTO",
+      "overwrite": "true",
+      "path": "/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz"
+    }
   }
 }
 

acceptance/bundle/ai_runtime_task/local_code_source/script

@@ -11,18 +11,23 @@ title "each task's tarball holds only synced files (both under the repo root .ai
 trace list_code_snapshot.py
 
 title "both code_source_paths point into the bundle .air_snapshots, command_paths rewritten, deps on environments spec\n"
-# --del-field raw_body drops the binary tarball upload payload (kept readable). Filters
-# use a leading // so Git Bash on Windows does not path-convert them. --keep is not
-# passed, so print_requests.py consumes out.requests.txt.
-trace print_requests.py --sort --del-field raw_body '//.air_snapshots/' '//jobs/create'
+# Uploads go to POST /workspace/import, which carries the target filename in the
+# multipart body rather than the URL, so .air_snapshots is matched with jq on the
+# recorded body instead of by path filter. The tarball payload is binary and is
+# already summarized as "[binary content N bytes]" by the request recorder.
+# Filters use a leading // so Git Bash on Windows does not path-convert them.
+# --keep is not passed, so print_requests.py consumes out.requests.txt.
+trace print_requests.py --sort '//workspace/import' '//jobs/create' |
+  jq 'select((.body.multipart_form.path // .path) | test("[.]air_snapshots/|/jobs/create"))'
 
 title "re-planning unchanged code is a no-op (no changes)\n"
 trace $CLI bundle plan
 
 title "editing a file changes the snapshot hash (content-addressed name changes)\n"
 update_file.py src/train.py 'print("training")' 'print("training v2")'
 trace $CLI bundle deploy
-trace print_requests.py --sort --del-field raw_body '//.air_snapshots/'
+trace print_requests.py --sort '//workspace/import' |
+  jq 'select((.body.multipart_form.path // .path) | test("[.]air_snapshots/"))'
 
 title "destroy removes the deployed bundle (including the synced snapshots)\n"
 trace $CLI bundle destroy --auto-approve

acceptance/bundle/apps/app_yaml/out.app.yml.txt

@@ -1,8 +1,12 @@
 {
   "method": "POST",
-  "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app/app.yml",
-  "q": {
-    "overwrite": "true"
-  },
-  "raw_body": "command:\n  - python\n  - app.py\n"
+  "path": "/api/2.0/workspace/import",
+  "body": {
+    "multipart_form": {
+      "content": "command:\n  - python\n  - app.py\n",
+      "format": "AUTO",
+      "overwrite": "true",
+      "path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/app/app.yml"
+    }
+  }
 }

acceptance/bundle/apps/app_yaml/output.txt

@@ -19,7 +19,7 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> jq select(.path | test("app.yml")) out.requests.txt
+>>> jq select(.body.multipart_form.path | strings | test("app.yml")) out.requests.txt
 
 >>> [CLI] bundle destroy --auto-approve
 The following resources will be deleted:

acceptance/bundle/apps/app_yaml/script

@@ -1,7 +1,7 @@
 trace $CLI bundle validate
 trace $CLI bundle plan
 trace $CLI bundle deploy
-trace jq 'select(.path | test("app.yml"))' out.requests.txt | sed 's/\\r//g'  > out.app.yml.txt
+trace jq 'select(.body.multipart_form.path | strings | test("app.yml"))' out.requests.txt | sed 's/\\r//g'  > out.app.yml.txt
 #trace print_requests.py //apps  # currently fails due to TF inserting description=""
 rm out.requests.txt
 

acceptance/bundle/artifacts/ai_runtime_code_source/output.txt

@@ -24,5 +24,5 @@ Deployment complete!
 }
 
 === Expecting the code_source tarball to be uploaded
->>> jq .path
-"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/code.tgz"
+>>> jq -r .body.multipart_form.path | strings
+/Workspace/foo/bar/artifacts/.internal/code.tgz

acceptance/bundle/artifacts/ai_runtime_code_source/script

@@ -4,6 +4,6 @@ title "Expecting code_source_path rewritten to the uploaded artifact remote path
 trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks[].ai_runtime_task' out.requests.txt
 
 title "Expecting the code_source tarball to be uploaded"
-trace jq .path < out.requests.txt | grep import | grep code.tgz | sort
+trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep code.tgz | sort
 
 rm out.requests.txt

acceptance/bundle/artifacts/artifact_upload_for_workspace/output.txt

@@ -50,9 +50,9 @@ Deployment complete!
 ]
 
 === Expecting wheel to be uploaded
->>> jq .path
-"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl"
-"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl"
+>>> jq -r .body.multipart_form.path | strings
+/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl
+/Workspace/foo/bar/artifacts/.internal/source.whl
 
 === Expecting environment dependencies to be updated
 >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments out.requests.txt

acceptance/bundle/artifacts/artifact_upload_for_workspace/script

@@ -8,7 +8,7 @@ title "Expecting 2 wheels in libraries section in /jobs/create"
 trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt
 
 title "Expecting wheel to be uploaded"
-trace jq .path < out.requests.txt | grep import | grep whl | sort
+trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort
 
 title "Expecting environment dependencies to be updated"
 trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt

acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/output.txt

@@ -6,9 +6,9 @@ Deploying resources...
 Deployment complete!
 
 === Expecting wheel to be uploaded
->>> jq .path
-"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl"
-"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source.whl"
+>>> jq -r .body.multipart_form.path | strings
+/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source.whl
+/Workspace/foo/bar/artifacts/.internal/source.whl
 
 === Expecting delete request to artifact_path/.internal folder
 >>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) out.requests.txt

acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/script

@@ -5,7 +5,7 @@ echo "test wheel content" > whl/source.whl
 trace $CLI bundle deploy
 
 title "Expecting wheel to be uploaded"
-trace jq .path < out.requests.txt | grep import | grep whl | sort
+trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort
 
 title "Expecting delete request to artifact_path/.internal folder"
 trace jq -s '.[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal"))' out.requests.txt

acceptance/bundle/artifacts/upload_multiple_libraries/output.txt

@@ -40,15 +40,15 @@ Deployment complete!
 ]
 
 === Expecting 4 wheels to be uploaded
->>> jq .path
-"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source1.whl"
-"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source2.whl"
-"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source3.whl"
-"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source4.whl"
-"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source1.whl"
-"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source2.whl"
-"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source3.whl"
-"/api/2.0/workspace-files/import-file/Workspace/foo/bar/artifacts/.internal/source4.whl"
+>>> jq -r .body.multipart_form.path | strings
+/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source1.whl
+/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source2.whl
+/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source3.whl
+/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/whl/source4.whl
+/Workspace/foo/bar/artifacts/.internal/source1.whl
+/Workspace/foo/bar/artifacts/.internal/source2.whl
+/Workspace/foo/bar/artifacts/.internal/source3.whl
+/Workspace/foo/bar/artifacts/.internal/source4.whl
 
 === Expecting environment dependencies to be updated
 >>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | .body.environments out.requests.txt

acceptance/bundle/artifacts/upload_multiple_libraries/script

@@ -11,7 +11,7 @@ title "Expecting 5 wheels in libraries section in /jobs/create"
 trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.tasks' out.requests.txt
 
 title "Expecting 4 wheels to be uploaded"
-trace jq .path < out.requests.txt | grep import | grep whl | sort
+trace jq -r '.body.multipart_form.path | strings' < out.requests.txt | grep whl | sort
 
 title "Expecting environment dependencies to be updated"
 trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt