diff --git a/Documentation/Architecture/ExtractionManifest.json b/Documentation/Architecture/ExtractionManifest.json index 66bf9f6..4d9844a 100644 --- a/Documentation/Architecture/ExtractionManifest.json +++ b/Documentation/Architecture/ExtractionManifest.json @@ -197,7 +197,8 @@ "Private FIPS 180-4 SHA-256 is an original Swift implementation with checked cumulative production accounting, bounded scratch and inner cancellation. The separate trusted fixture tool mirrors its primitive core byte-for-byte; public_check verifies parity. No CryptoKit fallback or runtime dependency is introduced.", "JSONHTML formatted rendering, original fixed CSS and shared strict-writer token classification are original Apache-2.0 code. No third-party renderer, browser numeric authority or runtime dependency is adapted.", "JSONHTML typed annotations, iterative exact-path binding, static/interactive attachment and coherent JSONDocument span presentation are original Apache-2.0 code. No schema engine, third-party renderer, source archive or new dependency is adapted.", - "CLI foundation, argv parser, POSIX adapters, prepaid bounded diagnostics and shared-meter package adapters are original Apache-2.0 Swift code. No legacy filter engine or jq source is copied. See Release1.4/Foundation.md for native test-harness reuse provenance." + "CLI foundation, argv parser, POSIX adapters, prepaid bounded diagnostics and shared-meter package adapters are original Apache-2.0 Swift code. No legacy filter engine or jq source is copied. See Release1.4/Foundation.md for native test-harness reuse provenance.", + "Tests/Fixtures/CLI/NewCases.json is the byte-identical 111-case authored CLI briefing fixture, SHA-256 6f863f3e98d2dea12e59018e7cf3a074ae0ebee5f1ab950a651df72e5eb18a4c. Fixture expected JSON texts use native exact comparison, not a floating-point oracle. The query compiler/VM and lifetime ledger are original Apache-2.0 implementations; no legacy eager evaluator or jq implementation is copied." ], "policy": "Record actual adaptations and retain applicable upstream copyrights and licenses. Reference inputs and comparison experiments are not distributed runtime dependencies.", "jsonhtml_export_original_contributions": { diff --git a/Documentation/Architecture/Release1.4/Acceptance.json b/Documentation/Architecture/Release1.4/Acceptance.json index e0743ac..269cb7d 100644 --- a/Documentation/Architecture/Release1.4/Acceptance.json +++ b/Documentation/Architecture/Release1.4/Acceptance.json @@ -1,7 +1,7 @@ { "schema": "swift-json-cli-acceptance/1", "prepared": "2026-10-14", - "implementation_status": "foundation boundary only; integration unqualified", + "implementation_status": "Foundation and pull query-core milestones; complete C0-C5 integration remains unqualified", "requirements": [ { "id": "CL001", @@ -366,9 +366,13 @@ "title": "Native query lexer/literals", "verification": "Exact numbers and scalar text/keys with source byte ranges; malformed source compiles to exit3.", "obligation": "mandatory", - "status": "not_run", + "status": "partial", "evidence_required": true, - "fixture_ids": [] + "fixture_ids": [], + "evidence": [ + "Documentation/Architecture/Release1.4/QueryCore.md", + "Documentation/Architecture/Release1.4/Qualification.md" + ] }, { "id": "CL028", @@ -376,10 +380,14 @@ "title": "Grammar precedence", "verification": "Pipeline/comma/default/assignment/operator precedence and delimiters tested beyond old parser goldens.", "obligation": "mandatory", - "status": "not_run", + "status": "partial", "evidence_required": true, "fixture_ids": [ "pipeline-comma-precedence" + ], + "evidence": [ + "Documentation/Architecture/Release1.4/QueryCore.md", + "Documentation/Architecture/Release1.4/Qualification.md" ] }, { @@ -388,9 +396,13 @@ "title": "Pull VM cardinality", "verification": "Empty/one/many outputs without whole-job array; next() demand limits observable reads/retention.", "obligation": "mandatory", - "status": "not_run", + "status": "passed", "evidence_required": true, - "fixture_ids": [] + "fixture_ids": [], + "evidence": [ + "Documentation/Architecture/Release1.4/QueryCore.md", + "Documentation/Architecture/Release1.4/Qualification.md" + ] }, { "id": "CL030", @@ -398,9 +410,13 @@ "title": "Pipeline and comma order", "verification": "Every upstream result is evaluated and ordered; late errors do not resume a failed branch.", "obligation": "mandatory", - "status": "not_run", + "status": "passed", "evidence_required": true, - "fixture_ids": [] + "fixture_ids": [], + "evidence": [ + "Documentation/Architecture/Release1.4/QueryCore.md", + "Documentation/Architecture/Release1.4/Qualification.md" + ] }, { "id": "CL031", @@ -408,10 +424,14 @@ "title": "Cartesian operators/constructors", "verification": "No .first collapse; object versus arithmetic nesting orders and shared input effects are tested.", "obligation": "mandatory", - "status": "not_run", + "status": "partial", "evidence_required": true, "fixture_ids": [ "object-product-order" + ], + "evidence": [ + "Documentation/Architecture/Release1.4/QueryCore.md", + "Documentation/Architecture/Release1.4/Qualification.md" ] }, { @@ -420,7 +440,7 @@ "title": "Navigation and wrong-type", "verification": "Missing/null convention, has, wrong-type error, optional catch and checked indices/slices.", "obligation": "mandatory", - "status": "not_run", + "status": "passed", "evidence_required": true, "fixture_ids": [ "missing-field-null", @@ -428,6 +448,10 @@ "optional-wrong-type-empty", "has-present-null", "negative-index" + ], + "evidence": [ + "Documentation/Architecture/Release1.4/QueryCore.md", + "Documentation/Architecture/Release1.4/Qualification.md" ] }, { @@ -436,7 +460,7 @@ "title": "Exact Unicode identity", "verification": "Scalar-distinct strings/keys, scalar length/slicing/order and non-BMP behavior.", "obligation": "mandatory", - "status": "not_run", + "status": "passed", "evidence_required": true, "fixture_ids": [ "scalar-key-lookup", @@ -444,6 +468,10 @@ "scalar-length", "utf8-length", "scalar-slice" + ], + "evidence": [ + "Documentation/Architecture/Release1.4/QueryCore.md", + "Documentation/Architecture/Release1.4/Qualification.md" ] }, { @@ -452,7 +480,7 @@ "title": "Truth/default/Boolean generators", "verification": "Only false/null falsey; // handles full streams; left-driven short-circuit and if generators.", "obligation": "mandatory", - "status": "not_run", + "status": "passed", "evidence_required": true, "fixture_ids": [ "truthiness", @@ -460,6 +488,10 @@ "fallback-empty", "Boolean-short-circuit", "if-condition-stream" + ], + "evidence": [ + "Documentation/Architecture/Release1.4/QueryCore.md", + "Documentation/Architecture/Release1.4/Qualification.md" ] }, { @@ -468,10 +500,14 @@ "title": "Constructed duplicate keys", "verification": "Explicit replace-preserve-position policy; parsed exact duplicate keys still reject.", "obligation": "mandatory", - "status": "not_run", + "status": "passed", "evidence_required": true, "fixture_ids": [ "constructed-key-replacement" + ], + "evidence": [ + "Documentation/Architecture/Release1.4/QueryCore.md", + "Documentation/Architecture/Release1.4/Qualification.md" ] }, { @@ -480,9 +516,13 @@ "title": "Exact comparison/order", "verification": "Metered decimal/string/array/object equality and total order with stable ties.", "obligation": "mandatory", - "status": "not_run", + "status": "passed", "evidence_required": true, - "fixture_ids": [] + "fixture_ids": [], + "evidence": [ + "Documentation/Architecture/Release1.4/QueryCore.md", + "Documentation/Architecture/Release1.4/Qualification.md" + ] }, { "id": "CL037", @@ -490,7 +530,7 @@ "title": "Common builtin inventory", "verification": "All C2 builtins and type/domain/empty cases; map collects and map_values updates/deletes correctly.", "obligation": "mandatory", - "status": "not_run", + "status": "partial", "evidence_required": true, "fixture_ids": [ "empty-array-collector", @@ -498,6 +538,10 @@ "map-values-first", "map-values-delete", "normals-nonzero-native" + ], + "evidence": [ + "Documentation/Architecture/Release1.4/QueryCore.md", + "Documentation/Architecture/Release1.4/Qualification.md" ] }, { @@ -506,11 +550,15 @@ "title": "Lexical bindings/arguments", "verification": "as, --arg/argjson/rawfile/slurpfile, $ARGS, duplicate/reserved names, strict auxiliary-file admission.", "obligation": "mandatory", - "status": "not_run", + "status": "partial", "evidence_required": true, "fixture_ids": [ "binding-original-input", "argjson-exact" + ], + "evidence": [ + "Documentation/Architecture/Release1.4/QueryCore.md", + "Documentation/Architecture/Release1.4/Qualification.md" ] }, { @@ -519,11 +567,15 @@ "title": "Shared input cursor", "verification": "Driver/input/inputs share consumption; EOF catchable only for input; malformed tails/IO remain fatal.", "obligation": "mandatory", - "status": "not_run", + "status": "passed", "evidence_required": true, "fixture_ids": [ "null-input-once", "shared-inputs" + ], + "evidence": [ + "Documentation/Architecture/Release1.4/QueryCore.md", + "Documentation/Architecture/Release1.4/Qualification.md" ] }, { @@ -532,13 +584,17 @@ "title": "JSON/raw framing", "verification": "values/document/NDJSON and strict raw UTF-8 lines/slurp; files have independent boundaries.", "obligation": "mandatory", - "status": "not_run", + "status": "passed", "evidence_required": true, "fixture_ids": [ "values-framing", "adjacent-roots-rejected", "ndjson-blank-line-rejected", "raw-line-crlf" + ], + "evidence": [ + "Documentation/Architecture/Release1.4/QueryCore.md", + "Documentation/Architecture/Release1.4/Qualification.md" ] }, { @@ -547,13 +603,17 @@ "title": "Raw output/NUL/join", "verification": "Correct string bytes and nonstring JSON; embedded NUL rejected; delimiters/errors/counts tested.", "obligation": "mandatory", - "status": "not_run", + "status": "passed", "evidence_required": true, "fixture_ids": [ "raw-lf", "raw-nul-delimiter", "raw-nul-value-rejected", "join-no-delimiter" + ], + "evidence": [ + "Documentation/Architecture/Release1.4/QueryCore.md", + "Documentation/Architecture/Release1.4/Qualification.md" ] }, { @@ -562,10 +622,14 @@ "title": "ASCII output and sort keys", "verification": "Owned escaping, surrogate pairs and exact scalar order; option conflicts tested.", "obligation": "mandatory", - "status": "not_run", + "status": "passed", "evidence_required": true, "fixture_ids": [ "ascii-surrogate-pair" + ], + "evidence": [ + "Documentation/Architecture/Release1.4/QueryCore.md", + "Documentation/Architecture/Release1.4/Qualification.md" ] }, { @@ -574,12 +638,16 @@ "title": "Query exit-status", "verification": "Final semantic value only after completion; truthy0/empty string/collections; false/null1 and noresult4.", "obligation": "mandatory", - "status": "not_run", + "status": "passed", "evidence_required": true, "fixture_ids": [ "exit-status-truthy-zero", "exit-status-false", "exit-status-empty" + ], + "evidence": [ + "Documentation/Architecture/Release1.4/QueryCore.md", + "Documentation/Architecture/Release1.4/Qualification.md" ] }, { @@ -588,9 +656,13 @@ "title": "Plan reuse and isolation", "verification": "Concurrent independent invocations share immutable plan only; mutable iterator not concurrently accessed.", "obligation": "mandatory", - "status": "not_run", + "status": "passed", "evidence_required": true, - "fixture_ids": [] + "fixture_ids": [], + "evidence": [ + "Documentation/Architecture/Release1.4/QueryCore.md", + "Documentation/Architecture/Release1.4/Qualification.md" + ] }, { "id": "CL045", diff --git a/Documentation/Architecture/Release1.4/Progress.json b/Documentation/Architecture/Release1.4/Progress.json index 176a9e8..a9f5c5f 100644 --- a/Documentation/Architecture/Release1.4/Progress.json +++ b/Documentation/Architecture/Release1.4/Progress.json @@ -3,12 +3,12 @@ "target_release": "1.4.0", "renderer_base": "3c1e8a1a1bc8d03699665b8ab98f5ece7962aff7", "main_at_orientation": "25fb04f747e583209b4c4511abf39e11ef5b4f80", - "branch": "codex/1.4-json-cli-foundation", - "status": "Compiling foundation boundary qualified on native macOS; C0/C1 complete closure and C2-C5 remain pending", + "branch": "codex/1.4-json-cli-query-core", + "status": "Native pull query-core milestone qualified; complete C0-C5 integration remains open", "gates": { "C0": "partial", "C1": "partial", - "C2": "not_run", + "C2": "partial", "C3": "not_run", "C4": "not_run", "C5": "not_run" @@ -21,9 +21,9 @@ "core_metrics": 49, "cli_metrics": 16, "publication_performed": false, - "root_test_functions": 232, - "root_parameter_cases": 177, - "native_source_manifest_sha256": "864d9b3dd8e1d79a0a6cbec862476843ddca2ff8a1582471b80fba54b863459e", + "root_test_functions": 256, + "root_parameter_cases": 199, + "native_source_manifest_sha256": "c8f09d2eaa6cf14f3efb11e8a2645796075c54274fa6b8af9db464fe27b173aa", "qualified_lanes": [ "native", "safety", @@ -32,10 +32,31 @@ "remaining_foundation_requirements": [ "clean-source reproduction", "selected mobile SDK consumers", - "immutable filter-plan compiler probe and implemented IR", - "all 16 CLI cap boundaries", - "multi-record shared accounting", "slow/partial/no-progress sink probes", - "query compile exit 3 and full exit precedence" - ] + "query compile exit 3 and full exit precedence", + "actual operation-site coverage for all 16 CLI caps", + "inherited corpus/oracle compatibility dispositions", + "complete query grammar with C3/C4 and auxiliary binding subprocess coverage" + ], + "history": [ + { + "branch": "codex/1.4-json-cli-foundation", + "status": "Compiling foundation boundary qualified on native macOS; C0/C1 complete closure and C2-C5 remain pending", + "gates": { + "C0": "partial", + "C1": "partial", + "C2": "partial", + "C3": "not_run", + "C4": "not_run", + "C5": "not_run" + }, + "root_test_functions": 232, + "root_parameter_cases": 177, + "native_source_manifest_sha256": "864d9b3dd8e1d79a0a6cbec862476843ddca2ff8a1582471b80fba54b863459e" + } + ], + "query_functions": 24, + "query_parameter_cases": 22, + "query_vectors": 38, + "query_vectors_passed": 38 } diff --git a/Documentation/Architecture/Release1.4/Qualification.md b/Documentation/Architecture/Release1.4/Qualification.md index 460214c..b3d4622 100644 --- a/Documentation/Architecture/Release1.4/Qualification.md +++ b/Documentation/Architecture/Release1.4/Qualification.md @@ -48,3 +48,44 @@ performed by this boundary. Materialized receipts and their exact source-input records are in the local foundation evidence bundle. Run the listed commands to regenerate evidence; local build outputs and private input packets are excluded from public Git/source archives. + +## Pull query-core milestone + +Base foundation f469f1d (draft PR #23), stacked query branch +`codex/1.4-json-cli-query-core`. Native Swift 6.4/Xcode 27.2/SDK 27.2/macOS 27 +qualification ran through Scripts/ci.py on source manifest +`c8f09d2eaa6cf14f3efb11e8a2645796075c54274fa6b8af9db464fe27b173aa`. +Native: 20 commands, exits 0; debug/release each 256 root functions and 199 parameter +invocations. Safety: ASan same full counts; TSan eight shared-state functions; +strict memory safety/warnings as errors compiled cleanly. DocC: five catalogs. +All three receipts record unchanged source inputs. Receipts/logs/event streams: +`.build/ci/cli-c2-native-serial`, `cli-c2-final-safety`, `cli-c2-final-docs`. + +Foundation subprocess cases remain 53/53; authored C2 cases 38/38, policy 0. +The public exact comparator uses JSON/JSONStreaming. Original 49 cap probes, +318 JSON cases (287 evaluated, 31 policy) and 2,337 schema cases (2,300 evaluated, +37 policy), authored 82/9/4,396 and the 20 rooted downstream cases are preserved. + +Additional review tests verify quota failure cleanup during keys/slices/maps, +objects and catch-message construction. Cancellation after a first yield checks +terminal behavior plus zero live frames/collections/input storage. A first test +retained a second optional alias; it was corrected to release that reference. +An initial parallel native/docs invocation failed because SwiftPM's lock-wait +banner preceded dump-package JSON. The complete serial native rerun passed; +the failed receipt is retained and is not a package qualification pass. + +These receipts precede only the milestone ledger/prose update. Runtime/test/harness +source remains identical; final C5 qualification will use the integration head. +The inherited 168 corpus rows and auxiliary binding modes still need their complete +mode-specific dispositions. Arithmetic and advanced filters have separate gates. +No simulator/device, Linux or WASI CLI success is inferred from this local run. + + +Hosted C2 Linux runs on x86_64/ARM64 passed 256 root functions/199 parameter cases, +ASan/TSan safety and the 53 CLI subprocess cases, then the comparator failed to +compile: Glibc's mutable `stderr` global is not concurrency-safe in async main. +The qualification tool now writes bounded failure text through +FileHandle.standardError. No unsafe annotation/flag or library behavior changed. +Failed run 38593758299 and its actual compiler log are retained; a fresh hosted +run is required before Linux closure. Local comparator rebuild and all 38 C2 +vectors verify the diagnostic-only fix on the supported Mac. diff --git a/Documentation/Architecture/Release1.4/QueryCore.md b/Documentation/Architecture/Release1.4/QueryCore.md new file mode 100644 index 0000000..9709179 --- /dev/null +++ b/Documentation/Architecture/Release1.4/QueryCore.md @@ -0,0 +1,55 @@ +# Pull query core + +The dependent query branch builds on foundation `f469f1d` / draft PR #23. +Its private compiler produces a flat immutable checked-Sendable node/range array. +Token grouping and compilation use explicit work stacks, metered source/token/IR +and scratch admission. The evaluator uses explicit demand continuations and reuses +arena slots. It does not port the previous eager `[JSONValue]` evaluator. + +The current core includes ordered comma/pipe generators, exact paths/index/slices, +recursive descent, arrays/object Cartesian construction, conditions, fallback, +short-circuit booleans, bindings, typed try/optional, input/inputs and the C2 primitive +builtin inventory. Constructed duplicates replace values at their first position; +parsed duplicates remain strict errors. Strings/keys use decoded scalar identity. +Comparison uses native exact decimals and metered iterative container walks. +Arithmetic/assignment/richer calls remain explicit compile errors until their gates. + +One CLIInputCursor feeds the native JSONTokenMachine/JSONValueAssembler. Values +require real whitespace or file EOF; documents wait for EOF; NDJSON rejects blank +physical lines; root-array elements are provisional until enclosing syntax and EOF +finish. Raw lines use strict UTF-8 and distinct CRLF rules. Driver and input/inputs +share the same cursor. Successful execution drains unconsumed tails, including null +input and short-circuit cases; accepted stdout cannot be recalled. + +Invocation-only value leases keep selected aliases and replay/cache values charged +until their final release. New value graphs undergo full owned native admission; +selectors share their source reservation. ARC notifications enqueue checked refunds +in the job's private cell, drained before the next checkpoint. Work never refunds; +terminal storage cleanup cannot replace or reopen the native meter. Plans alone are +Sendable; mutable leases are never presented as shared values. Logical retention is +not process RSS or a claim about framework/kernel instructions. + +The 38 byte-identical authored C2 vectors pass using the public JSON/JSONStreaming +CLIExactCompare consumer. It preserves exact decimals and scalar identity, and +handles the CLI's empty result stream explicitly; the library sequence's existing +empty-input behavior is unchanged. Fixture SHA-256: +`6f863f3e98d2dea12e59018e7cf3a074ae0ebee5f1ab950a651df72e5eb18a4c`. + +Current tests include sixteen ledger boundary cases, four actual compiler admission +cases, configured depth/frame tests and alias/terminal release. Retention checks over +2,000 and 20,000 records finish with zero input storage and stay within the derived +bound of two 16 KiB buffers plus a scalar/lease reservation. An initial arbitrary +20,000-byte threshold omitted the measured syscall/chunk overlap; the final bound +comes from the actual adapter and additionally checks a tenfold input increase. +An omitted slice bound now explicitly uses Optional.none, avoiding Swift's inference +of JSON null through ExpressibleByNilLiteral. Swift Testing macros evaluate an inout +noncopyable call before #require; no unsafe ownership annotation repairs it. + +The complete C0-C5 release boundary remains open. Backend quota rendezvous, pinned independent +oracle/corpus results, all platform consumers and reproduction remain required. + +The final milestone ran 36 CLI functions (including 24 new query functions) within +256 root functions, 199 parameter cases, native/release/ASan and eight TSan shared +cases. Failure cleanup tests cover partial constructors/selectors and catch-message +ownership. Arithmetic precedence/Cartesian behavior, full binding subprocess modes, +inherited fixture dispositions and final integration remain open requirements. diff --git a/Scripts/ci.py b/Scripts/ci.py index 632ff34..0d04736 100644 --- a/Scripts/ci.py +++ b/Scripts/ci.py @@ -7,7 +7,7 @@ from testing_events import read_events, ROOT_PARAMETER_CASES ROOT = pathlib.Path(__file__).resolve().parents[1] -SHARED_TESTS = 'independentCodecFacadesAndExactValuesCanBeSharedAcrossTasks|concurrentSchemaEvaluationsHaveExclusiveDynamicRegexAndAnnotationState|sharedRegexProgramHasExclusiveConcurrentSearchState|immutableIndexedObjectCanBeSharedAcrossTasks|qaSharedPayloadAndRootHaveIndependentConcurrentReadScopes|sha256InputCapacityAndConcurrentInvocationsRemainIndependent|sharedFormattedHTMLRendererHasIndependentConcurrentInvocations|sharedAnnotatedDocumentRendererHasIndependentConcurrentInvocations|sharedExportPreparationsHaveIndependentConcurrentState|applePDFSimultaneousJobsHaveIndependentStateAndCancellation|e2SharedBridgeInputsHaveInvocationOwnedState|e2SharedCompiledPlanFacadeHasIndependentInvocationState' +SHARED_TESTS = 'independentCodecFacadesAndExactValuesCanBeSharedAcrossTasks|concurrentSchemaEvaluationsHaveExclusiveDynamicRegexAndAnnotationState|sharedRegexProgramHasExclusiveConcurrentSearchState|immutableIndexedObjectCanBeSharedAcrossTasks|qaSharedPayloadAndRootHaveIndependentConcurrentReadScopes|sha256InputCapacityAndConcurrentInvocationsRemainIndependent|sharedFormattedHTMLRendererHasIndependentConcurrentInvocations|sharedAnnotatedDocumentRendererHasIndependentConcurrentInvocations|sharedExportPreparationsHaveIndependentConcurrentState|applePDFSimultaneousJobsHaveIndependentStateAndCancellation|sharedFilterPlanHasIndependentConcurrentInvocations|e2SharedBridgeInputsHaveInvocationOwnedState|e2SharedCompiledPlanFacadeHasIndependentInvocationState' R0_TESTS = 'readerProbe|indexProbes|duplicateRecognitionPrecedesValueAndClosingSyntax' def snapshot(): @@ -54,7 +54,7 @@ def commands(lane, output): ['python3', '-B', 'Scripts/qualify_html_browser.py', '--fixtures', str(output / 'fixtures'), '--output', str(output / 'browser')], ] if lane == 'cli': - return [['swift', 'build', '-c', 'release', '--product', 'swift-json'], ['swift', 'test', '--filter', 'cli'], ['python3', '-B', 'Scripts/qualify_cli.py', '--binary', '.build/release/swift-json', '--output', str(output / 'subprocess.json')]] + return [['swift', 'build', '-c', 'release', '--product', 'swift-json'], ['swift', 'test', '--filter', 'cli|sharedFilterPlan'], ['python3', '-B', 'Scripts/qualify_cli.py', '--binary', '.build/release/swift-json', '--output', str(output / 'subprocess.json')], ['swift', 'build', '-c', 'release', '--package-path', 'Tools/CLIQualification', '--product', 'CLIExactCompare'], ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C2', '--output', str(output / 'vectors.json')]] if lane == 'native': return [ ['python3', '-B', 'Scripts/public_check.py'], @@ -64,6 +64,8 @@ def commands(lane, output): ['python3', '-B', 'Scripts/verify_testing_events.py'], ['swift', 'test'], ['swift', 'test', '-c', 'release'], ['python3', '-B', 'Scripts/qualify_cli.py', '--binary', '.build/release/swift-json', '--output', str(output / 'subprocess.json')], + ['swift', 'build', '-c', 'release', '--package-path', 'Tools/CLIQualification', '--product', 'CLIExactCompare'], + ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C2', '--output', str(output / 'vectors.json')], ['python3', '-B', 'Scripts/qualify_cli_ownership.py', '--output', str(output / 'cli-ownership')], ['python3', '-B', 'Scripts/qualify_release_1_1.py', '--mode', 'ownership', '--output', str(output / 'ownership.json')], ['python3', '-B', 'Scripts/qualify_release_1_1.py', '--mode', 'reader', '--output', str(output / 'reader.json')], @@ -117,12 +119,12 @@ def check_result(command, code, log, events=None): if events is not None: counts = events['bundle_counts'] if command[:2] == ['swift', 'test']: - expected = [12] if 'cli' in command else [38] if 'export|Export|apple' in command else [14] if 'Tools/WasmAudit' in command else [2, 3, 3, 11] if 'qa' in command else [7] if R0_TESTS in command else [3] if 'Tools/MCPReadiness' in command or 'Tools/MobileQualification' in command else [1] if 'Tools/Qualification' in command else [1, 1, 2, 3 if sys.platform == 'linux' else 4, 4] if 'thread' in command else sorted([10, 11, 12, 12, 14, 61 if sys.platform == 'linux' else 75, 55, 119]) + expected = [36] if 'cli|sharedFilterPlan' in command else [38] if 'export|Export|apple' in command else [14] if 'Tools/WasmAudit' in command else [2, 3, 3, 11] if 'qa' in command else [7] if R0_TESTS in command else [3] if 'Tools/MCPReadiness' in command or 'Tools/MobileQualification' in command else [1] if 'Tools/Qualification' in command else sorted([1, 1, 3 if sys.platform == 'linux' else 4, 1, 2, 4]) if 'thread' in command else sorted([119, 55, 14, 11, 12, 61 if sys.platform == 'linux' else 75, 36, 10]) passed = passed and sorted(counts) == expected if '--filter' not in command and not any(x.startswith('Tools/') for x in command): if events is not None: - passed = passed and events['module_function_counts'] == {'JSONTests':119,'JSONBridgesTests':10,'JSONSchemaTests':55,'JSONStreamingTests':14,'JSONPatchTests':11,'JSONRegexRuntimeTests':12,'JSONHTMLTests':61 if sys.platform == 'linux' else 75,'JSONCLITests':12} - passed = passed and (events['parameterized_functions'] == ROOT_PARAMETER_CASES if events is not None else sum(int(n) for n in re.findall(r'with (\d+) test cases passed', log)) == 177) + passed = passed and events['module_function_counts'] == {'JSONTests':119,'JSONSchemaTests':55,'JSONStreamingTests':14,'JSONPatchTests':11,'JSONRegexRuntimeTests':12,'JSONHTMLTests':61 if sys.platform == 'linux' else 75,'JSONCLITests':36,'JSONBridgesTests':10} + passed = passed and (events['parameterized_functions'] == ROOT_PARAMETER_CASES if events is not None else sum(int(n) for n in re.findall(r'with (\d+) test cases passed', log)) == 199) if 'Tools/Qualification' in command: probes = re.findall(r'^CAP_PROBE (\w+) (\S+)$', log, re.MULTILINE) matrix = json.loads((ROOT / 'Documentation/Qualification/LimitsMatrix.json').read_text()) diff --git a/Scripts/qualify_cli_cases.py b/Scripts/qualify_cli_cases.py new file mode 100644 index 0000000..c67fd12 --- /dev/null +++ b/Scripts/qualify_cli_cases.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Run authored CLI vectors and use an independent native exact-value consumer.""" +import argparse, hashlib, json, pathlib, subprocess, tempfile +ROOT=pathlib.Path(__file__).resolve().parents[1] +def main(): + p=argparse.ArgumentParser();p.add_argument('--binary',required=True);p.add_argument('--comparator',required=True);p.add_argument('--gate',default='C2');p.add_argument('--output',required=True);a=p.parse_args() + binary=(ROOT/a.binary).resolve(); comparator=(ROOT/a.comparator).resolve(); fixture=ROOT/'Tests/Fixtures/CLI/NewCases.json';declaration=json.loads(fixture.read_text());rows=[] + with tempfile.TemporaryDirectory(prefix='cli-vectors-') as directory: + directory=pathlib.Path(directory) + for case in declaration['cases']: + if case['gate']!=a.gate:continue + argv=[case['command'],*case.get('options',[])] + if case['command']=='query':argv += declaration['default_query_options']+[case['filter']] + argv += case.get('operands',[]) + result=subprocess.run([str(binary),*argv],input=case.get('stdin_utf8','').encode(),capture_output=True,timeout=30) + passed=result.returncode==case['expected_exit'];comparison=None + if 'expected_stdout_utf8' in case:passed=passed and result.stdout==case['expected_stdout_utf8'].encode() + if 'expected_json_values' in case and result.returncode==case['expected_exit']: + actual=directory/'actual.json';expected=directory/'expected.json';actual.write_bytes(result.stdout);expected.write_text(json.dumps(case['expected_json_values'],ensure_ascii=False)) + checked=subprocess.run([str(comparator),str(actual),str(expected)],capture_output=True,timeout=30) + comparison={'exit':checked.returncode,'stdout':checked.stdout.decode(),'stderr':checked.stderr.decode()};passed=passed and checked.returncode==0 + row={'id':case['id'],'requirement_ids':case['requirement_ids'],'command':['swift-json',*argv],'exit_code':result.returncode,'expected_exit':case['expected_exit'],'status':'passed' if passed else 'failed','stdout_sha256':hashlib.sha256(result.stdout).hexdigest(),'stderr_sha256':hashlib.sha256(result.stderr).hexdigest(),'comparison':comparison} + if not passed:row.update(stdout=result.stdout.decode(errors='replace'),stderr=result.stderr.decode(errors='replace'));print(json.dumps(row),flush=True) + rows.append(row) + receipt={'schema_version':1,'gate':a.gate,'status':'passed' if all(x['status']=='passed' for x in rows) and rows else 'failed','observed':len(rows),'passed':sum(x['status']=='passed' for x in rows),'failed':sum(x['status']=='failed' for x in rows),'policy':0,'fixture_sha256':hashlib.sha256(fixture.read_bytes()).hexdigest(),'binary_sha256':hashlib.sha256(binary.read_bytes()).hexdigest(),'comparator_sha256':hashlib.sha256(comparator.read_bytes()).hexdigest(),'cases':rows} + output=ROOT/a.output;output.parent.mkdir(parents=True,exist_ok=True);output.write_text(json.dumps(receipt,indent=2)+'\n');print(json.dumps({k:v for k,v in receipt.items() if k!='cases'}));return 0 if receipt['status']=='passed' else 1 +if __name__=='__main__':raise SystemExit(main()) diff --git a/Scripts/testing_events.py b/Scripts/testing_events.py index 6dc361d..e841074 100644 --- a/Scripts/testing_events.py +++ b/Scripts/testing_events.py @@ -4,6 +4,9 @@ import json ROOT_PARAMETER_CASES = { + 'cliEveryCounterHasInclusiveCheckedBoundaryAndTerminalFailure(metric:)': 16, + 'cliCompilerEnforcesCapsAtActualSourceTokenAndIRGrowth(metric:)': 4, + 'cliQueryManyTinyRecordsDoNotRetainTheWholeInputJob(records:)': 2, 'cliConflictingAndUnknownOptionsReject(argv:)': 4, 'equivalentDecimalSpellings(text:)': 4, 'inclusiveAdmission(attempt:)': 3, 'numberGrammarIsStrict(text:)': 10, 'exactDivisibilityUsesDecimalMathematics(a:b:expected:)': 6, diff --git a/Sources/JSONCLIEngine/CLIArguments.swift b/Sources/JSONCLIEngine/CLIArguments.swift index cce1435..7c4b502 100644 --- a/Sources/JSONCLIEngine/CLIArguments.swift +++ b/Sources/JSONCLIEngine/CLIArguments.swift @@ -8,12 +8,14 @@ package struct CLIArguments: Sendable { package var options: [String: String] = [:] package var repeated: [(String, String)] = [] package var hasHelp = false + package var capturedArguments: [String]? + package var captureJSONArguments = false private static let commonValues: Set = ["limits", "limits-file", "output", "diagnostics", "report"] private static let values: [String: Set] = [ "format": ["indent", "number-style"], "validate": ["schema", "resource", "max-issues"], "pointer": [], "patch": ["patch"], "stream": [], "schema": ["resource"], "render": ["format", "presentation", "theme", "title", "html"], - "query": ["filter-file", "input-mode", "indent", "number-style", "arg", "argjson", "rawfile", "slurpfile", "division-scale", "rounding"] + "query": ["filter-file", "input-framing", "indent", "number-style", "arg", "argjson", "rawfile", "slurpfile", "division-scale", "rounding"] ] private static let booleans: [String: Set] = [ "format": ["compact", "tab", "sort-keys"], "validate": [], "pointer": ["allow-missing"], "patch": [], @@ -47,10 +49,14 @@ package struct CLIArguments: Sendable { if argument == "--" { terminated = true; continue } if argument.hasPrefix("--") { let pair = argument.dropFirst(2).split(separator: "=", maxSplits: 1, omittingEmptySubsequences: false) - let name = String(pair[0]); let inline = pair.count == 2 ? String(pair[1]) : nil + let name = String(pair[0]) == "compact-output" ? "compact" : String(pair[0]); let inline = pair.count == 2 ? String(pair[1]) : nil if name == "help" || name == "overwrite" || Self.booleans[first]!.contains(name) { guard inline == nil else { throw CLIError(.usage, "booleanValue") } - try install(name, "true"); continue + try install(name, "true") + if name == "args" || name == "jsonargs" { + capturedArguments = Array(argv[i...]); captureJSONArguments = name == "jsonargs"; i = argv.count + } + continue } guard Self.commonValues.contains(name) || Self.values[first]!.contains(name) else { throw CLIError(.usage, "unknownOption") } let value: String @@ -76,6 +82,13 @@ package struct CLIArguments: Sendable { if options["compact"] != nil && (options["indent"] != nil || options["tab"] != nil) || options["indent"] != nil && options["tab"] != nil { throw CLIError(.usage, "conflictingIndentation") } if let indent = options["indent"], Int(indent).map({ (0...8).contains($0) }) != true { throw CLIError(.usage, "invalidIndentation") } if let style = options["number-style"], !["preserved", "normalized"].contains(style) { throw CLIError(.usage, "invalidNumberStyle") } + if command == "query" { + if options["null-input"] != nil && options["slurp"] != nil { throw CLIError(.usage, "nullSlurpConflict") } + if options["raw-input"] != nil && options["input-framing"] != nil { throw CLIError(.usage, "rawFramingConflict") } + let raw = ["raw-output", "raw-output0", "join-output"].filter { options[$0] != nil } + if raw.count > 1 || !raw.isEmpty && options["ascii-output"] != nil { throw CLIError(.usage, "outputModeConflict") } + if (options["division-scale"] == nil) != (options["rounding"] == nil) { throw CLIError(.usage, "divisionPolicyPairRequired") } + } if positionals.filter({ $0 == "-" }).count > 1 { throw CLIError(.usage, "repeatedStdin") } } package var overwrite: Bool { options["overwrite"] != nil } diff --git a/Sources/JSONCLIEngine/CLIDiagnostics.swift b/Sources/JSONCLIEngine/CLIDiagnostics.swift index c770f2b..ca06549 100644 --- a/Sources/JSONCLIEngine/CLIDiagnostics.swift +++ b/Sources/JSONCLIEngine/CLIDiagnostics.swift @@ -22,6 +22,8 @@ package struct CLIDiagnostics { } package static func classify(_ error: any Error) -> CLIError { if let error = error as? CLIError { return error } + if let failure = error as? FilterCompileFailure { return .init(.compile, failure.code) } + if let failure = error as? FilterLanguageFailure { return .init(.language, failure.code) } if error is CancellationError { return .init(.cancelled, "cancelled") } if let error = error as? JSONFailure { let category: CLIError.Category = error.category == .resource ? .resource : (error.category == .configuration ? .usage : .input) diff --git a/Sources/JSONCLIEngine/CLIEngine.swift b/Sources/JSONCLIEngine/CLIEngine.swift index 3f5e7fb..ae188cb 100644 --- a/Sources/JSONCLIEngine/CLIEngine.swift +++ b/Sources/JSONCLIEngine/CLIEngine.swift @@ -81,10 +81,11 @@ package enum CLIEngine { } if arguments.command == "--version" { guard arguments.positionals.isEmpty || arguments.positionals == ["--json"] else { throw CLIError(.usage, "invalidVersionOption") } - let version = arguments.positionals.isEmpty ? "swift-json 1.4.0-dev\n" : "{\"version\":\"1.4.0-dev\",\"revision\":\"unknown\",\"features\":[\"format\",\"validate\",\"pointer\",\"patch\",\"render\"]}\n" + let version = arguments.positionals.isEmpty ? "swift-json 1.4.0-dev\n" : "{\"version\":\"1.4.0-dev\",\"revision\":\"unknown\",\"features\":[\"format\",\"validate\",\"pointer\",\"patch\",\"render\",\"query\"],\"dialect\":\"swift-json-filter/1\"}\n" return try text(version, arguments: arguments, context: &context) } - if arguments.command == "query" || arguments.command == "stream" || arguments.command == "schema" { throw CLIError(.usage, "capabilityUnavailable") } + if arguments.command == "query" { return try CLIQueryCommand.run(arguments, context: &context) } + if arguments.command == "stream" || arguments.command == "schema" { throw CLIError(.usage, "capabilityUnavailable") } let inputPath: String let pointerText: String? if arguments.command == "pointer" { @@ -212,6 +213,7 @@ package enum CLIEngine { pointer Select an RFC 6901 Pointer; null and absence remain distinct. patch Apply a staged RFC 6902 patch supplied with --patch PATH. render Produce qualified static HTML, tab-indented text or Markdown. + query FILTER Pull exact filter results with strict file/stdin completion. INPUT defaults to stdin (-). Root JSON is strict UTF-8; duplicate keys fail. --limits conservative|balanced|offline finite resource profile @@ -219,7 +221,8 @@ package enum CLIEngine { --output PATH [--overwrite] atomic file installation after completion --diagnostics human|json --report none|json stderr only No online references, shell execution, server or remote providers. - Query/stream/schema adapters are not yet advertised by this foundation build. + Query core: paths, generators, conditions, bindings, map, presence and exact comparison. + Arithmetic, richer filters and provider/stream/schema adapters have separate gates. """ } diff --git a/Sources/JSONCLIEngine/CLIInputCursor.swift b/Sources/JSONCLIEngine/CLIInputCursor.swift new file mode 100644 index 0000000..be1da45 --- /dev/null +++ b/Sources/JSONCLIEngine/CLIInputCursor.swift @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import JSON +import JSONFilterRuntime + +package enum CLIInputFraming: String, Sendable { case values, document, ndjson, array, rawLines, rawText } +/// One lazy invocation cursor serves both the command driver and input/inputs. +/// It feeds the native grammar, never a permissive second JSON parser. +package struct CLIInputCursor: FilterInput { + private let files: [String] + private let framing: CLIInputFraming + private var fileIndex = 0 + private var source: CLIInput? + private var buffer: [UInt8] = [] + private var index = 0 + private var eof = false + private var fileRecords = 0 + private var machine: JSONTokenMachine + private var assembler = JSONValueAssembler() + private var ready: JSON? + private var recordSawData = false + private var recordStarted = false + private var recordEntry: UInt64 = 0 + private var entryBuffer: UInt64 = 0 + private var entryOuter: UInt64 = 0 + private var outerBytes: UInt64 = 0 + private var outerStarted = false + private var outerClosed = false + private var failed = false + package init(files: [String], framing: CLIInputFraming = .values) { + self.files = files.isEmpty ? ["-"] : files; self.framing = framing + machine = .init(topLevelArrayStreaming: framing == .array) + } + private static func whitespace(_ byte: UInt8) -> Bool { [9,10,13,32].contains(byte) } + private mutating func peek(context: inout CLIJobContext) throws -> UInt8? { + try context.checkpoint() + if index < buffer.count { return buffer[index] } + if !buffer.isEmpty { try context.meter.release(UInt64(buffer.count)); buffer = []; index = 0 } + if eof { return nil } + if source == nil { + guard fileIndex < files.count else { return nil } + try context.work(UInt64(files[fileIndex].utf8.count) + 1) + source = try CLIInput(path: files[fileIndex]) + } + if let chunk = try source!.next(context: &context) { buffer = chunk; index = 0; return buffer[0] } + eof = true; return nil + } + private mutating func beginRecord(context: inout CLIJobContext) throws { + try context.domain(.streaming); try context.meter.resetValueBounds() + recordStarted = true; recordEntry = context.meter.used(.retainedBytes) + entryBuffer = UInt64(buffer.count); entryOuter = outerBytes + } + private func ownedRecordBytes(context: borrowing CLIJobContext) throws -> UInt64 { + let end = context.meter.used(.retainedBytes) + entryBuffer + entryOuter + let start = recordEntry + UInt64(buffer.count) + outerBytes + guard end >= start else { throw CLIError(.usage, "invalidCursorOwnership") } + return end - start + } + private mutating func finishFile() { + source = nil; fileIndex += 1; eof = false; fileRecords = 0 + machine = .init(topLevelArrayStreaming: framing == .array); assembler = .init() + outerStarted = false; outerClosed = false; outerBytes = 0 + } + private mutating func yield(_ value: JSON, context: inout CLIJobContext) throws -> FilterYield { + let bytes = try ownedRecordBytes(context: context) + try context.charge(.totalRecords); try context.meter.charge(.totalRecords) + let lease = try CLIValueLease(ownedRetained: bytes, context: &context) + recordStarted = false; recordSawData = false; ready = nil; fileRecords += 1 + return .init(value: value, lease: lease) + } + private mutating func consume(_ event: JSONMachineEvent, context: inout CLIJobContext) throws { + if framing == .array { + if !outerStarted { + guard case .startArray = event.kind else { throw JSONFailure(.syntax, code: .invalidSyntax) } + outerStarted = true; outerBytes = JSONLogicalWeights.frame; return + } + if machine.hasCompleteRoot { + guard case .endArray = event.kind else { throw JSONFailure(.syntax, code: .invalidSyntax) } + outerClosed = true; outerBytes = 0; return + } + } + if let value = try assembler.consume(event, meter: &context.meter) { ready = value } + } + package mutating func next(context: inout CLIJobContext) throws -> FilterYield? { + guard !failed else { throw CLIError(.input, "cursorTerminal") } + do { + switch framing { + case .rawLines, .rawText, .ndjson: return try line(context: &context) + case .values, .document, .array: return try json(context: &context) + } + } catch { failed = true; context.latch(error); throw error } + } + private mutating func json(context: inout CLIJobContext) throws -> FilterYield? { + while fileIndex < files.count { + if !recordStarted { try beginRecord(context: &context) } + if let value = ready { + if framing == .array { + // This element is provisional. Closing outer syntax and EOF are + // still mandatory when next()/drain() resumes this cursor. + return try yield(value, context: &context) + } + if framing == .values { + if let byte = try peek(context: &context) { + guard Self.whitespace(byte) else { throw JSONFailure(.syntax, code: .trailingData, location: .init(byteOffset: machine.offset)) } + _ = try machine.next(in: buffer, index: &index, through: index + 1, meter: &context.meter) + let offset = machine.offset + let result = try yield(value, context: &context) + machine = .init(offset: offset); assembler = .init(); return result + } + _ = try machine.finish(meter: &context.meter) + let result = try yield(value, context: &context) + finishFile(); return result + } + } + if let byte = try peek(context: &context) { + if !Self.whitespace(byte) { recordSawData = true } + if let event = try machine.next(in: buffer, index: &index, through: index + 1, meter: &context.meter) { + try consume(event, context: &context) + } + continue + } + if !recordSawData && ready == nil && framing == .values { + recordStarted = false; finishFile(); continue + } + if let event = try machine.finish(meter: &context.meter) { try consume(event, context: &context); continue } + if framing == .array { + guard outerStarted && outerClosed else { throw JSONFailure(.syntax, code: .unexpectedEOF) } + recordStarted = false; finishFile(); continue + } + guard let value = ready else { throw JSONFailure(.syntax, code: .unexpectedEOF) } + let result = try yield(value, context: &context) + finishFile(); return result + } + return nil + } + private mutating func line(context: inout CLIJobContext) throws -> FilterYield? { + while fileIndex < files.count { + if !recordStarted { try beginRecord(context: &context) } + var bytes: [UInt8] = [], sawLF = false + while let byte = try peek(context: &context) { + try context.work(); try context.meter.require(.inputBytes, UInt64(bytes.count) + 1) + try context.meter.retain(1); bytes.append(byte); index += 1 + if framing != .rawText && byte == 10 { sawLF = true; break } + } + if bytes.isEmpty && framing != .rawText { + if framing == .ndjson && fileRecords == 0 { throw JSONFailure(.syntax, code: .unexpectedEOF) } + recordStarted = false; finishFile(); continue + } + let value: JSON + if framing == .ndjson { + value = try JSONParser().parse(bytes, meter: &context.meter) + } else { + if framing == .rawLines && sawLF { + bytes.removeLast(); try context.meter.release(1) + if bytes.last == 13 { bytes.removeLast(); try context.meter.release(1) } + } + try context.work(UInt64(bytes.count)); try context.meter.retain(UInt64(bytes.count)) + guard let text = String(bytes: bytes, encoding: .utf8) else { throw JSONFailure(.syntax, code: .invalidUTF8) } + value = .string(text) + try JSONAdmission.value(value, meter: &context.meter, owned: true) + try context.meter.release(UInt64(bytes.count)) + } + let owned = try ownedRecordBytes(context: context) + guard owned >= UInt64(bytes.count) else { throw CLIError(.usage, "invalidLineOwnership") } + let graph = owned - UInt64(bytes.count) + try context.meter.release(UInt64(bytes.count)) + // The line source reservation has transferred; close() must not refund + // its discarded byte buffer if the final record quota fails. + recordEntry = context.meter.used(.retainedBytes) - graph + entryBuffer = UInt64(buffer.count); entryOuter = 0 + try context.charge(.totalRecords); try context.meter.charge(.totalRecords) + let lease = try CLIValueLease(ownedRetained: graph, context: &context) + recordStarted = false; fileRecords += 1 + if !sawLF || framing == .rawText { finishFile() } + return .init(value: value, lease: lease) + } + return nil + } + package mutating func drain(context: inout CLIJobContext) throws { + while var value = try next(context: &context) { try value.release(context: &context) } + } + package mutating func close(context: inout CLIJobContext) throws { + if recordStarted { + let bytes = try ownedRecordBytes(context: context) + try context.meter.release(bytes) + } + if !buffer.isEmpty { try context.meter.release(UInt64(buffer.count)) } + if outerBytes > 0 { try context.meter.release(outerBytes) } + buffer = []; source = nil; ready = nil; assembler = .init(); machine = .init() + outerBytes = 0; recordStarted = false; failed = true + try context.reclaimValueLeases() + } +} diff --git a/Sources/JSONCLIEngine/CLIQueryCommand.swift b/Sources/JSONCLIEngine/CLIQueryCommand.swift new file mode 100644 index 0000000..d6f7949 --- /dev/null +++ b/Sources/JSONCLIEngine/CLIQueryCommand.swift @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import JSON +import JSONFilterRuntime + +package enum CLIQueryCommand { + package static func run(_ arguments: CLIArguments, context: inout CLIJobContext) throws -> Int32 { + let source: String, files: [String] + if let path = arguments.options["filter-file"] { + source = try textFile(path, maximum: context.limits[.querySourceBytes], context: &context) + files = arguments.positionals + } else { + guard let first = arguments.positionals.first else { throw CLIError(.usage, "missingFilter") } + source = first; files = Array(arguments.positionals.dropFirst()) + } + guard arguments.options["division-scale"] == nil else { throw CLIError(.usage, "arithmeticNotYetQualified") } + let inputFiles = files.isEmpty ? ["-"] : files + let auxiliary = arguments.repeated.enumerated().compactMap { index, pair -> String? in + guard pair.0 == "bindingValue", index > 0, ["rawfile","slurpfile"].contains(arguments.repeated[index-1].0) else { return nil } + return pair.1 + } + [arguments.options["filter-file"], arguments.options["limits-file"]].compactMap { $0 } + guard (inputFiles + auxiliary).filter({ $0 == "-" }).count <= 1 else { throw CLIError(.usage, "repeatedStdin") } + let bindingEntry = context.used(.bindings) + var bindings: [String: FilterYield] = [:] + defer { + bindings = [:] + try? context.release(.bindings, context.used(.bindings) - bindingEntry) + try? context.reclaimValueLeases() + } + bindings = try makeBindings(arguments, context: &context) + let program = try FilterCompiler.compile(source, variables: Set(bindings.keys), context: &context) + try context.domain(.core) + let output = try CLIOutput(destination: arguments.options["output"], overwrite: arguments.overwrite, inputs: inputFiles + auxiliary, context: &context) + let framing: CLIInputFraming + if arguments.options["raw-input"] != nil { framing = arguments.options["slurp"] == nil ? .rawLines : .rawText } + else { + guard let selected = CLIInputFraming(rawValue: arguments.options["input-framing"] ?? "values"), [.values,.document,.ndjson].contains(selected) else { throw CLIError(.usage, "invalidInputFraming") } + framing = selected + } + var cursor = CLIInputCursor(files: inputFiles, framing: framing) + defer { try? cursor.close(context: &context) } + var finalTruth: Bool? + func evaluate(_ input: FilterYield, cursor: inout CLIInputCursor, context: inout CLIJobContext) throws { + try context.domain(.core) + var vm = try FilterVM(program: program, input: input, bindings: bindings, context: &context) + defer { try? vm.close(context: &context) } + while var value = try vm.next(input: &cursor, context: &context) { + finalTruth = FilterTruth.test(value.value) + try CLIQueryOutput.emit(value.value, arguments: arguments, output: output, context: &context) + try value.release(context: &context) + } + } + if arguments.options["null-input"] != nil { + try evaluate(.init(value: .null), cursor: &cursor, context: &context) + } else if arguments.options["slurp"] != nil { + var collected: [FilterYield] = [], count: UInt64 = 0, scratch: UInt64 = 0 + defer { try? context.release(.collectionValues, count); try? context.meter.release(scratch) } + while let value = try cursor.next(context: &context) { + try context.charge(.collectionValues); count += 1; try context.meter.retain(24); scratch += 24; collected.append(value) + } + let input: FilterYield + if framing == .rawText { + var text = "", bytes: UInt64 = 0 + defer { try? context.meter.release(bytes) } + for value in collected { + guard case .string(let part) = value.value else { throw CLIError(.input, "invalidRawSlurp") } + for scalar in part.unicodeScalars { + let amount = UInt64(scalar.utf8.count) + try context.work(amount); try context.meter.require(.stringBytes, bytes + amount); try context.meter.retain(amount) + text.unicodeScalars.append(scalar); bytes += amount + } + } + input = try FilterValues.produced(.string(text), context: &context) + } else { + try context.meter.retain(count * 16); scratch += count * 16; try context.work(count) + input = try FilterValues.array(collected.map(\.value), admittedCollectionCount: true, context: &context) + try context.meter.release(scratch); scratch = 0 + // The finished array lease adopts the collector's logical slots. + count = 0 + } + collected = [] + try evaluate(input, cursor: &cursor, context: &context) + } else { + while var value = try cursor.next(context: &context) { + try evaluate(value, cursor: &cursor, context: &context) + try value.release(context: &context) + } + } + // first()/short-circuiting stops a filter, never the enclosing file's syntax. + try cursor.drain(context: &context) + try output.finish(inputs: inputFiles + auxiliary, context: &context) + bindings = [:]; try context.reclaimValueLeases() + if arguments.options["exit-status"] != nil { return finalTruth.map { $0 ? 0 : 1 } ?? 4 } + return 0 + } + private static func validName(_ text: String) -> Bool { + let bytes = Array(text.utf8) + guard let first = bytes.first, first == 95 || (65...90).contains(first) || (97...122).contains(first), text != "ARGS" else { return false } + return bytes.allSatisfy { $0 == 95 || (48...57).contains($0) || (65...90).contains($0) || (97...122).contains($0) } + } + private static func makeBindings(_ arguments: CLIArguments, context: inout CLIJobContext) throws -> [String: FilterYield] { + try context.checkpoint() + let entry = context.meter.used(.retainedBytes) + var owned: UInt64 = 0 + defer { + let total = context.meter.used(.retainedBytes) + if total >= entry + owned { try? context.meter.release(total - entry - owned) } + else { context.latch(CLIError(.usage, "invalidBindingOwnership")) } + } + var bindings: [String: FilterYield] = [:], named: [JSONMember] = [], index = 0 + while index < arguments.repeated.count { + let (mode, name) = arguments.repeated[index]; index += 1 + guard validName(name), bindings[name] == nil, index < arguments.repeated.count, arguments.repeated[index].0 == "bindingValue" else { throw CLIError(.usage, "invalidBinding") } + let source = arguments.repeated[index].1; index += 1 + try context.charge(.bindings); try context.work(UInt64(name.utf8.count) + 1) + let value: FilterYield + switch mode { + case "arg": value = try FilterValues.produced(.string(source), context: &context) + case "argjson": + try context.meter.resetValueBounds() + let entry = context.meter.used(.retainedBytes) + let json = try JSONParser().parse(Array(source.utf8), meter: &context.meter) + value = .init(value: json, lease: try CLIValueLease(ownedRetained: context.meter.used(.retainedBytes) - entry, context: &context)) + case "rawfile": value = try FilterValues.produced(.string(textFile(source, maximum: context.meter.limits[.stringBytes], context: &context)), context: &context) + case "slurpfile": + var cursor = CLIInputCursor(files: [source]), values: [FilterYield] = [], slots: UInt64 = 0, scratch: UInt64 = 0 + defer { + try? cursor.close(context: &context) + try? context.release(.collectionValues, slots) + try? context.meter.release(scratch) + } + while let value = try cursor.next(context: &context) { try context.charge(.collectionValues); slots += 1; try context.meter.retain(24); scratch += 24; values.append(value) } + try context.meter.retain(slots * 16); scratch += slots * 16; try context.work(slots) + value = try FilterValues.array(values.map(\.value), admittedCollectionCount: true, context: &context) + try context.meter.release(scratch); scratch = 0; slots = 0 + default: throw CLIError(.usage, "invalidBinding") + } + bindings[name] = value; owned += value.lease?.retained ?? 0 + try context.meter.retain(UInt64(name.utf8.count) + 32) + try context.meter.retain(24); named.append(.init(try JSONKey(name, meter: &context.meter), value: value.value)) + } + var positional: [JSON] = [] + if let captured = arguments.capturedArguments { + for argument in captured { + try context.charge(.collectionValues); try context.meter.retain(16); try context.work(UInt64(argument.utf8.count) + 1) + if arguments.captureJSONArguments { try context.meter.resetValueBounds(); positional.append(try JSONParser().parse(Array(argument.utf8), meter: &context.meter)) } + else { positional.append(.string(argument)) } + } + } + let namedObject = try JSONObject.indexed(admitted: named, meter: &context.meter) + let args = try CLIEngine.object([("named", .object(namedObject)), ("positional", .array(positional))], context: &context) + try context.charge(.bindings) + bindings["ARGS"] = try FilterValues.produced(args, collections: UInt64(positional.count), context: &context) + owned += bindings["ARGS"]?.lease?.retained ?? 0 + return bindings + } + private static func textFile(_ path: String, maximum: UInt64, context: inout CLIJobContext) throws -> String { + let source = try CLIInput(path: path) + var bytes: [UInt8] = [] + defer { try? context.meter.release(UInt64(bytes.count)) } + while let chunk = try source.next(context: &context) { + guard UInt64(bytes.count) + UInt64(chunk.count) <= maximum else { throw CLIError(.resource, "limitExceeded", metric: "textSourceBytes") } + try context.work(UInt64(chunk.count)); try context.meter.retain(UInt64(chunk.count)) + bytes.append(contentsOf: chunk); try context.meter.release(UInt64(chunk.count)) + } + try context.work(UInt64(bytes.count)); try context.meter.retain(UInt64(bytes.count)) + guard let text = String(bytes: bytes, encoding: .utf8) else { throw CLIError(.input, "invalidUTF8") } + return text + } +} diff --git a/Sources/JSONCLIEngine/CLIQueryOutput.swift b/Sources/JSONCLIEngine/CLIQueryOutput.swift new file mode 100644 index 0000000..85d7fa6 --- /dev/null +++ b/Sources/JSONCLIEngine/CLIQueryOutput.swift @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONFilterRuntime + +package enum CLIQueryOutput { + package static func emit(_ value: JSON, arguments: CLIArguments, output: CLIOutput, context: inout CLIJobContext) throws { + try context.domain(.writing); try context.meter.resetValueBounds() + let retainedAtEntry = context.meter.used(.retainedBytes) + defer { + let retained = context.meter.used(.retainedBytes) + if retained >= retainedAtEntry { try? context.meter.release(retained - retainedAtEntry) } + else { context.latch(CLIError(.usage, "invalidOutputOwnership")) } + } + let raw = arguments.options["raw-output"] != nil || arguments.options["raw-output0"] != nil || arguments.options["join-output"] != nil + var bytes: [UInt8] = [] + func append(_ byte: UInt8, context: inout CLIJobContext) throws { + try context.work(); try context.meter.charge(.outputBytes); try context.meter.retain(1) + bytes.append(byte) + } + if raw, case .string(let text) = value { + // Validate the entire string before handing off any bytes of this record. + if arguments.options["raw-output0"] != nil { + for byte in text.utf8 { try context.work(); if byte == 0 { throw CLIError(.language, "nulInRawOutput") } } + } + for byte in text.utf8 { try append(byte, context: &context) } + } else { + try JSONAdmission.value(value, meter: &context.meter) + var cursor = JSONWritingCursor(value: value, options: arguments.writingOptions(limits: context.meter.limits)) + let ascii = arguments.options["ascii-output"] != nil + let hex: [UInt8] = Array("0123456789abcdef".utf8) + func escape(_ unit: UInt32, context: inout CLIJobContext) throws { + try append(92, context: &context); try append(117, context: &context) + for shift in [12,8,4,0] { try append(hex[Int((unit >> shift) & 15)], context: &context) } + } + while let byte = try cursor.next(meter: &context.meter) { + if !ascii || byte < 128 { try append(byte, context: &context); continue } + let count: Int, initial: UInt32 + if byte < 224 { count = 2; initial = UInt32(byte & 31) } + else if byte < 240 { count = 3; initial = UInt32(byte & 15) } + else { count = 4; initial = UInt32(byte & 7) } + var scalar = initial + for _ in 1..> 10), context: &context) + try escape(56320 + (remainder & 1023), context: &context) + } + } + } + if arguments.options["join-output"] == nil { try append(arguments.options["raw-output0"] == nil ? 10 : 0, context: &context) } + try context.charge(.totalResults) + try output.write(bytes, context: &context); output.resultAccepted(context: &context) + } +} diff --git a/Sources/JSONFilterRuntime/CLIJobContext.swift b/Sources/JSONFilterRuntime/CLIJobContext.swift index 35a9c9f..c274ad0 100644 --- a/Sources/JSONFilterRuntime/CLIJobContext.swift +++ b/Sources/JSONFilterRuntime/CLIJobContext.swift @@ -86,6 +86,7 @@ package struct CLIError: Error, Sendable, Equatable { /// Mutable runtime state belongs to the invocation; immutable plans may be shared. package struct CLIJobContext: ~Copyable { package var meter: JSONWorkMeter + package let leaseRefunds: CLILeaseRefunds package private(set) var limits: CLILimits private var counters = Array(repeating: UInt64(0), count: CLIResourceMetric.allCases.count) private var failure: (any Error)? @@ -98,8 +99,19 @@ package struct CLIJobContext: ~Copyable { try limits.validate() meter = try .init(limits: limits.effectiveCore, domain: .parsing, trackAcceptedCapacities: true) self.limits = limits + leaseRefunds = CLILeaseRefunds() + } + /// Refund dead value storage even when data operations are terminal. Work is + /// never refunded; the single native ledger remains authoritative. + package mutating func reclaimValueLeases() throws { + let bytes = leaseRefunds.retained, values = leaseRefunds.collections + leaseRefunds.retained = 0; leaseRefunds.collections = 0 + if bytes > 0 { try meter.release(bytes) } + if values > 0 { try release(.collectionValues, values) } + if let error = leaseRefunds.failure { latch(error); throw error } } package mutating func checkpoint() throws { + try reclaimValueLeases() if let failure { throw failure } do { try meter.checkpoint() } catch { latch(error); throw error } diff --git a/Sources/JSONFilterRuntime/CLIValueLease.swift b/Sources/JSONFilterRuntime/CLIValueLease.swift new file mode 100644 index 0000000..4a1d0fb --- /dev/null +++ b/Sources/JSONFilterRuntime/CLIValueLease.swift @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON + +/// Invocation-only ARC notifications. The job drains refunds before its next +/// checkpoint; destructors cannot replace a meter or reopen terminal health. +package final class CLILeaseRefunds { + package var retained: UInt64 = 0 + package var collections: UInt64 = 0 + package var failure: (any Error)? + package func enqueue(retained bytes: UInt64, collections values: UInt64) { + let (nextBytes, byteOverflow) = retained.addingReportingOverflow(bytes) + let (nextValues, valueOverflow) = collections.addingReportingOverflow(values) + if byteOverflow || valueOverflow { + failure = CLIError(.resource, "counterOverflow", metric: "valueLeaseRefund") + } else { retained = nextBytes; collections = nextValues } + } +} +/// A lease holds reservations for one admitted immutable value graph. Selectors +/// share its lifetime; newly constructed graphs undergo full owned admission. It is deliberately not Sendable; only plans are shared. +package final class CLIValueLease { + private let refunds: CLILeaseRefunds + package let retained: UInt64 + private let collections: UInt64 + package init(ownedRetained: UInt64, collections: UInt64 = 0, + context: inout CLIJobContext) throws { + try context.checkpoint() + let metadata = UInt64(48) + try context.meter.retain(metadata) + refunds = context.leaseRefunds + retained = ownedRetained + metadata + self.collections = collections + } + deinit { refunds.enqueue(retained: retained, collections: collections) } +} +/// Immutable JSON payload plus an invocation-owned lifetime. Copies share the same +/// reservation rather than refunding a source still borrowed by another result. +package struct FilterYield { + package var value: JSON + package var lease: CLIValueLease? + package var collectedValues: UInt64 + package init(value: JSON, lease: CLIValueLease? = nil, collectedValues: UInt64 = 0) { self.value = value; self.lease = lease; self.collectedValues = collectedValues } + package func selecting(_ value: JSON) -> Self { + let retainedCollections: UInt64 + switch value { case .array, .object: retainedCollections = collectedValues; default: retainedCollections = 0 } + return .init(value: value, lease: lease, collectedValues: retainedCollections) + } + package mutating func release(context: inout CLIJobContext) throws { + value = .null; lease = nil; collectedValues = 0 + try context.reclaimValueLeases() + } +} diff --git a/Sources/JSONFilterRuntime/FilterCompiler.swift b/Sources/JSONFilterRuntime/FilterCompiler.swift new file mode 100644 index 0000000..df0331f --- /dev/null +++ b/Sources/JSONFilterRuntime/FilterCompiler.swift @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON + +/// Token-range compilation uses an explicit work stack and flat node IDs. +/// Source depth never becomes Swift call-stack depth. +package enum FilterCompiler { + private struct Work { + let range: Range + let destination: Int + let depth: UInt64 + let variables: Set + } + package static let builtinArities: [String: Set] = [ + "select":[1], "map":[1], "map_values":[1], "has":[1], "length":[0], "utf8bytelength":[0], "type":[0], + "keys":[0], "keys_unsorted":[0], "values":[0], "empty":[0], "not":[0], + "arrays":[0], "objects":[0], "iterables":[0], "booleans":[0], "numbers":[0], "normals":[0], "finites":[0], "strings":[0], "nulls":[0], "scalars":[0], + "error":[0,1], "input":[0], "inputs":[0] + ] + package static func compile(_ source: String, variables: Set = ["ARGS"], context: inout CLIJobContext) throws -> FilterProgram { + let tokens = try FilterLexer.lex(source, context: &context) + guard !tokens.isEmpty else { throw FilterCompileFailure("emptyFilter", at: 0) } + var matches: [Int:Int] = [:], nesting: [(Int,String)] = [] + for i in tokens.indices { + try context.work() + guard let text = tokens[i].text else { continue } + if ["(","[","{","if"].contains(text) { + try context.require(.queryDepth, UInt64(nesting.count)+1); try context.meter.retain(32) + nesting.append((i,text)) + } else if [")","]","}","end"].contains(text) { + guard let open = nesting.popLast(), ["(":")","[":"]","{":"}","if":"end"][open.1] == text else { throw FilterCompileFailure("unmatchedDelimiter", at: tokens[i].range.lowerBound) } + try context.work(); try context.meter.retain(32); try context.meter.release(32) + matches[open.0] = i; matches[i] = open.0 + } + } + guard nesting.isEmpty else { throw FilterCompileFailure("unclosedDelimiter", at: tokens[nesting.last!.0].range.lowerBound) } + var nodes: [FilterProgram.Node] = [], ranges: [Range] = [], work: [Work] = [] + func slot(_ span: Range, depth: UInt64, variables: Set, context: inout CLIJobContext) throws -> Int { + guard !span.isEmpty else { throw FilterCompileFailure("missingExpression", at: source.utf8.count) } + try context.require(.queryDepth, depth); try context.charge(.astNodes); try context.meter.retain(128) + let id = nodes.count; nodes.append(.pending); ranges.append(tokens[span.lowerBound].range.lowerBound.., context: inout CLIJobContext) throws -> [Int] { + var result: [Int] = [], i = span.lowerBound + while i < span.upperBound { + try context.work(); try context.meter.retain(8); taskScratch += 8; result.append(i) + if let close = matches[i], close > i { i = close + 1 } else { i += 1 } + } + return result + } + let root = try slot(0.., context: inout CLIJobContext) throws -> Int { try slot(range,depth:task.depth+1,variables:task.variables,context:&context) } + let positions = try top(span, context:&context) + if (first == "if" || first == "elif") && tokens[hi-1].text == "end" { + guard tokens[hi-1].text == "end" else { throw FilterCompileFailure("missingEnd", at: tokens[lo].range.lowerBound) } + let inner = try top((lo+1)..<(hi-1),context:&context) + guard let then = inner.first(where:{tokens[$0].text == "then"}), let branch = inner.first(where:{tokens[$0].text == "else" || tokens[$0].text == "elif"}), then < branch else { throw FilterCompileFailure("invalidConditional", at: tokens[lo].range.lowerBound) } + let condition = try child((lo+1)..","<=",">="], ["+","-"], ["*","/","%"]] + for (level, operators) in precedences.enumerated() { + var found: [Int] = [] + for position in positions where position > lo && position+1 < hi { + try context.work() + if let name = tokens[position].text, operators.contains(name) { + let previous = tokens[position-1].text + // Unary minus is not an infix split after an operator or delimiter. + if name == "-", previous.map({ ["|",",","//","+","-","*","/","%","=","(","[",":",";"].contains($0) }) == true { continue } + try context.meter.retain(8); taskScratch += 8; found.append(position) + } + } + if found.isEmpty { continue } + if level == 7 || level == 8 { throw FilterCompileFailure("arithmeticNotYetQualified", at: tokens[found[0]].range.lowerBound) } + if level == 3 { throw FilterCompileFailure("assignmentNotYetQualified", at: tokens[found[0]].range.lowerBound) } + if level == 6 && found.count > 1 { throw FilterCompileFailure("chainedComparison", at: tokens[found[1]].range.lowerBound) } + let position = level == 0 || level == 2 ? found.first! : found.last!, op = tokens[position].text! + if op == "|", let binding = positions.first(where: { $0 < position && tokens[$0].text == "as" }) { + guard binding+2 == position, case .variable(let name) = tokens[binding+1].kind else { throw FilterCompileFailure("invalidBinding", at: tokens[binding].range.lowerBound) } + let lhs = try child(lo.. lo { + let base = try child(lo.. 2 && tokens[lo+1].text == "(" && matches[lo+1] == hi-1 { + guard let arities = builtinArities[name] else { throw FilterCompileFailure("unsupportedFunction",at:tokens[lo].range.lowerBound) } + var arguments: [Int] = [] + if span.count > 2 { + let inside=(lo+2)..<(hi-1) + try context.meter.retain(UInt64(inside.count) * 16); taskScratch += UInt64(inside.count) * 16 + let semis=try top(inside,context:&context).filter{tokens[$0].text == ";"} + if !inside.isEmpty { for (start,end) in zip([inside.lowerBound]+semis.map{$0+1},semis+[inside.upperBound]) { arguments.append(try child(start.. [FilterToken] { + let count = UInt64(source.utf8.count) + try context.charge(.querySourceBytes, count); try context.meter.retain(count); try context.work(count) + let bytes = Array(source.utf8) + var tokens: [FilterToken] = [], i = 0 + func append(_ kind: FilterToken.Kind, _ start: Int, _ end: Int, context: inout CLIJobContext) throws { + try context.charge(.queryTokens); try context.meter.retain(48) + tokens.append(.init(kind: kind, range: start.. Bool { byte == 95 || (65...90).contains(byte) || (97...122).contains(byte) } + func nameByte(_ byte: UInt8) -> Bool { nameStart(byte) || (48...57).contains(byte) } + while i < bytes.count { + try context.work(); let start = i, byte = bytes[i] + if [9,10,13,32].contains(byte) { i += 1; continue } + if byte == 35 { while i < bytes.count && bytes[i] != 10 { try context.work(); i += 1 }; continue } + if byte == 34 { + i += 1; var escaped = false, closed = false + while i < bytes.count { + try context.work(); let b = bytes[i]; i += 1 + if escaped { escaped = false; continue } + if b == 92 { escaped = true; continue } + if b == 34 { closed = true; break } + } + guard closed else { throw FilterCompileFailure("unterminatedString", at: start) } + let value: JSON + let copyBytes = UInt64(i-start) + try context.work(copyBytes); try context.meter.retain(copyBytes) + defer { try? context.meter.release(copyBytes) } + do { value = try JSONParser().parse(Array(bytes[start..=", "|=", "+=", "-=", "*=", "/=", "%="].contains(pair) { + i += 2; try append(.symbol(pair), start, i, context: &context); continue + } + } + guard [46,44,124,40,41,91,93,123,125,58,59,63,43,45,42,47,37,60,62,61,64].contains(byte) else { throw FilterCompileFailure("unexpectedByte", at: start) } + i += 1; try append(.symbol(String(Unicode.Scalar(byte))), start, i, context: &context) + } + return tokens + } +} diff --git a/Sources/JSONFilterRuntime/FilterPrimitiveBuiltins.swift b/Sources/JSONFilterRuntime/FilterPrimitiveBuiltins.swift new file mode 100644 index 0000000..8c17ab1 --- /dev/null +++ b/Sources/JSONFilterRuntime/FilterPrimitiveBuiltins.swift @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON + +package enum FilterPrimitiveBuiltins { + package static func apply(_ name:String,value:JSON,argument:JSON?=nil,context:inout CLIJobContext)throws->FilterYield? { + try context.work() + switch name { + case "empty":return nil + case "not":return try FilterValues.produced(.boolean(!FilterTruth.test(value)),context:&context) + case "values":if case .null=value{return nil};return .init(value:value) + case "has":guard let argument else{throw FilterLanguageFailure("wrongArity")};return try FilterValues.produced(.boolean(FilterValues.has(value,key:argument,context:&context)),context:&context) + case "type": + let text:String + switch value{case .null:text="null";case .boolean:text="boolean";case .number:text="number";case .string:text="string";case .array:text="array";case .object:text="object"} + return try FilterValues.produced(.string(text),context:&context) + case "length": + switch value { + case .null:return try FilterValues.integer(0,context:&context) + case .array(let a):return try FilterValues.integer(a.count,context:&context) + case .object(let o):return try FilterValues.integer(o.count,context:&context) + case .string(let s):var count=0;for _ in s.unicodeScalars{try context.work();count += 1};return try FilterValues.integer(count,context:&context) + case .number(let n):let c=n.components;let components=DecimalComponents(negative:false,digits:c.digits,exponent:c.exponent);return try FilterValues.produced(.number(JSONNumber(components:components)),context:&context) + case .boolean:throw FilterLanguageFailure("invalidLengthDomain") + } + case "utf8bytelength": + guard case .string(let text)=value else{throw FilterLanguageFailure("expectedString")} + var count=0;for _ in text.utf8{try context.work();count += 1};return try FilterValues.integer(count,context:&context) + case "keys","keys_unsorted": + let count:Int + switch value{case .object(let o):count=o.count;case .array(let a):count=a.count;default:throw FilterLanguageFailure("expectedContainer")} + try context.charge(.collectionValues,UInt64(count)) + var scratch:UInt64=0,transferred=false + defer { try? context.meter.release(scratch); if !transferred { try? context.release(.collectionValues,UInt64(count)) } } + try context.meter.retain(UInt64(count)*16);scratch=UInt64(count)*16 + var values:[JSON]=[] + switch value { + case .object(let o): + if name=="keys" { + let members=try FilterValues.sortedMembers(o,context:&context) + defer{try? context.meter.release(UInt64(count)*JSONLogicalWeights.member)} + for member in members{try context.work();values.append(.string(member.key.string))} + } else {for member in o{try context.work();values.append(.string(member.key.string))}} + case .array:for index in 0..] +} +package struct FilterToken: Sendable { + package enum Kind: Sendable { case symbol(String), name(String), variable(String), literal(JSON) } + package let kind: Kind + package let range: Range + package var text: String? { + switch kind { case .symbol(let s), .name(let s): s; default: nil } + } +} diff --git a/Sources/JSONFilterRuntime/FilterTruth.swift b/Sources/JSONFilterRuntime/FilterTruth.swift new file mode 100644 index 0000000..86866c0 --- /dev/null +++ b/Sources/JSONFilterRuntime/FilterTruth.swift @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +package enum FilterTruth { + package static func test(_ value: JSON) -> Bool { + switch value { case .null, .boolean(false): false; default: true } + } +} diff --git a/Sources/JSONFilterRuntime/FilterVM.swift b/Sources/JSONFilterRuntime/FilterVM.swift new file mode 100644 index 0000000..4bc87af --- /dev/null +++ b/Sources/JSONFilterRuntime/FilterVM.swift @@ -0,0 +1,557 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON + +package protocol FilterInput { + mutating func next(context: inout CLIJobContext) throws -> FilterYield? +} +/// Explicit, demand-driven continuations. The arena reuses slots; no language +/// recursion becomes Swift call-stack recursion. Only immutable plans are shared. +package struct FilterVM { + private enum Event { case value(FilterYield), end, failure(FilterLanguageFailure) } + private enum Action { case pull(Int), emit(Event) } + private struct ObjectChoice { let key: JSONKey; let value: FilterYield; let keySource: FilterYield } + private struct Frame { + let node: Int + let input: FilterYield + var bindings: [String: FilterYield] + var phase = 0 + var children: [Int] = [] + var event: Event? + var values: [FilterYield] = [] + var members: [(JSONKey, FilterYield)] = [] + var slots: UInt64 = 0 + var collectionSlots: UInt64 = 0 + var bindingSlots: UInt64 = 0 + var scopeBytes: UInt64 = 0 + var index = 0 + var current: FilterYield? + var flag = false + var groups: [[FilterYield]] = [] + var choices: [[ObjectChoice]] = [] + var selections: [Int] = [] + } + private let program: FilterProgram + private var frames: [Frame?] = [] + private var available: [Int] = [] + private var root = 0 + private var finished = false + package init(program: FilterProgram, input: FilterYield, bindings: [String: FilterYield] = [:], context: inout CLIJobContext) throws { + self.program = program + root = try create(program.root, input: input, bindings: bindings, context: &context) + } + private mutating func create(_ node: Int, input: FilterYield, bindings: [String: FilterYield], context: inout CLIJobContext) throws -> Int { + try context.checkpoint() + let entry = context.meter.used(.retainedBytes) + try context.charge(.runtimeFrames) + var complete = false + defer { + if !complete { + try? context.release(.runtimeFrames, 1) + try? context.meter.release(context.meter.used(.retainedBytes) - entry) + } + } + try context.work(4) // includes finite terminal cleanup + try context.meter.retain(256) + let frame = Frame(node: node, input: input, bindings: bindings) + if let id = available.popLast() { frames[id] = frame; complete = true; return id } + try context.meter.retain(64); frames.append(frame); complete = true; return frames.count - 1 + } + private mutating func dispose(_ id: Int, context: inout CLIJobContext) throws { + // Cleanup work and its bounded integer stack are prepaid at frame creation. + var pending = [id] + while let id = pending.popLast() { + guard let frame = frames[id] else { continue } + pending.append(contentsOf: frame.children) + try context.meter.release(256 + frame.slots + frame.scopeBytes) + try context.release(.runtimeFrames, 1) + try context.release(.collectionValues, frame.collectionSlots) + try context.release(.bindings, frame.bindingSlots) + frames[id] = nil; available.append(id) + } + try context.reclaimValueLeases() + } + package mutating func close(context: inout CLIJobContext) throws { + if !finished { try dispose(root, context: &context) } + try context.meter.release(UInt64(frames.count) * 64) + frames = []; available = []; finished = true + try context.reclaimValueLeases() + } + package mutating func next(input: inout Input, context: inout CLIJobContext) throws -> FilterYield? { + if finished { return nil } + var demands: [Int] = [], current = root + defer { try? context.meter.release(UInt64(demands.count) * 8) } + while true { + try context.work() + let action: Action + do { action = try step(current, input: &input, context: &context) } + catch let failure as FilterLanguageFailure { action = .emit(.failure(failure)) } + switch action { + case .pull(let child): + try context.meter.retain(8); demands.append(current); current = child + case .emit(let event): + if let parent = demands.popLast() { + try context.meter.release(8); frames[parent]!.event = event; current = parent + } else { + switch event { + case .value(let value): return value + case .end: try close(context: &context); return nil + case .failure(let failure): throw failure + } + } + } + } + } + private mutating func child(_ node: Int, frame: inout Frame, value: FilterYield? = nil, bindings: [String: FilterYield]? = nil, context: inout CLIJobContext) throws -> Int { + let id = try create(node, input: value ?? frame.input, bindings: bindings ?? frame.bindings, context: &context) + frame.children.append(id); return id + } + private func event(_ frame: inout Frame) -> Event? { let event = frame.event; frame.event = nil; return event } + private func capture(_ value: FilterYield, frame: inout Frame, context: inout CLIJobContext) throws { + try context.charge(.collectionValues); frame.collectionSlots += 1 + try context.meter.retain(24); frame.slots += 24 + frame.values.append(value) + } + private func releaseCache(_ frame: inout Frame, context: inout CLIJobContext, transferring: UInt64 = 0) throws { + frame.values = []; frame.members = [] + try context.meter.release(frame.slots); frame.slots = 0 + guard transferring <= frame.collectionSlots else { throw CLIError(.usage, "invalidCollectionTransfer") } + try context.release(.collectionValues, frame.collectionSlots - transferring); frame.collectionSlots = 0 + try context.reclaimValueLeases() + } + private mutating func step(_ id: Int, input: inout Input, context: inout CLIJobContext) throws -> Action { + var frame = frames[id]! + frames[id] = nil // Unique mutation of the frame's COW buffers. + defer { frames[id] = frame } + let incoming = event(&frame), node = program.nodes[frame.node] + if case .failure(let failure) = incoming { + switch node { + case .optional: frame.phase = 99; return .emit(.end) + case .attempt(_, let handler) where frame.phase == 1: + if let handler { + let value = try FilterValues.produced(.string(failure.message), context: &context) + for child in frame.children { try dispose(child, context: &context) }; frame.children = [] + frame.phase = 2; return .pull(try child(handler, frame: &frame, value: value, context: &context)) + } + frame.phase = 99; return .emit(.end) + default: return .emit(.failure(failure)) + } + } + if frame.phase == 99 { return .emit(.end) } + switch node { + case .pending: throw FilterCompileFailure("incompleteIR", at: 0) + case .identity, .literal, .variable, .recursive: + if case .recursive = node { + if frame.phase == 0 { try capture(frame.input, frame: &frame, context: &context); frame.phase = 1 } + guard let value = frame.values.popLast() else { return .emit(.end) } + frame.collectionSlots -= 1; frame.slots -= 24 + try context.release(.collectionValues, 1); try context.meter.release(24) + switch value.value { + case .array(let values): for child in values.reversed() { try capture(value.selecting(child), frame: &frame, context: &context) } + case .object(let object): for member in object.reversed() { try capture(value.selecting(member.value), frame: &frame, context: &context) } + default: break + } + return .emit(.value(value)) + } + guard frame.phase == 0 else { return .emit(.end) }; frame.phase = 1 + switch node { + case .identity: return .emit(.value(frame.input)) + case .literal(let value): return .emit(.value(.init(value: value))) + case .variable(let name): guard let value = frame.bindings[name] else { throw FilterLanguageFailure("unboundVariable") }; return .emit(.value(value)) + default: throw FilterLanguageFailure("invalidNode") + } + case .comma(let a, let b): + if frame.phase == 0 { frame.phase = 1; return .pull(try child(a, frame: &frame, context: &context)) } + if let incoming { + if case .end = incoming { + if frame.phase == 2 { return .emit(.end) } + try dispose(frame.children.removeLast(), context: &context); frame.phase = 2 + return .pull(try child(b, frame: &frame, context: &context)) + } + return .emit(incoming) + } + return .pull(frame.children.last!) + case .pipe(let a, let b), .bind(let a, _, let b): + if frame.phase == 0 { frame.phase = 1; return .pull(try child(a, frame: &frame, context: &context)) } + if let incoming { + if frame.phase == 1 { + switch incoming { + case .end: return .emit(.end) + case .value(let value): + frame.phase = 2 + if case .bind(_, let name, _) = node { + try context.charge(.bindings); frame.bindingSlots += 1 + var bytes = UInt64(frame.bindings.count + 1) * 32 + UInt64(name.utf8.count) + for key in frame.bindings.keys { try context.work(UInt64(key.utf8.count) + 1); bytes += UInt64(key.utf8.count) } + try context.meter.retain(bytes); frame.scopeBytes += bytes + var bindings = frame.bindings; bindings[name] = value + let target = try child(b, frame: &frame, bindings: bindings, context: &context) + frames[target]!.scopeBytes += bytes; frames[target]!.bindingSlots = 1 + frame.scopeBytes -= bytes; frame.bindingSlots -= 1 + return .pull(target) + } + return .pull(try child(b, frame: &frame, value: value, context: &context)) + case .failure(let failure): throw failure + } + } + if case .end = incoming { try dispose(frame.children.removeLast(), context: &context); frame.phase = 1; return .pull(frame.children[0]) } + return .emit(incoming) + } + return .pull(frame.children.last!) + case .array(let body): + if frame.phase == 0 { frame.phase = 1; return .pull(try child(body, frame: &frame, context: &context)) } + if let incoming { + switch incoming { + case .value(let value): try capture(value, frame: &frame, context: &context); return .pull(frame.children[0]) + case .end: + let count = UInt64(frame.values.count), bytes = count * 16 + try context.meter.retain(bytes); frame.slots += bytes; try context.work(count) + let values = frame.values.map(\.value) + let result = try FilterValues.array(values, admittedCollectionCount: true, context: &context) + try context.meter.release(bytes); frame.slots -= bytes + try releaseCache(&frame, context: &context, transferring: count) + frame.phase = 99; return .emit(.value(result)) + case .failure(let failure): throw failure + } + } + return .pull(frame.children[0]) + case .optional(let body), .attempt(let body, _): + if frame.phase == 0 { frame.phase = 1; return .pull(try child(body, frame: &frame, context: &context)) } + if let incoming { return .emit(incoming) } + return .pull(frame.children.last!) + case .fallback(let a, let b): + if frame.phase == 0 { frame.phase = 1; return .pull(try child(a, frame: &frame, context: &context)) } + if let incoming { + if frame.phase == 2 { return .emit(incoming) } + switch incoming { + case .value(let value): if FilterTruth.test(value.value) { frame.flag = true; return .emit(.value(value)) }; return .pull(frame.children[0]) + case .end: + if frame.flag { return .emit(.end) } + try dispose(frame.children.removeLast(), context: &context); frame.phase = 2 + return .pull(try child(b, frame: &frame, context: &context)) + case .failure(let failure): throw failure + } + } + return .pull(frame.children.last!) + case .conditional(let condition, let yes, let no): + if frame.phase == 0 { frame.phase = 1; return .pull(try child(condition, frame: &frame, context: &context)) } + if let incoming { + if frame.phase == 1 { + switch incoming { + case .end: return .emit(.end) + case .value(let value): frame.phase = 2; return .pull(try child(FilterTruth.test(value.value) ? yes : no, frame: &frame, context: &context)) + case .failure(let failure): throw failure + } + } + if case .end = incoming { try dispose(frame.children.removeLast(), context: &context); frame.phase = 1; return .pull(frame.children[0]) } + return .emit(incoming) + } + return .pull(frame.children.last!) + case .iterate(let base): + if frame.phase == 0 { frame.phase = 1; return .pull(try child(base, frame: &frame, context: &context)) } + if let incoming { + switch incoming { + case .end: return .emit(.end) + case .value(let value): frame.current = value; frame.index = 0; frame.phase = 2 + case .failure(let failure): throw failure + } + } + if frame.phase == 2 { + let source = frame.current!, count: Int + switch source.value { case .array(let a): count = a.count; case .object(let o): count = o.count; default: throw FilterLanguageFailure("cannotIterate") } + if frame.index < count { + let value: JSON + switch source.value { case .array(let a): value = a[frame.index]; case .object(let o): value = o.members[frame.index].value; default: throw FilterLanguageFailure("cannotIterate") } + frame.index += 1; return .emit(.value(source.selecting(value))) + } + frame.current = nil; frame.phase = 1 + } + return .pull(frame.children[0]) + case .unary(let op, let body): + if frame.phase == 0 { frame.phase = 1; return .pull(try child(body, frame: &frame, context: &context)) } + if let incoming { + switch incoming { + case .end: return .emit(.end) + case .value(let value): + guard op == "-", case .number(let n) = value.value else { throw FilterLanguageFailure("expectedNumber") } + let c = n.components, negative = !c.isZero && !c.negative + let result = JSON.number(JSONNumber(components: .init(negative: negative, digits: c.digits, exponent: c.exponent))) + return .emit(.value(try FilterValues.produced(result, context: &context))) + case .failure(let failure): throw failure + } + } + return .pull(frame.children[0]) + case .binary(let op, let a, let b) where op == "and" || op == "or": + if frame.phase == 0 { frame.phase = 1; return .pull(try child(a, frame: &frame, context: &context)) } + if let incoming { + if case .end = incoming { + if frame.phase == 1 { return .emit(.end) } + try dispose(frame.children.removeLast(), context: &context); frame.phase = 1; return .pull(frame.children[0]) + } + if case .value(let value) = incoming { + let truth = FilterTruth.test(value.value) + if frame.phase == 1 { + if op == "and" && !truth || op == "or" && truth { return .emit(.value(try FilterValues.produced(.boolean(truth), context: &context))) } + frame.phase = 2; return .pull(try child(b, frame: &frame, context: &context)) + } + return .emit(.value(try FilterValues.produced(.boolean(truth), context: &context))) + } + } + return .pull(frame.children.last!) + case .index(let a, let b), .binary(_, let a, let b): + if frame.phase == 0 { frame.phase = 1; return .pull(try child(b, frame: &frame, context: &context)) } + if let incoming { + if frame.phase == 1 { + switch incoming { + case .end: return .emit(.end) + case .value(let value): + frame.current = value; frame.index = 0 + if !frame.flag { frame.phase = 2; return .pull(try child(a, frame: &frame, context: &context)) } + frame.phase = 3 + case .failure(let failure): throw failure + } + } else if frame.phase == 2 { + switch incoming { + case .value(let value): try capture(value, frame: &frame, context: &context); return .pull(frame.children[1]) + case .end: try dispose(frame.children.removeLast(), context: &context); frame.phase = 3; frame.flag = true + case .failure(let failure): throw failure + } + } + } + if frame.phase == 3 { + if frame.index < frame.values.count { + let lhs = frame.values[frame.index], rhs = frame.current!; frame.index += 1 + if case .index = node { return .emit(.value(lhs.selecting(try FilterValues.index(lhs.value, key: rhs.value, context: &context)))) } + guard case .binary(let op, _, _) = node else { throw FilterLanguageFailure("invalidState") } + let result: Bool + if op == "==" || op == "!=" { let equal = try JSONComparison.equals(lhs.value, rhs.value, meter: &context.meter); result = op == "==" ? equal : !equal } + else { + let order = try FilterValues.compare(lhs.value, rhs.value, context: &context) + switch op { case "<": result = order < 0; case ">": result = order > 0; case "<=": result = order <= 0; case ">=": result = order >= 0; default: throw FilterCompileFailure("arithmeticNotYetQualified", at: program.sourceRanges[frame.node].lowerBound) } + } + return .emit(.value(try FilterValues.produced(.boolean(result), context: &context))) + } + frame.current = nil; frame.phase = 1 + } + return .pull(frame.children[0]) + case .call(let name, let arguments): return try call(name, arguments: arguments, frame: &frame, incoming: incoming, input: &input, context: &context) + case .slice(let base, let lower, let upper): + return try slice(base, lower: lower, upper: upper, frame: &frame, incoming: incoming, context: &context) + case .object(let fields): + return try object(fields, frame: &frame, incoming: incoming, context: &context) + } + } + private func captureGroup(_ value: FilterYield, group: Int, frame: inout Frame, context: inout CLIJobContext) throws { + try context.charge(.collectionValues); frame.collectionSlots += 1 + try context.meter.retain(24); frame.slots += 24 + frame.groups[group].append(value) + } + private func advance(_ selections: inout [Int], counts: [Int], context: inout CLIJobContext) throws -> Bool { + for offset in selections.indices.reversed() { + try context.work(); selections[offset] += 1 + if selections[offset] < counts[offset] { return true } + selections[offset] = 0 + } + return false + } + private mutating func slice(_ base: Int, lower: Int?, upper: Int?, frame: inout Frame, incoming: Event?, context: inout CLIJobContext) throws -> Action { + let nodes = [base, lower, upper] + if frame.phase == 0 { + try context.meter.retain(120); frame.slots += 120 + frame.groups = [[], [], []]; frame.selections = [0, 0, 0]; frame.phase = 1; frame.index = 0 + } + if let incoming { + switch incoming { + case .value(let value): try captureGroup(value, group: frame.index, frame: &frame, context: &context); return .pull(frame.children.last!) + case .end: try dispose(frame.children.removeLast(), context: &context); frame.index += 1 + case .failure(let failure): throw failure + } + } + if frame.phase == 1 { + while frame.index < 3 { + try context.work() + if let node = nodes[frame.index] { return .pull(try child(node, frame: &frame, context: &context)) } + try captureGroup(.init(value: .null), group: frame.index, frame: &frame, context: &context); frame.index += 1 + } + if frame.groups.contains(where: { $0.isEmpty }) { frame.phase = 99; return .emit(.end) } + frame.phase = 2 + } + if frame.flag { frame.phase = 99; return .emit(.end) } + let input = frame.groups[0][frame.selections[0]] + let result = try FilterValues.slice(input.value, start: lower == nil ? Optional.none : frame.groups[1][frame.selections[1]].value, + end: upper == nil ? Optional.none : frame.groups[2][frame.selections[2]].value, context: &context) + frame.flag = try !advance(&frame.selections, counts: frame.groups.map(\.count), context: &context) + return .emit(.value(result)) + } + private mutating func object(_ fields: [(Int, Int)], frame: inout Frame, incoming: Event?, context: inout CLIJobContext) throws -> Action { + if frame.phase == 0 { + try context.meter.retain(UInt64(fields.count) * 40 + 48); frame.slots += UInt64(fields.count) * 40 + 48 + frame.groups = [[], []]; frame.index = 0; frame.phase = 1 + } + if let incoming { + let group = frame.phase == 1 ? 0 : 1 + switch incoming { + case .value(let value): + if group == 0, case .string = value.value {} else if group == 0 { throw FilterLanguageFailure("objectKeyMustBeString") } + try captureGroup(value, group: group, frame: &frame, context: &context); return .pull(frame.children.last!) + case .end: + try dispose(frame.children.removeLast(), context: &context) + if frame.phase == 1 { + if frame.groups[0].isEmpty { frame.phase = 99; return .emit(.end) } + frame.phase = 2; return .pull(try child(fields[frame.index].1, frame: &frame, context: &context)) + } + if frame.groups[1].isEmpty { frame.phase = 99; return .emit(.end) } + var choices: [ObjectChoice] = [] + for key in frame.groups[0] { + guard case .string(let text) = key.value else { throw FilterLanguageFailure("objectKeyMustBeString") } + for value in frame.groups[1] { + try context.work(); try context.charge(.collectionValues); frame.collectionSlots += 1 + try context.meter.retain(64); frame.slots += 64 + choices.append(.init(key: try JSONKey(text, meter: &context.meter), value: value, keySource: key)) + } + } + // Temporary key/value result lists are no longer held. Choice slots + // keep the exact sources alive, and work is never refunded. + let records = UInt64(frame.groups[0].count + frame.groups[1].count) + try context.release(.collectionValues, records); frame.collectionSlots -= records + try context.meter.release(records * 24); frame.slots -= records * 24 + frame.groups = [[], []]; frame.choices.append(choices); frame.index += 1; frame.phase = 1 + case .failure(let failure): throw failure + } + } + if frame.phase == 1, frame.index < fields.count { return .pull(try child(fields[frame.index].0, frame: &frame, context: &context)) } + if frame.phase == 1 { + frame.selections = Array(repeating: 0, count: fields.count); frame.phase = 3 + } + if frame.flag { frame.phase = 99; return .emit(.end) } + try context.checkpoint() + let entry = context.meter.used(.retainedBytes) + var scratch: UInt64 = 0 + var building = true + defer { try? context.meter.release(building ? context.meter.used(.retainedBytes) - entry : scratch) } + var members: [JSONMember] = [], index = JSONExactKeyIndex() + for field in fields.indices { + try context.work(); let selected = frame.choices[field][frame.selections[field]] + if let position = try index.find(selected.key, meter: &context.meter) { + members[position] = JSONMember(members[position].key, value: selected.value.value) + } else { + try context.meter.require(.objectMembers, UInt64(members.count) + 1) + try context.meter.retain(24) + _ = try index.insert(selected.key, position: members.count, meter: &context.meter) + members.append(.init(selected.key, value: selected.value.value)) + } + } + scratch = context.meter.used(.retainedBytes) - entry + building = false + let result = try FilterValues.produced(.object(JSONObject(admitted: members, index: index)), context: &context) + frame.flag = try !advance(&frame.selections, counts: frame.choices.map(\.count), context: &context) + return .emit(.value(result)) + } + private mutating func call(_ name: String, arguments: [Int], frame: inout Frame, incoming: Event?, input: inout Input, context: inout CLIJobContext) throws -> Action { + if name == "input" || name == "inputs" { + if name == "input" && frame.phase > 0 { return .emit(.end) } + frame.phase += 1 + if let value = try input.next(context: &context) { return .emit(.value(value)) } + if name == "input" { throw FilterLanguageFailure("endOfInput") } + return .emit(.end) + } + if arguments.isEmpty { + if frame.phase > 0 { return .emit(.end) }; frame.phase = 1 + if name == "error" { throw try languageError(frame.input.value, frame: &frame, context: &context) } + if let value = try FilterPrimitiveBuiltins.apply(name, value: frame.input.value, context: &context) { + // Selectors borrow the original input's lifetime; constructed values + // already have their own complete admission lease. + return .emit(.value(value.lease == nil ? frame.input.selecting(value.value) : value)) + } + return .emit(.end) + } + if name == "select" || name == "has" || name == "error" { + if frame.phase == 0 { frame.phase = 1; return .pull(try child(arguments[0], frame: &frame, context: &context)) } + if let incoming { + switch incoming { + case .end: return .emit(.end) + case .value(let value): + if name == "error" { throw try languageError(value.value, frame: &frame, context: &context) } + if name == "select" { + if FilterTruth.test(value.value) { return .emit(.value(frame.input)) } + return .pull(frame.children[0]) + } + let result = try FilterPrimitiveBuiltins.apply(name, value: frame.input.value, argument: value.value, context: &context) + return .emit(result.map(Event.value) ?? .end) + case .failure(let failure): throw failure + } + } + return .pull(frame.children[0]) + } + if name == "map" || name == "map_values" { + let count: Int + switch frame.input.value { case .array(let a): count = a.count; case .object(let o): count = o.count; default: throw FilterLanguageFailure("expectedContainer") } + if let incoming { + switch incoming { + case .value(let value): + if name == "map_values", case .object(let object) = frame.input.value { + try context.charge(.collectionValues); frame.collectionSlots += 1 + try context.meter.retain(40); frame.slots += 40 + frame.members.append((object.members[frame.index].key, value)) + } else { try capture(value, frame: &frame, context: &context) } + if name == "map_values" { try dispose(frame.children.removeLast(), context: &context); frame.index += 1; frame.phase = 0 } + else { return .pull(frame.children.last!) } + case .end: try dispose(frame.children.removeLast(), context: &context); frame.index += 1; frame.phase = 0 + case .failure(let failure): throw failure + } + } + if frame.index == count { + let result: FilterYield, transfer = frame.collectionSlots + if name == "map_values", case .object = frame.input.value { + let bytes = UInt64(frame.members.count) * 24 + try context.meter.retain(bytes); frame.slots += bytes; try context.work(UInt64(frame.members.count)) + let members = frame.members.map { JSONMember($0.0, value: $0.1.value) } + let entry = context.meter.used(.retainedBytes) + let object: JSONObject + do { object = try JSONObject.indexed(admitted: members, meter: &context.meter) } + catch { frame.slots += context.meter.used(.retainedBytes) - entry; throw error } + let indexBytes = context.meter.used(.retainedBytes) - entry + frame.slots += indexBytes + result = try FilterValues.produced(.object(object), collections: transfer, context: &context) + try context.meter.release(indexBytes); frame.slots -= indexBytes + try context.meter.release(bytes); frame.slots -= bytes + } else { + let bytes = UInt64(frame.values.count) * 16 + try context.meter.retain(bytes); frame.slots += bytes; try context.work(UInt64(frame.values.count)) + let values = frame.values.map(\.value) + result = try FilterValues.array(values, admittedCollectionCount: true, context: &context) + try context.meter.release(bytes); frame.slots -= bytes + } + try releaseCache(&frame, context: &context, transferring: transfer) + frame.phase = 99; return .emit(.value(result)) + } + if frame.phase == 0 { + let value: JSON + switch frame.input.value { case .array(let a): value = a[frame.index]; case .object(let o): value = o.members[frame.index].value; default: throw FilterLanguageFailure("expectedContainer") } + frame.phase = 1 + return .pull(try child(arguments[0], frame: &frame, value: frame.input.selecting(value), context: &context)) + } + return .pull(frame.children.last!) + } + throw FilterCompileFailure("unsupportedBuiltin", at: program.sourceRanges[frame.node].lowerBound) + } + private func languageError(_ value: JSON, frame: inout Frame, context: inout CLIJobContext) throws -> FilterLanguageFailure { + try context.checkpoint() + let entry = context.meter.used(.retainedBytes) + var building = true, scratch: UInt64 = 0 + defer { try? context.meter.release(building ? context.meter.used(.retainedBytes) - entry : scratch) } + let text: String + if case .string(let source) = value { text = source } + else { + try context.meter.resetValueBounds() + let bytes = try JSONWriter(options: .init(limits: context.meter.limits, numberStyle: .preserved)).write(value, meter: &context.meter) + try context.work(UInt64(bytes.count)); try context.meter.retain(UInt64(bytes.count)) + text = String(decoding: bytes, as: UTF8.self) + } + try context.require(.diagnosticBytes, UInt64(text.utf8.count)) + scratch = context.meter.used(.retainedBytes) - entry; building = false + // Keep the admitted message live until propagation consumes this frame. + // A catch handler admits its own copy before disposing the failing body. + frame.current = try FilterValues.produced(.string(text), context: &context) + return .init("explicitError", message: text) + } +} diff --git a/Sources/JSONFilterRuntime/FilterValues.swift b/Sources/JSONFilterRuntime/FilterValues.swift new file mode 100644 index 0000000..2beaacc --- /dev/null +++ b/Sources/JSONFilterRuntime/FilterValues.swift @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import JSON + +package enum FilterValues { + package static func produced(_ value:JSON, collections:UInt64=0, context:inout CLIJobContext)throws->FilterYield { + try context.checkpoint() + let entry=context.meter.used(.retainedBytes) + do { + try context.meter.resetValueBounds() + try JSONAdmission.value(value,meter:&context.meter,owned:true) + let bytes=context.meter.used(.retainedBytes)-entry + let lease=try CLIValueLease(ownedRetained:bytes,collections:collections,context:&context) + return .init(value:value,lease:lease,collectedValues:collections) + } catch { + try context.meter.release(context.meter.used(.retainedBytes)-entry) + throw error + } + } + package static func integer(_ value:Int, context:inout CLIJobContext)throws->FilterYield { + try context.work(20) + return try produced(.number(JSONNumber(integer:value)),context:&context) + } + package static func array(_ values:[JSON], admittedCollectionCount:Bool=false, context:inout CLIJobContext)throws->FilterYield { + let count=UInt64(values.count) + if !admittedCollectionCount { try context.charge(.collectionValues,count) } + do { return try produced(.array(values),collections:count,context:&context) } + catch { if !admittedCollectionCount { try context.release(.collectionValues,count) };throw error } + } + package static func index(_ value:JSON,key:JSON,context:inout CLIJobContext)throws->JSON { + switch key { + case .string(let text): + switch value {case .null:return .null;case .object(let object):return try object.lookup(text,meter:&context.meter) ?? .null;default:throw FilterLanguageFailure("cannotIndex")} + case .number(let number): + guard number.isInteger else {throw FilterLanguageFailure("nonintegralIndex")} + switch value { + case .null:return .null + case .array(let values): + let length=JSONNumber(integer:values.count),negative=JSONNumber(integer:-values.count) + if try number.compare(to:length,meter:&context.meter) != .orderedAscending || number.compare(to:negative,meter:&context.meter) == .orderedAscending {return .null} + let integer=try number.integer(as:Int.self,meter:&context.meter) + return values[integer < 0 ? values.count+integer:integer] + default:throw FilterLanguageFailure("cannotIndex") + } + default:throw FilterLanguageFailure("invalidIndexKind") + } + } + package static func has(_ value:JSON,key:JSON,context:inout CLIJobContext)throws->Bool { + switch (value,key) { + case (.object(let object),.string(let text)):return try object.lookup(text,meter:&context.meter) != nil + case (.array(let values),.number(let number)): + guard number.isInteger else {throw FilterLanguageFailure("nonintegralIndex")} + return try number.compare(to:JSONNumber(integer:0),meter:&context.meter) != .orderedAscending && number.compare(to:JSONNumber(integer:values.count),meter:&context.meter) == .orderedAscending + default:throw FilterLanguageFailure("invalidHasDomain") + } + } + package static func textCompare(_ a:String,_ b:String,context:inout CLIJobContext)throws->Int { + var left=a.utf8.makeIterator(),right=b.utf8.makeIterator() + while true { + try context.work();let a=left.next(),b=right.next() + if a==b {if a==nil{return 0};continue} + guard let a else{return -1};guard let b else{return 1} + return a[JSONMember] { + let bytes=UInt64(object.count)*JSONLogicalWeights.member + try context.meter.retain(bytes*2) + var completed = false + defer { if !completed { try? context.meter.release(bytes*2) } } + try context.work(UInt64(object.count)) + var values=object.members, scratch=object.members, width=1 + while widthvalues.count/2{break};width *= 2 + } + try context.meter.release(bytes); completed = true + return values + } + package static func compare(_ a:JSON,_ b:JSON,context:inout CLIJobContext)throws->Int { + func rank(_ v:JSON)->Int {switch v{case .null:0;case .boolean(false):1;case .boolean(true):2;case .number:3;case .string:4;case .array:5;case .object:6}} + var pending:[(JSON,JSON)]=[] + try context.meter.retain(16);pending.append((a,b)) + defer{try? context.meter.release(UInt64(pending.count)*16)} + while let (a,b)=pending.popLast() { + try context.meter.release(16);try context.work() + let ra=rank(a),rb=rank(b);if ra != rb{return raInt { + guard let value else{return fallback} + guard case .number(let number)=value,number.isInteger else{throw FilterLanguageFailure("invalidSliceBound")} + if try number.compare(to:JSONNumber(integer:length),meter:&context.meter) != .orderedAscending{return length} + if try number.compare(to:JSONNumber(integer:-length),meter:&context.meter) == .orderedAscending{return 0} + let index=try number.integer(as:Int.self,meter:&context.meter) + return index<0 ? length+index:index + } + package static func slice(_ value:JSON,start:JSON?,end:JSON?,context:inout CLIJobContext)throws->FilterYield { + if case .null=value{return .init(value:.null)} + switch value { + case .array(let values): + let lower=try bound(start,length:values.count,default:0,context:&context),upper=try bound(end,length:values.count,default:values.count,context:&context) + let count=max(0,upper-lower) + try context.charge(.collectionValues,UInt64(count)) + var transferred=false, scratch:UInt64=0 + defer { try? context.meter.release(scratch); if !transferred { try? context.release(.collectionValues, UInt64(count)) } } + try context.meter.retain(UInt64(count)*16); scratch=UInt64(count)*16; try context.work(UInt64(count)) + let result=Array(values[lower..<(lower+count)]) + let output=try array(result,admittedCollectionCount:true,context:&context);transferred=true;return output + case .string(let text): + var length=0 + for _ in text.unicodeScalars{try context.work();length += 1} + let lower=try bound(start,length:length,default:0,context:&context),upper=try bound(end,length:length,default:length,context:&context) + var output="",offset=0,bytes:UInt64=0 + defer { try? context.meter.release(bytes) } + for scalar in text.unicodeScalars { + try context.work() + if offset>=lower && offset0 then ., (.-1|down) else . end; 3|down", + "expected_json_values": [ + "3", + "2", + "1", + "0" + ] + }, + { + "id": "walk-child-before-parent", + "gate": "C4", + "requirement_ids": [ + "CL058" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "{\"a\":[1,2]}\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": "walk(if type==\"number\" then .+1 else . end)", + "expected_json_values": [ + "{\"a\":[2,3]}" + ] + }, + { + "id": "recurse-null-leaves", + "gate": "C4", + "requirement_ids": [ + "CL058" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "{\"a\":null}\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": "[recurse]", + "expected_json_values": [ + "[{\"a\":null},null]" + ] + }, + { + "id": "normals-nonzero-native", + "gate": "C2", + "requirement_ids": [ + "CL037" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "0\n1e-999\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": "normals", + "expected_json_values": [ + "1e-999" + ] + }, + { + "id": "split-empty-scalars", + "gate": "C4", + "requirement_ids": [ + "CL063" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "\"e\\u0301\"\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": "split(\"\")", + "expected_json_values": [ + "[\"e\",\"\\u0301\"]" + ] + }, + { + "id": "index-empty-pattern", + "gate": "C4", + "requirement_ids": [ + "CL063" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "\"abc\"\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": "index(\"\")", + "expected_json_values": [ + "null" + ] + }, + { + "id": "try-prior-yield", + "gate": "C4", + "requirement_ids": [ + "CL059" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "null\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": "try (1,error(\"x\"),2) catch \"caught\"", + "expected_json_values": [ + "1", + "\"caught\"" + ] + }, + { + "id": "sort-exact-integers", + "gate": "C4", + "requirement_ids": [ + "CL060" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "[9007199254740993,9007199254740992]\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": "sort", + "expected_json_values": [ + "[9007199254740992,9007199254740993]" + ] + }, + { + "id": "total-order", + "gate": "C4", + "requirement_ids": [ + "CL060" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "[{},[],\"a\",1,true,false,null]\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": "sort", + "expected_json_values": [ + "[null,false,true,1,\"a\",[],{}]" + ] + }, + { + "id": "group-scalar-distinct", + "gate": "C4", + "requirement_ids": [ + "CL060" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "[\"é\",\"e\\u0301\",\"é\"]\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": "group_by(.)", + "expected_json_values": [ + "[[\"e\\u0301\"],[\"é\",\"é\"]]" + ] + }, + { + "id": "first-generator", + "gate": "C4", + "requirement_ids": [ + "CL061" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "null\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": "first(1,2,3)", + "expected_json_values": [ + "1" + ] + }, + { + "id": "all-any-empty", + "gate": "C4", + "requirement_ids": [ + "CL062" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "[]\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": "[any,all]", + "expected_json_values": [ + "[false,true]" + ] + }, + { + "id": "explode-non-BMP", + "gate": "C4", + "requirement_ids": [ + "CL064" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "\"😀\"\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": "explode", + "expected_json_values": [ + "[128512]" + ] + }, + { + "id": "implode-surrogate-rejected", + "gate": "C4", + "requirement_ids": [ + "CL064" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "[55296]\n", + "expected_exit": 5, + "comparison": "process-contract", + "status": "not_run", + "filter": "implode" + }, + { + "id": "tonumber-exact", + "gate": "C4", + "requirement_ids": [ + "CL064" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "\"9007199254740993\"\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": "tonumber", + "expected_json_values": [ + "9007199254740993" + ] + }, + { + "id": "tonumber-whitespace-rejected", + "gate": "C4", + "requirement_ids": [ + "CL064" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "\" \"\n", + "expected_exit": 5, + "comparison": "process-contract", + "status": "not_run", + "filter": "tonumber" + }, + { + "id": "setpath-created-array", + "gate": "C4", + "requirement_ids": [ + "CL065" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "{}\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": "setpath([\"a\",2];7)", + "expected_json_values": [ + "{\"a\":[null,null,7]}" + ] + }, + { + "id": "assignment-set-generator", + "gate": "C4", + "requirement_ids": [ + "CL066" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "[1,2]\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": ".[]=(10,20)", + "expected_json_values": [ + "[10,10]", + "[20,20]" + ] + }, + { + "id": "assignment-update-first", + "gate": "C4", + "requirement_ids": [ + "CL066" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "[1,2]\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": ".[]|=(10,20)", + "expected_json_values": [ + "[10,10]" + ] + }, + { + "id": "assignment-update-delete", + "gate": "C4", + "requirement_ids": [ + "CL066" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "[1,2]\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": ".[]|=empty", + "expected_json_values": [ + "[]" + ] + }, + { + "id": "assignment-overlap-rejected", + "gate": "C4", + "requirement_ids": [ + "CL067" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "{\"a\":0}\n", + "expected_exit": 5, + "comparison": "process-contract", + "status": "not_run", + "filter": "(.,.a)=1", + "expected_diagnostic_code": "overlappingPaths" + }, + { + "id": "sparse-growth-fails", + "gate": "C4", + "requirement_ids": [ + "CL068" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "[]\n", + "expected_exit": 6, + "comparison": "process-contract", + "status": "not_run", + "filter": "setpath([1000000000];1)", + "probe": { + "assert_preallocation_admission": true + } + }, + { + "id": "interpolation-generators", + "gate": "C4", + "requirement_ids": [ + "CL069" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "null\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": "\"x\\((1,2))\"", + "expected_json_values": [ + "\"x1\"", + "\"x2\"" + ] + }, + { + "id": "interpolation-empty", + "gate": "C4", + "requirement_ids": [ + "CL069" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "null\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": "\"x\\(empty)\"", + "expected_json_values": [] + }, + { + "id": "format-html", + "gate": "C4", + "requirement_ids": [ + "CL070" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "\"<&>\"\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": "@html", + "expected_json_values": [ + "\"<&>\"" + ] + }, + { + "id": "format-base64", + "gate": "C4", + "requirement_ids": [ + "CL070" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "\"hello\"\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "filter": "@base64", + "expected_json_values": [ + "\"aGVsbG8=\"" + ] + }, + { + "id": "regex-not-advertised", + "gate": "C4", + "requirement_ids": [ + "CL071" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "\"a\"\n", + "expected_exit": 3, + "comparison": "process-contract", + "status": "not_run", + "filter": "test(\"a\")" + }, + { + "id": "env-not-available", + "gate": "C4", + "requirement_ids": [ + "CL072" + ], + "command": "query", + "options": [], + "operands": [], + "stdin_utf8": "null\n", + "expected_exit": 3, + "comparison": "process-contract", + "status": "not_run", + "filter": "env" + }, + { + "id": "stream-root-array", + "gate": "C5", + "requirement_ids": [ + "CL074" + ], + "command": "stream", + "options": [], + "operands": [ + "array" + ], + "stdin_utf8": "[1,{\"a\":2},null]\n", + "expected_exit": 0, + "comparison": "native-exact-value-stream", + "status": "not_run", + "expected_json_values": [ + "1", + "{\"a\":2}", + "null" + ] + }, + { + "id": "stream-root-array-bad-tail", + "gate": "C5", + "requirement_ids": [ + "CL074" + ], + "command": "stream", + "options": [], + "operands": [ + "array" + ], + "stdin_utf8": "[1,2] nope", + "expected_exit": 5, + "comparison": "process-contract", + "status": "not_run", + "probe": { + "assert_final_completed": false, + "stdout_may_be_valid_prefix": true + } + }, + { + "id": "schema-provider-object-envelope", + "gate": "C5", + "requirement_ids": [ + "CL078" + ], + "command": "schema", + "options": [ + "--name", + "sample" + ], + "operands": [ + "provider", + "openai-format", + "fixtures/new/schema-provider.json" + ], + "stdin_utf8": "", + "expected_exit": 0, + "comparison": "process-contract", + "status": "not_run", + "probe": { + "assert_path_types": { + "/schema": "object" + }, + "assert_no_network": true + } + }, + { + "id": "schema-provider-optionality-reject", + "gate": "C5", + "requirement_ids": [ + "CL079" + ], + "command": "schema", + "options": [ + "--name", + "sample" + ], + "operands": [ + "provider", + "openai-format", + "fixtures/new/schema-provider-optional.json" + ], + "stdin_utf8": "", + "expected_exit": 1, + "comparison": "process-contract", + "status": "not_run", + "probe": { + "assert_no_lowering": true + } + }, + { + "id": "render-static-inert", + "gate": "C5", + "requirement_ids": [ + "CL075" + ], + "command": "render", + "options": [ + "--format", + "html" + ], + "operands": [], + "stdin_utf8": "{\"x\":\"\",\"n\":9007199254740993}\n", + "expected_exit": 0, + "comparison": "process-contract", + "status": "not_run", + "probe": { + "assert_no_script_element": true, + "assert_exact_value": true, + "assert_no_external_assets": true + } + }, + { + "id": "render-markdown", + "gate": "C5", + "requirement_ids": [ + "CL077" + ], + "command": "render", + "options": [ + "--format", + "markdown" + ], + "operands": [], + "stdin_utf8": "{\"a\":[1.2300e+2]}\n", + "expected_exit": 0, + "comparison": "stdout-byte-exact", + "status": "not_run", + "expected_stdout_utf8": "```json\n{\n\t\"a\": [\n\t\t1.2300e+2\n\t]\n}\n```\n" + } + ] +} diff --git a/Tests/JSONCLITests/QueryBudgetTests.swift b/Tests/JSONCLITests/QueryBudgetTests.swift new file mode 100644 index 0000000..e7e6e2f --- /dev/null +++ b/Tests/JSONCLITests/QueryBudgetTests.swift @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: Apache-2.0 +import Testing +import JSON +@testable import JSONFilterRuntime + +@Test(arguments: CLIResourceMetric.allCases) +func cliEveryCounterHasInclusiveCheckedBoundaryAndTerminalFailure(metric: CLIResourceMetric) throws { + var limits = CLILimits(); limits[metric] = 3 + var context = try CLIJobContext(limits: limits) + try context.charge(metric, 3) + #expect(context.used(metric) == 3) + do { try context.charge(metric); Issue.record("Expected cap+1 rejection") } + catch let error as CLIError { #expect(error.category == .resource && error.metric == metric.rawValue) } + #expect(throws: CLIError.self) { try context.checkpoint() } +} +@Test(arguments: [CLIResourceMetric.querySourceBytes, .queryTokens, .astNodes, .queryDepth]) +func cliCompilerEnforcesCapsAtActualSourceTokenAndIRGrowth(metric: CLIResourceMetric) throws { + var limits = CLILimits(); limits[metric] = 1 + var allowed = try CLIJobContext(limits: limits) + let program = try FilterCompiler.compile(".", context: &allowed) + #expect(program.nodes.count == 1) + limits[metric] = 0 + var blocked = try CLIJobContext(limits: limits) + do { _ = try FilterCompiler.compile(".", context: &blocked); Issue.record("Expected actual compiler admission failure") } + catch let failure as CLIError { #expect(failure.category == .resource && failure.metric == metric.rawValue) } +} +@Test func cliParserDepthUsesConfiguredWorkStackAndRejectsWithoutRecursion() throws { + var limits = CLILimits(); limits[.queryDepth] = 64 + var context = try CLIJobContext(limits: limits) + let source = String(repeating: "(", count: 65) + "." + String(repeating: ")", count: 65) + do { _ = try FilterCompiler.compile(source, context: &context); Issue.record("Expected depth cap") } + catch let failure as CLIError { #expect(failure.metric == "queryDepth") } +} +@Test func cliRuntimeFramesAndBindingsAreLiveOwnedCaps() throws { + var limits = CLILimits(); limits[.runtimeFrames] = 0 + var context = try CLIJobContext(limits: limits) + let program = try FilterCompiler.compile(".", context: &context) + do { _ = try FilterVM(program: program, input: .init(value: .null), context: &context); Issue.record("Expected frame cap before growth") } + catch let failure as CLIError { #expect(failure.metric == "runtimeFrames") } +} +@Test func cliQueryPlanIsImmutableAndCheckedSendable() throws { + func requireSendable(_ value: T) {} + var context = try CLIJobContext() + let program = try FilterCompiler.compile(".a, .b", context: &context) + requireSendable(program) + #expect(program.nodes.count > 1 && program.sourceRanges.count == program.nodes.count) +} + +@Test func cliQueryOperationalFailureBypassesTryAndCleansInvocationStorage() throws { + var compilation = try CLIJobContext() + let program = try FilterCompiler.compile("try [1,2,3] catch null", context: &compilation) + var limits = CLILimits(); limits[.collectionValues] = 2 + var context = try CLIJobContext(limits: limits) + let baseline = context.meter.used(.retainedBytes) + var vm = try FilterVM(program: program, input: .init(value: .null), context: &context) + struct NoInput: FilterInput { mutating func next(context: inout CLIJobContext) throws -> FilterYield? { nil } } + var input = NoInput() + do { _ = try vm.next(input: &input, context: &context); Issue.record("Resource failure must not become catch null") } + catch let failure as CLIError { #expect(failure.category == .resource && failure.metric == "collectionValues") } + try vm.close(context: &context) + #expect(context.used(.runtimeFrames) == 0 && context.used(.collectionValues) == 0) + #expect(context.meter.used(.retainedBytes) == baseline) + #expect(throws: CLIError.self) { try context.checkpoint() } +} +@Test func cliQueryCancellationHappensInsideOwnedEvaluationAndCleansFrames() async throws { + var compilation = try CLIJobContext() + let program = try FilterCompiler.compile(".[]", context: &compilation) + let task = Task { + var context = try CLIJobContext() + let baseline = context.meter.used(.retainedBytes) + var value = try FilterValues.produced(.array((0..<100).map { .number(JSONNumber(integer: $0)) }), context: &context) + var vm = try FilterVM(program: program, input: value, context: &context) + try value.release(context: &context) + struct NoInput: FilterInput { mutating func next(context: inout CLIJobContext) throws -> FilterYield? { nil } } + var input = NoInput() + var pending = try vm.next(input: &input, context: &context) + var first = try #require(pending); pending = nil + try first.release(context: &context) + context.meter.checkpointObserver = { _ in withUnsafeCurrentTask { $0?.cancel() } } + #expect(throws: CancellationError.self) { _ = try vm.next(input: &input, context: &context) } + try vm.close(context: &context) + #expect(context.used(.runtimeFrames) == 0 && context.used(.collectionValues) == 0) + #expect(context.meter.used(.retainedBytes) == baseline) + } + try await task.value +} + +@Test func cliQueryConstructorsAndSelectorsReleaseScratchAfterMidOperationFailure() throws { + let object = try JSONParser().parse(Array("{\"a\":1,\"b\":2,\"c\":3,\"d\":4,\"e\":5,\"f\":6}".utf8)) + let array = JSON.array((0..<20).map { .number(JSONNumber(integer: $0)) }) + struct NoInput: FilterInput { mutating func next(context: inout CLIJobContext) throws -> FilterYield? { nil } } + for (source, value) in [("keys", object), (".[1:15]", array), ("map_values(.)", object), ("{a: .a,b: .b}", object), ("try error(.) catch .", object)] { + var compilation = try CLIJobContext() + let program = try FilterCompiler.compile(source, context: &compilation) + var rejected = 0 + for cap: UInt64 in [360, 720, 1440, 2880] { + var limits = CLILimits(); limits[.retainedBytes] = cap + var context = try CLIJobContext(limits: limits) + var vm = try FilterVM(program: program, input: .init(value: value), context: &context) + var input = NoInput() + do { while var result = try vm.next(input: &input, context: &context) { try result.release(context: &context) } } + catch { rejected += 1 } + try vm.close(context: &context) + #expect(context.used(.runtimeFrames) == 0 && context.used(.collectionValues) == 0) + #expect(context.meter.used(.retainedBytes) == 0, "\(source) cap=\(cap)") + } + #expect(rejected > 0) + } +} diff --git a/Tests/JSONCLITests/QueryCoreTests.swift b/Tests/JSONCLITests/QueryCoreTests.swift new file mode 100644 index 0000000..ef22c05 --- /dev/null +++ b/Tests/JSONCLITests/QueryCoreTests.swift @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: Apache-2.0 +import Testing +import JSON +@testable import JSONFilterRuntime + +private struct EmptyFilterInput: FilterInput { + mutating func next(context: inout CLIJobContext) throws -> FilterYield? { nil } +} +private func evaluateFilter(_ source: String, input: String = "null") throws -> [JSON] { + var context = try CLIJobContext() + let program = try FilterCompiler.compile(source, context: &context) + let entry = context.meter.used(.retainedBytes) + let value = try JSONParser().parse(Array(input.utf8), meter: &context.meter) + let owned = context.meter.used(.retainedBytes) - entry + let lease = try CLIValueLease(ownedRetained: owned, context: &context) + var vm = try FilterVM(program: program, input: .init(value: value, lease: lease), context: &context) + defer { try? vm.close(context: &context) } + var input = EmptyFilterInput(), values: [JSON] = [] + while var result = try vm.next(input: &input, context: &context) { + values.append(result.value) + try result.release(context: &context) + } + return values +} +@Test func cliQueryCommaAndPipeAreOrderedPullStreams() throws { + #expect(try evaluateFilter("1,2 | .") == [.number(1), .number(2)]) + #expect(try evaluateFilter(".[] | .", input: "[1,2,3]") == [.number(1), .number(2), .number(3)]) + #expect(try evaluateFilter("[1,2,3]") == [.array([.number(1), .number(2), .number(3)])]) +} +@Test func cliQueryDecodedScalarKeysAndPresenceStayExact() throws { + let input = "{\"é\":1,\"é\":2,\"null\":null}" + #expect(try evaluateFilter(".[\"é\"], .[\"é\"]", input: input) == [.number(1), .number(2)]) + #expect(try evaluateFilter("has(\"null\"),has(\"missing\")", input: input) == [.boolean(true), .boolean(false)]) + #expect(try evaluateFilter(".missing?", input: input) == [.null]) +} +@Test func cliQueryWrongTypesAreCatchableButEmptyIsNotNull() throws { + #expect(try evaluateFilter(".[]?", input: "null").isEmpty) + #expect(try evaluateFilter("try .[] catch .", input: "null") == [.string("cannotIterate")]) + #expect(try evaluateFilter("empty // 9") == [.number(9)]) + #expect(try evaluateFilter("(false,null,0) // 9") == [.number(0)]) +} +@Test func cliQueryLogicalShortCircuitAndBranchesUseOriginalInput() throws { + #expect(try evaluateFilter("false and error(\"unreachable\")") == [.boolean(false)]) + #expect(try evaluateFilter("true or error(\"unreachable\")") == [.boolean(true)]) + #expect(try evaluateFilter("if (true,false) then . else null end", input: "7") == [.number(7), .null]) + #expect(try evaluateFilter("if . == 0 then 0 elif . == 1 then 2 else 3 end", input: "1") == [.number(2)]) +} +@Test func cliQueryMapsKeepAllResultsAndFirstUpdatePolicy() throws { + #expect(try evaluateFilter("map((.,.))", input: "[1,2]") == [.array([.number(1), .number(1), .number(2), .number(2)])]) + #expect(try evaluateFilter("map_values((.,9))", input: "[1,2]") == [.array([.number(1), .number(2)])]) + #expect(try evaluateFilter("map_values(empty)", input: "[1,2]") == [.array([])]) +} +@Test func cliQueryUnicodeLengthCountsScalarsAndRecursiveNullLeaves() throws { + #expect(try evaluateFilter("length,utf8bytelength", input: "\"é🦊\"") == [.number(3), .number(7)]) + #expect(try evaluateFilter("..", input: "[null,1]") == [.array([.null, .number(1)]), .null, .number(1)]) +} +@Test func cliQueryAliasesKeepSourceChargedUntilTheirLastRelease() throws { + var context = try CLIJobContext() + let baseline = context.meter.used(.retainedBytes) + var value = try FilterValues.produced(.array([.string(String(repeating: "x", count: 1000))]), context: &context) + var alias = value.selecting(value.value.arrayValue![0]) + try value.release(context: &context) + #expect(context.meter.used(.retainedBytes) > baseline) + try alias.release(context: &context) + #expect(context.meter.used(.retainedBytes) == baseline) +} +@Test func cliQueryTerminalQuotaStillReleasesValueStorage() throws { + var limits = CLILimits(); limits[.totalWorkUnits] = 2000 + var context = try CLIJobContext(limits: limits) + let baseline = context.meter.used(.retainedBytes) + var value = try FilterValues.produced(.string("owned"), context: &context) + do { try context.work(2000); Issue.record("Expected terminal work failure") } catch let failure as CLIError { #expect(failure.category == .resource) } + try value.release(context: &context) + #expect(context.meter.used(.retainedBytes) == baseline) + #expect(throws: CLIError.self) { try context.checkpoint() } +} +@Test func cliQueryObjectsUseCartesianOrderAndExactReplacement() throws { + let values = try evaluateFilter("{a:(1,2),b:(3,4)}") + let expected = try ["{\"a\":1,\"b\":3}","{\"a\":1,\"b\":4}","{\"a\":2,\"b\":3}","{\"a\":2,\"b\":4}"].map { try JSONParser().parse($0) } + #expect(values == expected) + let replacement = try evaluateFilter("{a:1,b:2,a:3}") + #expect(replacement[0].objectValue!.members.map(\.key.string) == ["a","b"]) + #expect(replacement[0].objectValue![JSONKey("a")] == .number(3)) + #expect(try evaluateFilter("{\"é\":1,\"é\":2}")[0].objectValue!.count == 2) +} +@Test func cliQuerySlicesUseScalarBoundariesAndCheckedNegativeIndices() throws { + #expect(try evaluateFilter(".[1:3]", input: "\"é🦊x\"") == [.string("́🦊")]) + #expect(try evaluateFilter(".[-2:]", input: "[1,2,3]") == [.array([.number(2),.number(3)])]) + #expect(try evaluateFilter(".[-1],.[1e1000]", input: "[1,2,3]") == [.number(3),.null]) +} +@Test func sharedFilterPlanHasIndependentConcurrentInvocations() async throws { + var compilation = try CLIJobContext() + let program = try FilterCompiler.compile(".a, .b", context: &compilation) + let outputs = try await withThrowingTaskGroup(of: [JSON].self, returning: [[JSON]].self) { group in + for _ in 0..<8 { + group.addTask { + var context = try CLIJobContext() + let input = try JSONParser().parse("{\"a\":1,\"b\":2}") + var value = try FilterValues.produced(input, context: &context) + var vm = try FilterVM(program: program, input: value, context: &context), source = EmptyFilterInput() + defer { try? vm.close(context: &context) } + try value.release(context: &context) + var results: [JSON] = [] + while var result = try vm.next(input: &source, context: &context) { results.append(result.value); try result.release(context: &context) } + #expect(context.used(.totalWorkUnits) > 0) + #expect(context.meter.used(.retainedBytes) == 0) + return results + } + } + var values: [[JSON]] = [] + for try await value in group { values.append(value) } + return values + } + #expect(outputs.count == 8 && outputs.allSatisfy { $0 == [.number(1), .number(2)] }) +} diff --git a/Tests/JSONCLITests/QueryInputTests.swift b/Tests/JSONCLITests/QueryInputTests.swift new file mode 100644 index 0000000..ab77db2 --- /dev/null +++ b/Tests/JSONCLITests/QueryInputTests.swift @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import Testing +import JSON +@testable import JSONCLIEngine +@testable import JSONFilterRuntime + +private func withInputFile(_ bytes: [UInt8], _ body: (String) throws -> T) throws -> T { + let root = FileManager.default.temporaryDirectory.appendingPathComponent("json-query-" + UUID().uuidString) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: false) + defer { try? FileManager.default.removeItem(at: root) } + let path = root.appendingPathComponent("input.json") + try Data(bytes).write(to: path) + return try body(path.path) +} +@Test func cliQueryValueFramingRequiresWhitespaceAndIndependentFileEOF() throws { + try withInputFile(Array("1 2\n{}".utf8)) { path in + var context = try CLIJobContext(), cursor = CLIInputCursor(files: [path,path]) + defer { try? cursor.close(context: &context) } + var values: [JSON] = [] + while var value = try cursor.next(context: &context) { values.append(value.value); try value.release(context: &context) } + #expect(values.count == 6 && values[0] == .number(1) && values[3] == .number(1)) + } + try withInputFile(Array("{}[]".utf8)) { path in + var context = try CLIJobContext(), cursor = CLIInputCursor(files: [path]) + defer { try? cursor.close(context: &context) } + #expect(throws: JSONFailure.self) { _ = try cursor.next(context: &context) } + } +} +@Test func cliQueryDocumentWaitsForEOFAndTailErrorsAreTerminal() throws { + try withInputFile(Array("1 2".utf8)) { path in + var context = try CLIJobContext(), cursor = CLIInputCursor(files: [path], framing: .document) + defer { try? cursor.close(context: &context) } + #expect(throws: JSONFailure.self) { _ = try cursor.next(context: &context) } + #expect(throws: CLIError.self) { _ = try cursor.next(context: &context) } + } + try withInputFile(Array("1\n{\"bad\":]".utf8)) { path in + var context = try CLIJobContext(), cursor = CLIInputCursor(files: [path]) + defer { try? cursor.close(context: &context) } + let maybeFirst = try cursor.next(context: &context); var first = try #require(maybeFirst); #expect(first.value == .number(1)); try first.release(context: &context) + #expect(throws: JSONFailure.self) { try cursor.drain(context: &context) } + } +} +@Test func cliQueryStrictNDJSONAndRawLinesPreserveTheirDifferentContracts() throws { + try withInputFile(Array("1\n\n2\n".utf8)) { path in + var context = try CLIJobContext(), cursor = CLIInputCursor(files: [path], framing: .ndjson) + defer { try? cursor.close(context: &context) } + let maybeFirst = try cursor.next(context: &context); var first = try #require(maybeFirst); #expect(first.value == .number(1)); try first.release(context: &context) + #expect(throws: JSONFailure.self) { _ = try cursor.next(context: &context) } + } + try withInputFile(Array("a\r\n\n🦊".utf8)) { path in + var context = try CLIJobContext(), cursor = CLIInputCursor(files: [path], framing: .rawLines) + defer { try? cursor.close(context: &context) } + var values: [JSON] = [] + while var value = try cursor.next(context: &context) { values.append(value.value); try value.release(context: &context) } + #expect(values == [.string("a"), .string(""), .string("🦊")]) + } +} +@Test func cliQueryRootArrayYieldsProvisionallyAndRequiresClosingEOF() throws { + try withInputFile(Array("[1,2".utf8)) { path in + var context = try CLIJobContext(), cursor = CLIInputCursor(files: [path], framing: .array) + defer { try? cursor.close(context: &context) } + let maybeFirst = try cursor.next(context: &context); var first = try #require(maybeFirst); #expect(first.value == .number(1)); try first.release(context: &context) + let maybeSecond = try cursor.next(context: &context); var second = try #require(maybeSecond); #expect(second.value == .number(2)); try second.release(context: &context) + #expect(throws: JSONFailure.self) { _ = try cursor.next(context: &context) } + } +} +@Test(arguments: [2000, 20000]) func cliQueryManyTinyRecordsDoNotRetainTheWholeInputJob(records: Int) throws { + try withInputFile(Array(String(repeating: "0\n", count: records).utf8)) { path in + var context = try CLIJobContext(), cursor = CLIInputCursor(files: [path]) + defer { try? cursor.close(context: &context) } + var count = 0 + while var value = try cursor.next(context: &context) { count += 1; try value.release(context: &context) } + #expect(count == records) + #expect(context.meter.used(.retainedBytes) == 0) + // One 16 KiB syscall buffer may overlap one accepted 16 KiB chunk; + // the completed scalar/lease has a small fixed reservation. This bound + // follows the actual adapter, rather than an arbitrary RSS threshold. + let chunk = min(UInt64(16384), context.limits.core[.chunkBytes]) + #expect(context.meter.statistics().peakLogicalRetainedBytes <= 2 * chunk + 256) + #expect(context.used(.totalRecords) == UInt64(records) && context.used(.totalWorkUnits) > UInt64(records)) + } +} diff --git a/Tools/CLIQualification/Package.swift b/Tools/CLIQualification/Package.swift new file mode 100644 index 0000000..7d084fa --- /dev/null +++ b/Tools/CLIQualification/Package.swift @@ -0,0 +1,9 @@ +// swift-tools-version: 6.4 +// SPDX-License-Identifier: Apache-2.0 +import PackageDescription +let package = Package(name: "CLIQualification", platforms: [.macOS("27.0")], + products: [.executable(name: "CLIExactCompare", targets: ["CLIExactCompare"])], + dependencies: [.package(name: "Swift-JSON", path: "../..")], + targets: [.executableTarget(name: "CLIExactCompare", dependencies: [ + .product(name: "JSON", package: "Swift-JSON"), .product(name: "JSONStreaming", package: "Swift-JSON")])], + swiftLanguageModes: [.v6]) diff --git a/Tools/CLIQualification/Sources/CLIExactCompare/Main.swift b/Tools/CLIQualification/Sources/CLIExactCompare/Main.swift new file mode 100644 index 0000000..0a9ae2f --- /dev/null +++ b/Tools/CLIQualification/Sources/CLIExactCompare/Main.swift @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import JSON +import JSONStreaming +#if canImport(Darwin) +import Darwin +#else +import Glibc +#endif + +private struct Bytes: AsyncSequence { + typealias Element = JSONByteChunk + let bytes: [UInt8] + struct Iterator: AsyncIteratorProtocol { + let bytes: [UInt8] + var index = 0 + mutating func next() async -> JSONByteChunk? { + guard index < bytes.count else { return nil } + let end = Swift.min(index + 16384, bytes.count) + defer { index = end } + return .init(Array(bytes[index.. Iterator { .init(bytes: bytes) } +} +@main struct CLIExactCompare { + static func main() async { + do { + let arguments = CommandLine.arguments + guard arguments.count == 3 else { throw NSError(domain: "qualification", code: 2) } + let actual = try bounded(arguments[1]), expected = try bounded(arguments[2]) + let declaration = try JSONParser().parse(expected) + guard case .array(let texts) = declaration else { throw NSError(domain: "qualification", code: 2) } + // CLI result streams may be empty. The library's value sequence + // requires a record; verify the zero-result contract explicitly. + if texts.isEmpty { + guard actual.allSatisfy({ [9,10,13,32].contains($0) }) else { throw NSError(domain: "qualification", code: 1) } + print("NATIVE_EXACT_COMPARE passed values=0"); return + } + var iterator = JSONValueSequence(Bytes(bytes: actual), options: .init(framing: .whitespaceSeparatedValues)).makeAsyncIterator() + var count = 0 + for text in texts { + guard case .string(let raw) = text, let actual = try await iterator.next() else { throw NSError(domain: "qualification", code: 1) } + let expected = try JSONParser().parse(raw) + guard try JSONComparison.equals(actual, expected) else { throw NSError(domain: "qualification", code: 1) } + count += 1 + } + guard try await iterator.next() == nil else { throw NSError(domain: "qualification", code: 1) } + print("NATIVE_EXACT_COMPARE passed values=\(count)") + } catch { FileHandle.standardError.write(Data("NATIVE_EXACT_COMPARE failed\n".utf8)); exit(1) } + } + private static func bounded(_ path: String) throws -> [UInt8] { + let handle = try FileHandle(forReadingFrom: URL(fileURLWithPath: path)) + defer { try? handle.close() } + var bytes: [UInt8] = [] + while let data = try handle.read(upToCount: 16384), !data.isEmpty { + guard bytes.count + data.count <= 33554432 else { throw NSError(domain: "qualification", code: 2) } + bytes.append(contentsOf: data) + } + return bytes + } +}