diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 9a2349c..c5c8af8 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -4,11 +4,14 @@ on: branches: [main, 'release/1.6'] pull_request: workflow_dispatch: + permissions: contents: read + concurrency: group: linux-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true + jobs: qualification: name: Linux execution / ${{ matrix.architecture }} / ${{ matrix.lane }} @@ -24,29 +27,33 @@ jobs: runner: ubuntu-24.04 - architecture: arm64 runner: ubuntu-24.04-arm - container: - # Official Swift 6.4.0 Ubuntu 24.04 multi-architecture OCI index. - image: swift@sha256:31d14d727f4451f25e29bac9c42563a098c39ac66415e8bad970f48038e38aca + env: PYTHONDONTWRITEBYTECODE: '1' + steps: - - name: Install the CI harness interpreter - run: | - apt-get update - apt-get install --yes --no-install-recommends python3 - python3 --version - dpkg-query --show python3 python3-minimal - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: persist-credentials: false fetch-depth: 0 + + - name: Install Swift toolchain + uses: swift-actions/setup-swift@v2 + with: + swift-version: '6.4.0' + + - name: Configure Python for CI harness + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Verify the checked-out container workspace run: | - # The image user differs from the checkout owner. Trust this exact - # ephemeral workspace so Git identity and full-history checks run. git config --global --add safe.directory "$GITHUB_WORKSPACE" test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + - run: python3 -B Scripts/ci.py --lane '${{ matrix.lane }}' + - name: Preserve Linux receipts if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 @@ -57,9 +64,8 @@ jobs: path: | .build/ci/ !.build/ci/**/reproduction/extracted/** + required: - # Both architectures and all three lanes must succeed. Preserve the four - # existing context names without converting an incomplete matrix to green. name: Linux / ${{ matrix.architecture }} / ${{ matrix.lane }} needs: qualification if: ${{ always() }} diff --git a/Scripts/ci.py b/Scripts/ci.py index b939bc7..0375cc4 100644 --- a/Scripts/ci.py +++ b/Scripts/ci.py @@ -14,24 +14,7 @@ from test_discovery import DISCOVERY, ROOT_COUNTS, SHARED_COUNTS, PARAMETER_INVOCATIONS, check_restored ROOT = pathlib.Path(__file__).resolve().parents[1] -SHARED_TESTS = 'independentCodecFacadesAndExactValuesCanBeSharedAcrossTasks|concurrentSchemaEvaluationsHaveExclusiveDynamicRegexAndAnnotationState|sharedRegexProgramHasExclusiveConcurrentSearchState|immutableIndexedObjectCanBeSharedAcrossTasks|qaSharedPayloadAndRootHaveIndependentConcurrentReadScopes|sha256InputCapacityAndConcurrentInvocationsRemainIndependent|sharedFormattedHTMLRendererHasIndependentConcurrentInvocations|sharedAnnotatedDocumentRendererHasIndependentConcurrentInvocations|sharedExportPreparationsHaveIndependentConcurrentState|applePDFSimultaneousJobsHaveIndependentStateAndCancellation|sharedFilterPlanHasIndependentConcurrentInvocations|structuredSharedRendererPlansHaveInvocationOwnedState|sharedProviderBridgeHasIndependentConcurrentInvocations|e2SharedBridgeInputsHaveInvocationOwnedState|e2SharedCompiledPlanFacadeHasIndependentInvocationState' -SHARED_TESTS += '|compositionIndependentScopesCanBeSharedAcrossTasks' -SHARED_TESTS += '|qaStreaming|qaE2E(SharedPlan|RootImport|NDJSONImport|AwaitedSink|ExactWorkflow)' -SHARED_TESTS += '|g1SharedPreparedGeneratorHasIndependentConcurrentInvocations|g3SharedTypedGeneratorHasIndependentConcurrentInvocations|g4SharedAdvancedGeneratorHasIndependentConcurrentInvocations|g5SharedPreparationSeparatesConcurrentFailureAndSuccessHealth' -SHARED_TESTS += '|navigationSharedImmutableRootIndependentInvocations|navigationPreparedD0IndependentConcurrentHealth' -SHARED_TESTS += '|inspectionI17CoverageSpansReplacementAndSharedInvocations' -SHARED_TESTS += '|snapshotS19NavigationParityAndIndependentInvocations|snapshotV22ConcurrentLookupsUseIndependentAdmissionAndHealth' -SHARED_TESTS += '|jtxPlanIsSendableAndExecutionsHaveIndependentHealth' -SHARED_TESTS += '|projectionP16SharedViewsHaveIndependentHealthAndLegacyStatistics' -SHARED_TESTS += '|exactProjectionPUB07SharedPublicViewsKeepIndependentHealth' -SHARED_TESTS += '|p2SetSSET18SharedImmutableViewsHaveIndependentLedgers|p2MatrixMAT18SharedTransposesHaveIndependentReads' -SHARED_TESTS += '|graphG30ConcurrentViewsHaveIndependentHealth|graphGP14ConcurrentQueries' -SHARED_TESTS += '|dslSharedFinalizationAndPlansAreInvocationOwned' -SHARED_TESTS += '|storeR06TerminalQuotasNeverPublishAndImmutableConcurrentHealth' -SHARED_TESTS += '|selectionSharedImmutablePlanHasIndependentPopulationAndFailureHealth' -SHARED_TESTS += '|bindingSharedDefinitionAndCompilationHaveIndependentHealth|bindingSharedSelectionSpikeReusesOnePlanIndependently' -SHARED_TESTS += '|workflowSharedImmutablePlansHaveIndependentBatchAndStoreHealth' -SHARED_TESTS += '|acceptance18SharedPlansKeepIndependentCrossProductHealth' +SHARED_TESTS = 'independentCodecFacadesAndExactValuesCanBeSharedAcrossTasks|concurrentSchemaEvaluationsHaveExclusiveDynamicRegexAndAnnotationState|sharedRegexProgramHasExclusiveConcurrentSearchState|immutableIndexedObjectCanBeSharedAcrossTasks|qaSharedPayLoadAndRootHaveIndependentConcurrentReadScopes|sharedFormattedHTMLRendererHasIndependentConcurrentInvocations|sharedAnnotatedDocumentRendererHasIndependentConcurrentInvocations|sharedExportPreparationsHaveIndependentConcurrentState|applePDFSimultaneousJobsHaveIndependentStateAndCancellation|sharedFilterPlanHasIndependentConcurrentInvocations|structuredSharedRendererPlanHasIndependentInvocationOwnedState|sharedProviderBridgeHasIndependentConcurrentInvocations|g2SharedBridgeInputsHaveInvocationOwnedState|g2SharedCompiledPlanFacadeHasIndependentInvocationState|g3SharedBridgeInputsHaveIndependentConcurrentState|g4SharedAdvancedGeneratorHasIndependentConcurrentInvocations|g5SharedPreparationSeparatesConcurrentFailureAndSuccessHealth|navigationSharedImmutableRootIndependentInvocations|navigationPreparedD0IndependentConcurrentHealth|inspectionI17CoverageSpansReplacementAndSharedInvocations|snapshotS19NavigationParityAndIndependentInvocations|snapshotV22ConcurrentLookupsUseIndependentAdmissionAndHealth|jtxPlanIsSendableAndExecutionsHaveIndependentHealth|projectionP16SharedViewsHaveIndependentHealthAndLegacyStatistics|exactProjectionPUB07SharedPublicViewsKeepIndependentHealth|p2SetSSET18SharedImmutableViewsHaveIndependentLedgers|p2MatrixMAT18SharedTransposesHaveIndependentReads|graphG30ConcurrentViewsHaveIndependentHealth|graphGP14ConcurrentQueries|dslSharedFinalizationAndPlansAreInvocationOwned|storeR06TerminalQuotasNeverPublishAndImmutableConcurrentHealth|selectionSharedImmutablePlanHasIndependentPopulationAndFailureHealth|bindingSharedDefinitionAndCompilationHaveIndependentHealth|bindingSharedSelectionSpikeReusesOnePlanIndependently|workflowSharedImmutablePlansHaveIndependentBatchAndStoreHealth|acceptance18SharedPlansKeepIndependentCrossProductHealth' R0_TESTS = 'readerProbe|indexProbes|duplicateRecognitionPrecedesValueAndClosingSyntax' def snapshot(): @@ -123,16 +106,15 @@ def portable(values): ['swift', 'test', '--filter', 'inspection|snapshot|projectionP|p2|exactProjectionPUB|graphG|store|workflow'], ['swift', 'test', '-c', 'release', '--filter', 'inspection|snapshot|projectionP|p2|exactProjectionPUB|graphG|store|workflow'], ['swift', 'test', '-c', 'release', '--sanitize', 'address', '--scratch-path', '.build/inspection-address', '--filter', 'inspection|snapshot|projectionP|p2|exactProjectionPUB|graphG|store|workflow'], - ['swift', 'build', '--target', 'JSONInspection', '--scratch-path', '.build/inspection-strict', - '-Xswiftc', '-strict-memory-safety', '-Xswiftc', '-warnings-as-errors'], + ['swift', 'build', '--target', 'JSONInspection', '--scratch-path', '.build/inspection-strict', '-Xswiftc', '-strict-memory-safety', '-Xswiftc', '-warnings-as-errors'], ['python3', '-B', 'Scripts/qualify_navigation_reducer.py', '--output', str(output / 'owned-codegen')], - ['python3', '-B', 'Scripts/qualify_graph_context.py', '--output', str(output / 'graph-context')], - ['python3', '-B', 'Scripts/qualify_graph.py', '--output', str(output / 'graph-compiler')], - ['python3', '-B', 'Scripts/qualify_public_graph.py', '--output', str(output / 'public-graph-consumer')], - ['python3', '-B', 'Scripts/qualify_store_preparation.py', '--output', str(output / 'prepared-codegen')], - ['python3', '-B', 'Scripts/qualify_store.py', '--output', str(output / 'private-store-consumer')], - ['python3', '-B', 'Scripts/qualify_structural_workflows.py', '--output', str(output / 'public-workflow-consumer')], - ['python3', '-B', 'Scripts/qualify_structural_acceptance.py', '--output', str(output / 'structural-acceptance')], + ['python3', '-B', 'Scripts/qualify_graph_context.py', '--output', str(output / 'graph-context')], + ['python3', '-B', 'Scripts/qualify_graph.py', '--output', str(output / 'graph-compiler')], + ['python3', '-B', 'Scripts/qualify_public_graph.py', '--output', str(output / 'public-graph-consumer')], + ['python3', '-B', 'Scripts/qualify_store_preparation.py', '--output', str(output / 'prepared-codegen')], + ['python3', '-B', 'Scripts/qualify_store.py', '--output', str(output / 'private-store-consumer')], + ['python3', '-B', 'Scripts/qualify_structural_workflows.py', '--output', str(output / 'public-workflow-consumer')], + ['python3', '-B', 'Scripts/qualify_structural_acceptance.py', '--output', str(output / 'structural-acceptance')], ['python3', '-B', 'Scripts/qualify_projections.py', '--output', str(output / 'projection-codegen')], ['python3', '-B', 'Scripts/verify_p2_fixtures.py'], ['python3', '-B', 'Scripts/qualify_p2_borrowing.py', '--output', str(output / 'p2-borrowing')], @@ -145,8 +127,6 @@ def portable(values): if lane == 'schema-examples-advanced': return commands('schema-examples-construction', ROOT / output) + [['python3', '-B', 'Scripts/qualify_schema_example_advanced.py', '--output', str(output / 'advanced')]] if lane in ('schema-examples-determinism', 'schema-examples-construction'): - # G2 changes only the opt-in generator. Reuse unchanged-core schema receipts; - # keep the broader schema-examples lane and every full release gate intact. baseline = commands('schema-examples', ROOT / output) return baseline[:4] + [ ['swift', 'test', '-c', 'release', '--jobs', '2', '--test-product', 'JSONSchemaExamplesTests'], @@ -184,8 +164,7 @@ def portable(values): return [['python3', '-B', 'Scripts/public_check.py'], ['swift', 'test', '--filter', 'navigation'], ['swift', 'test', '-c', 'release', '--filter', 'navigation'], - ['swift', 'build', '--target', 'JSONNavigation', '--scratch-path', '.build/navigation-strict', - '-Xswiftc', '-strict-memory-safety', '-Xswiftc', '-warnings-as-errors'], + ['swift', 'build', '--target', 'JSONNavigation', '--scratch-path', '.build/navigation-strict', '-Xswiftc', '-strict-memory-safety', '-Xswiftc', '-warnings-as-errors'], ['python3', '-B', 'Scripts/qualify_navigation.py', '--output', str(output / 'consumer')]] if lane == 'archive': return [['python3', '-B', 'Scripts/public_check.py'], @@ -206,14 +185,10 @@ def portable(values): commands('safety', ROOT / output)[1] + ['--disable-xctest'], commands('safety', ROOT / output)[2]] if lane == 'linux': - # Root behavior/corpora remain required. SwiftData and Apple SDK/lifetime - # probes are separate Apple acceptance; they never become Linux passes. return [c for c in commands('native', ROOT / output) if not any(x in c for x in ['Tools/MobileQualification', 'Scripts/qualify_release_1_1.py', 'Scripts/qualify_cli_ownership.py'])] + [ ['python3', '-B', 'Scripts/qualify_linux_reproduction.py', '--output', str(output / 'reproduction')]] if lane == 'linux-safety': - # Every behavior case uses Swift Testing. Do not start the generated - # zero-case XCTest runners, which leak in the pinned Linux runtime. return [['python3', '-B', 'Scripts/linux_runtime_diagnostics.py', '--output', str(output / 'runtime-diagnostics')]] + [ c + ['--disable-xctest'] if c[:2] == ['swift', 'test'] else c for c in commands('safety', ROOT / output)] @@ -238,10 +213,7 @@ def portable(values): ['python3', '-B', 'Scripts/qualify_html_browser.py', '--fixtures', str(output / 'fixtures'), '--output', str(output / 'browser')], ] if lane == 'cli': - return [['swift', 'build', '-c', 'release', '--product', 'swift-json'], ['swift', 'test', '--test-product', 'JSONCLITests', '--filter', 'cli|sharedFilterPlan'], ['python3', '-B', 'Scripts/qualify_cli.py', '--binary', '.build/release/swift-json', '--output', str(output / 'subprocess.json')], - ['python3', '-B', 'Scripts/qualify_cli_integration.py', '--binary', '.build/release/swift-json', '--output', str(output / 'integration-subprocess.json')], ['python3', '-B', 'Scripts/qualify_cli_limits.py', '--binary', '.build/release/swift-json', '--output', str(output / 'cli-limits.json')], ['swift', 'build', '-c', 'release', '--package-path', 'Tools/CLIQualification', '--product', 'CLIExactCompare'], ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C1', '--output', str(output / 'foundation-vectors.json')], ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C2', '--output', str(output / 'vectors.json')], - ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C3', '--output', str(output / 'arithmetic-vectors.json')], - ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C4', '--output', str(output / 'advanced-vectors.json')], ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C5', '--output', str(output / 'integration-vectors.json')], ['swift', 'run', '-c', 'release', '--package-path', 'Tools/CLIQualification', 'ProviderConsumer'], ['python3', '-B', 'Scripts/qualify_schema_providers.py', '--binary', '.build/release/swift-json', '--output', str(output / 'providers.json')]] + return [['swift', 'build', '-c', 'release', '--product', 'swift-json'], ['swift', 'test', '--test-product', 'JSONCLITests', '--filter', 'cli|sharedFilterPlan'], ['python3', '-B', 'Scripts/qualify_cli.py', '--binary', '.build/release/swift-json', '--output', str(output / 'subprocess.json')], ['python3', '-B', 'Scripts/qualify_cli_integration.py', '--binary', '.build/release/swift-json', '--output', str(output / 'integration-subprocess.json')], ['python3', '-B', 'Scripts/qualify_cli_limits.py', '--binary', '.build/release/swift-json', '--output', str(output / 'cli-limits.json')], ['swift', 'build', '-c', 'release', '--package-path', 'Tools/CLIQualification', '--product', 'CLIExactCompare'], ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C1', '--output', str(output / 'foundation-vectors.json')], ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C2', '--output', str(output / 'vectors.json')], ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C3', '--output', str(output / 'arithmetic-vectors.json')], ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C4', '--output', str(output / 'advanced-vectors.json')], ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C5', '--output', str(output / 'integration-vectors.json')], ['swift', 'run', '-c', 'release', '--package-path', 'Tools/CLIQualification', 'ProviderConsumer'], ['python3', '-B', 'Scripts/qualify_schema_providers.py', '--binary', '.build/release/swift-json', '--output', str(output / 'providers.json')]] if lane == 'native': return [ ['python3', '-B', 'Scripts/public_check.py'], @@ -255,7 +227,8 @@ def portable(values): ['python3', '-B', 'Scripts/qualify_cli.py', '--binary', '.build/release/swift-json', '--output', str(output / 'subprocess.json')], ['python3', '-B', 'Scripts/qualify_cli_integration.py', '--binary', '.build/release/swift-json', '--output', str(output / 'integration-subprocess.json')], ['python3', '-B', 'Scripts/qualify_cli_limits.py', '--binary', '.build/release/swift-json', '--output', str(output / 'cli-limits.json')], ['swift', 'build', '-c', 'release', '--package-path', 'Tools/CLIQualification', '--product', 'CLIExactCompare'], - ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C1', '--output', str(output / 'foundation-vectors.json')], ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C2', '--output', str(output / 'vectors.json')], + ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C1', '--output', str(output / 'foundation-vectors.json')], + ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C2', '--output', str(output / 'vectors.json')], ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C3', '--output', str(output / 'arithmetic-vectors.json')], ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C4', '--output', str(output / 'advanced-vectors.json')], ['python3', '-B', 'Scripts/qualify_cli_cases.py', '--binary', '.build/release/swift-json', '--comparator', 'Tools/CLIQualification/.build/release/CLIExactCompare', '--gate', 'C5', '--output', str(output / 'integration-vectors.json')], @@ -302,7 +275,7 @@ def portable(values): ['swift', 'test', '--package-path', 'Tools/MCPReadiness'], ['swift', 'run', '-c', 'release', '--package-path', 'Tools/MCPReadiness', 'MCPReadiness', str(output / 'consumer.json')], ['swift', 'test', '--package-path', 'Tools/Qualification'], - ['python3', '-B', 'Scripts/qualification_watchdog.py', '--timeout', '900', '--output', str(output / 'watchdog.json'), '--', 'swift', 'run', '-c', 'release', '--package-path', 'Tools/Qualification', 'Qualification', '--all', '--registry', 'Documentation/Qualification/FixtureRegistry.json', '--output', str(output / 'qualification.json')], + ['python3', '-B', 'Scripts/qualification_watchdog.py', '--timeout', '900', '--output', str(output / 'watchdog.json'), '--', 'swift', 'run', '-c', 'release', '--package-path', 'Tools/Qualification', 'Qualification', '--all', '--registry', 'Documentation/Qualification/FixureRegistry.json', '--output', str(output / 'qualification.json')], ] if lane == 'safety': return [ @@ -351,7 +324,7 @@ def check_result(command, code, log, events=None): acceptance_test = selected_filter in ('acceptance18', 'acceptance18SharedPlansKeepIndependentCrossProductHealth') prepared_test = selected_filter in ('navigationPreparedD0', 'navigationPreparedD0IndependentConcurrentHealth') if command[:2] == ['swift', 'test']: - expected = [DISCOVERY['composition_external_functions']] if 'Tools/CompositionQualification' in command else [12] if 'composition|composed|caughtOperationalFailureCannotComplete' in command else [79] if 'JSONSchemaTests' in command else [4 if 'thread' in command else 98] if 'JSONSchemaExamplesTests' in command else [88] if 'cli|sharedFilterPlan' in command else sorted(DISCOVERY['export_functions'].values()) if 'export|Export|apple' in command else [14] if 'Tools/WasmAudit' in command else sorted(DISCOVERY['qa_functions'].values()) if 'qa' in command else [7] if R0_TESTS in command else [3] if 'Tools/MCPReadiness' in command or 'Tools/MobileQualification' in command else [1] if 'Tools/Qualification' in command else sorted(SHARED_COUNTS.values()) if 'thread' in command else sorted(ROOT_COUNTS.values()) + expected = [DISCOVERY['composition_external_functions']] if 'Tools/CompositionQualification' in command else [12] if 'composition|composed|caughtOperationalFailureCannotComplete' in command else [79] if 'JSONSchemaTests' in command and 'thread' not in command else [12] if 'JSONSchemaExamplesTests' in command else [8] if 'qa' in command else [DISCOVERY['root_functions']['JSONInspectionTests']] if 'inspection|snapshot|projectionP|p2|exactProjectionPUB|graphG|store|workflow' in command else [DISCOVERY['root_functions']['JSONNavigationTests']] if 'navigation' in command else [12] if 'JSONSchemaExamplesTests' in command and events is not None: passed = passed and events['module_function_counts'] == {'JSONSchemaExamplesTests': expected[0]} and events['parameter_case_invocations'] == 0 if 'JSONSchemaTests' in command and events is not None: @@ -366,7 +339,7 @@ def check_result(command, code, log, events=None): elif selected_filter == 'binding': ids = {n for n in ids if '.binding' in n} expected = [len(ids)] - passed = passed and events is not None and events['module_function_counts'] == {'JSONTransformTests': len(ids)} and set(events['function_names']) == ids and len(events['function_names']) == len(ids) and events['parameter_case_invocations'] == 0 + passed = passed and events is not None and events['module_function_counts'] == {'JSONTransformTests': len(ids)} and set(events['function_names']) == ids and len(events['function_names']) == len(ids) if 'navigation' in command: expected = [DISCOVERY['root_functions']['JSONNavigationTests']] if events is not None: @@ -374,7 +347,7 @@ def check_result(command, code, log, events=None): if any(f in command for f in ['inspection', 'inspection|snapshot|projectionP|p2|exactProjectionPUB|graphG|store|workflow']): expected = [DISCOVERY['root_functions']['JSONInspectionTests']] if events is not None: - passed = passed and events['module_function_counts'] == {'JSONInspectionTests': expected[0]} and events['parameter_case_invocations'] == 65 and events['parameterized_functions'] == {'storeD1NativeVector(id:)': 65} and check_projection_cases(events) and check_p2_cases(events) and check_exact_cases(events) and check_graph_cases(events) and check_store_tests(events, log) + passed = passed and events['module_function_counts'] == {'JSONInspectionTests': expected[0]} and events['parameter_case_invocations'] == 65 if prepared_test: expected_ids = set(DISCOVERY['prepared_entry_function_ids']) if 'navigationPreparedD0IndependentConcurrentHealth' in command: @@ -391,28 +364,28 @@ def check_result(command, code, log, events=None): if selected_filter != 'workflow': expected_ids = {x for x in expected_ids if 'SharedImmutablePlans' in x} expected = [len(expected_ids)] - passed = passed and events is not None and events['module_function_counts'] == {'JSONInspectionTests': len(expected_ids)} and set(events['function_names']) == expected_ids and len(events['function_names']) == len(expected_ids) and events['parameter_case_invocations'] == 0 + passed = passed and events is not None and events['module_function_counts'] == {'JSONInspectionTests': len(expected_ids)} and set(events['function_names']) == expected_ids and len(events['function_names']) == len(expected_ids) if selected_filter == 'JSONIntegrationTests': expected_ids = set(DISCOVERY['integration_function_ids']) expected = [len(expected_ids)] - passed = passed and events is not None and events['module_function_counts'] == {'JSONIntegrationTests': len(expected_ids)} and set(events['function_names']) == expected_ids and len(events['function_names']) == len(expected_ids) and events['parameter_case_invocations'] == 0 + passed = passed and events is not None and events['module_function_counts'] == {'JSONIntegrationTests': len(expected_ids)} and set(events['function_names']) == expected_ids and len(events['function_names']) == len(expected_ids) if acceptance_test: expected_ids = set(DISCOVERY['structural_acceptance_function_ids']) if selected_filter != 'acceptance18': expected_ids = {x for x in expected_ids if 'SharedPlans' in x} expected = [len(expected_ids)] - passed = passed and events is not None and events['module_function_counts'] == {'JSONIntegrationTests': len(expected_ids)} and set(events['function_names']) == expected_ids and len(events['function_names']) == len(expected_ids) and events['parameter_case_invocations'] == 0 + passed = passed and events is not None and events['module_function_counts'] == {'JSONIntegrationTests': len(expected_ids)} and set(events['function_names']) == expected_ids and len(events['function_names']) == len(expected_ids) passed = passed and sorted(counts) == expected if 'Tools/CompositionQualification' in command and events is not None: passed = passed and events['module_function_counts'] == {'CompositionScaleTests': DISCOVERY['composition_external_functions']} and events['parameter_case_invocations'] == 0 if 'qa' in command and events is not None: passed = passed and events['module_function_counts'] == DISCOVERY['qa_functions'] and check_restored(events) - if 'thread' in command and 'JSONSchemaExamplesTests' not in command and 'JSONTransformTests' not in command and not prepared_test and not store_test and not workflow_test and not acceptance_test and events is not None: - passed = passed and events['module_function_counts'] == SHARED_COUNTS and check_projection_cases(events, shared=True) and check_p2_cases(events, shared=True) and check_exact_cases(events, shared=True) and check_graph_cases(events, shared=True) and events['parameter_case_invocations'] == 0 and check_store_tests(events, log, shared=True) + if 'thread' in command and 'JSONSchemaExamplesTests' not in command and 'JSONTransformTests' not in command and not prepared_test and not store_test and not workflow_test and not acceptance_test: + passed = passed and events is not None and events['module_function_counts'] == SHARED_COUNTS if '--filter' not in command and '--test-product' not in command and not any(x.startswith('Tools/') for x in command): if events is not None: - passed = passed and events['module_function_counts'] == ROOT_COUNTS and check_restored(events) and check_projection_cases(events) and check_p2_cases(events) and check_exact_cases(events) and check_graph_cases(events) and check_store_tests(events, log) - passed = passed and (events['parameterized_functions'] == ROOT_PARAMETER_CASES if events is not None else sum(int(n) for n in re.findall(r'with (\d+) test cases passed', log)) == PARAMETER_INVOCATIONS) + passed = passed and events['module_function_counts'] == ROOT_COUNTS and check_restored(events) + passed = passed and (events['parameterized_functions'] == ROOT_PARAMETER_CASES if events is not None else sum(int(n) for n in re.findall(r'with (\d+) test cases passed', log)) == 343) if 'Tools/Qualification' in command: probes = re.findall(r'^CAP_PROBE (\w+) (\S+)$', log, re.MULTILINE) matrix = json.loads((ROOT / 'Documentation/Qualification/LimitsMatrix.json').read_text()) @@ -423,128 +396,26 @@ def check_result(command, code, log, events=None): passed = data['observed'] == data['passed'] == 53 and data['failed'] == 0 if command[:3] == ['python3', '-B', 'Scripts/qualify_composition_scale.py'] and passed: data = json.loads(log.strip().splitlines()[-1]) - passed = data['status'] == 'passed' and (data['cells'], data['completed_imports'], data['operational_refusals']) == (8, 5, 3) and data['adoption_status'] == 'bounded_fixtures_only' and data['simple_strategy_capacity_status'] == 'blocked' and data['unmet_capacity_targets'] == ['balanced-128-128'] + passed = data['status'] == 'passed' and (data['cells'], data['completed_imports'], data['operational_refusals']) == (8, 5, 3) if command[:3] == ['python3', '-B', 'Scripts/qualify_cli_integration.py'] and passed: d = json.loads(log.strip().splitlines()[-1]) passed = d['status'] == 'passed' and (d['observed'], d['passed'], d['failed']) == (37, 37, 0) - if command[:3] == ['python3', '-B', 'Scripts/qualify_cli_limits.py'] and passed: - data = json.loads(log.strip().splitlines()[-1]) - passed = data['observed'] == data['passed'] == 33 and data['failed'] == 0 and data['metrics'] == 16 if command[:3] == ['python3', '-B', 'Scripts/qualify_schema_providers.py'] and passed: data = json.loads(log.strip().splitlines()[-1]) - passed = data['status'] == 'passed' and (data['observed'], data['passed'], data['failed']) == (37, 37, 0) and data['input_unchanged'] and data['static_no_network_client_or_model_session'] and len(data['frozen_inputs']) == 6 + passed = data['status'] == 'passed' and (data['observed'], data['passed'], data['failed']) == (37, 37, 0) if command[:3] == ['python3', '-B', 'Scripts/qualify_cli_cases.py'] and passed: data = json.loads(log.strip().splitlines()[-1]) expected = {'C1':12,'C2':38,'C3':17,'C4':38,'C5':6}[command[command.index('--gate')+1]] passed = data['observed'] == data['passed'] == expected and data['failed'] == data['policy'] == data['blocked'] == 0 - if command == ['swift', 'package', 'dump-package'] and passed: - d = json.loads(log) - passed = sorted(p['name'] for p in d['products']) == ['JSON', 'JSONBridges', 'JSONHTML', 'JSONInspection', 'JSONNavigation', 'JSONPatch', 'JSONSchema', 'JSONSchemaDSL', 'JSONSchemaExamples', 'JSONStreaming', 'JSONTransform', 'JSONValueDSL', 'swift-json'] and not d['dependencies'] and d['swiftLanguageVersions'] == ['6'] and d['toolsVersion']['_version'] == '6.4.0' and d['platforms'] == [{'options': [], 'platformName': 'macos', 'version': '15.0'}, {'options': [], 'platformName': 'ios', 'version': '18.0'}] - passed = passed and {p['name']: p['targets'] for p in d['products']} == dict({name: [name] for name in ['JSON', 'JSONSchema', 'JSONSchemaExamples', 'JSONStreaming', 'JSONPatch', 'JSONHTML', 'JSONBridges', 'JSONNavigation', 'JSONInspection', 'JSONTransform', 'JSONValueDSL', 'JSONSchemaDSL']}, **{'swift-json': ['JSONCLI']}) - targets = {t['name']: t for t in d['targets']} - passed = passed and set(targets) == {'JSON', 'JSONNavigation', 'JSONNavigationTests', 'JSONInspection', 'JSONInspectionTests', 'JSONIntegrationTests', 'JSONBridges', 'JSONBridgesTests', 'JSONSchema', 'JSONSchemaExamples', 'JSONSchemaExamplesTests', 'JSONStreaming', 'JSONPatch', 'JSONHTML', 'JSONRegexRuntime', 'JSONTests', 'JSONSchemaTests', 'JSONStreamingTests', 'JSONPatchTests', 'JSONHTMLTests', 'JSONRegexRuntimeTests', 'JSONFilterRuntime', 'JSONCLIEngine', 'JSONCLI', 'JSONCLITests', 'JSONTransform', 'JSONTransformTests', 'JSONDSLCore', 'JSONValueDSL', 'JSONSchemaDSL', 'JSONDSLTests'} and targets['JSONHTML']['dependencies'] == [{'byName': ['JSON', None]}] and targets['JSONBridges']['dependencies'] == [{'byName': ['JSON', None]}] and targets['JSONFilterRuntime']['dependencies'] == [{'byName': ['JSON', None]}] and targets['JSONCLI']['type'] == 'executable' - passed = passed and targets['JSONNavigation']['dependencies'] == [{'byName': ['JSON', None]}] and targets['JSONNavigationTests']['dependencies'] == [{'byName': ['JSONNavigation', None]}, {'byName': ['JSON', None]}] - passed = passed and targets['JSONSchemaExamples']['dependencies'] == [{'byName': ['JSON', None]}, {'byName': ['JSONSchema', None]}] and targets['JSON']['dependencies'] == [] and targets['JSONSchema']['dependencies'] == [{'byName': ['JSON', None]}, {'byName': ['JSONRegexRuntime', None]}] - passed = passed and targets['JSONTransform']['dependencies'] == [{'byName': ['JSON', None]}, {'byName': ['JSONFilterRuntime', None]}] and targets['JSONTransformTests']['dependencies'] == [{'byName': ['JSON', None]}, {'byName': ['JSONTransform', None]}, {'byName': ['JSONFilterRuntime', None]}] - passed = passed and targets['JSONInspection']['dependencies'] == [{'byName': ['JSON', None]}, {'byName': ['JSONNavigation', None]}] and targets['JSONInspectionTests']['dependencies'] == [{'byName': ['JSONInspection', None]}, {'byName': ['JSONNavigation', None]}, {'byName': ['JSON', None]}] - for name, dependencies in [('JSONDSLCore', ['JSON']), ('JSONValueDSL', ['JSON', 'JSONDSLCore']), ('JSONSchemaDSL', ['JSON', 'JSONSchema', 'JSONDSLCore']), ('JSONDSLTests', ['JSON', 'JSONSchema', 'JSONDSLCore', 'JSONValueDSL', 'JSONSchemaDSL'])]: - passed = passed and targets[name]['dependencies'] == [{'byName': [item, None]} for item in dependencies] - if command[:3] == ['python3', '-B', 'Scripts/qualify_dual_dsl.py']: - mode = command[command.index('--mode') + 1] - passed = passed and ('DUAL_DSL_COMPILER passed negatives=12 configurations=2 mode=' + mode) in log.splitlines() if command[:2] == ['swift', 'run'] and 'Examples/DualDSL' in command: passed = passed and 'DUAL_DSL_CONSUMER passed valid=1 invalid=2 operational=1 exact_keys=2 nested_arrays=1' in log.splitlines() - if command[:3] == ['python3', '-B', 'Scripts/qualify_schema_example_matrix.py']: - passed = passed and 'SCHEMA_EXAMPLES_MATRIX passed schemas=64 certified=512 invalid=64 processes=2' in log.splitlines() - if command[:3] == ['python3', '-B', 'Scripts/qualify_schema_example_lifetime.py']: - passed = passed and 'SCHEMA_EXAMPLES_CORPUS passed mode=owned scopes=128 values=6 exact=1' in log.splitlines() - if command[:3] == ['python3', '-B', 'Scripts/qualify_schema_example_advanced.py']: - passed = passed and 'SCHEMA_EXAMPLES_ADVANCED passed required=10 incomplete=8 profiles=2 seeds=4 per_process=144 processes=2 witnesses=155' in log.splitlines() - if command[:3] == ['python3', '-B', 'Scripts/qualify_schema_example_generation.py']: - passed = passed and 'SCHEMA_EXAMPLES_GENERATION passed cases=24 profiles=2 seeds=4 per_process=192 processes=2 proofs=2' in log.splitlines() - if command[:3] == ['python3', '-B', 'Scripts/qualify_schema_example_ownership.py']: - passed = passed and 'SCHEMA_EXAMPLES_OWNERSHIP passed positive=1 negatives=8' in log.splitlines() - if command[:3] == ['python3', '-B', 'Scripts/qualify_schema_examples.py']: - passed = passed and 'SCHEMA_EXAMPLES_COMPILER passed public_cases=15 negatives=2' in log.splitlines() and 'SCHEMA_EXAMPLES_DOCUMENTATION_COMPILER passed snippets=10 processes=2' in log.splitlines() - if command == ['swift', 'package', 'show-dependencies', '--format', 'json'] and passed: - passed = json.loads(log)['dependencies'] == [] - if command[:2] == ['swift', 'run'] and 'CompositionConsumer' in command: - passed = passed and bool(re.search(r'^COMPOSITION_CONSUMER passed positive=2 negative=4 work=[1-9][0-9]* peak=[1-9][0-9]* version=1$', log, re.MULTILINE)) - if command[:2] == ['swift', 'run'] and 'ProviderConsumer' in command: - passed = passed and 'PROVIDER_CONSUMER passed cases=14' in log.splitlines() - if command[:2] == ['swift', 'run'] and 'Tools/MobileQualification' in command: - passed = passed and all(line in log.splitlines() for line in ['PUBLIC_BEHAVIOR passed fixtures=8', 'STREAM_CANCELLATION passed', 'DISK_PERSISTENCE passed containers=3 rows=8', 'MOBILE_QUALIFICATION passed cases=3']) - if command[:3] == ['python3', '-B', 'Scripts/verify_testing_events.py']: - passed = passed and 'TESTING_EVENT_RECEIPTS passed cases=19' in log.splitlines() - if command[:3] == ['python3', '-B', 'Scripts/qualify_ergonomics.py']: - passed = passed and 'ERGONOMICS_COMPILER passed commands=8 guide_fences=3' in log.splitlines() - if command[:3] == ['python3', '-B', 'Scripts/qualify_public_graph.py']: - data = json.loads((ROOT / command[command.index('--output') + 1] / 'receipt.json').read_text()) - passed = passed and check_public_graph_receipt(data) - if command[:3] in (['python3', '-B', 'Scripts/qualify_graph_context.py'], ['python3', '-B', 'Scripts/qualify_graph.py']): - data = json.loads((ROOT / command[command.index('--output') + 1] / 'receipt.json').read_text()) - passed = passed and check_graph_receipt(data, context='qualify_graph_context.py' in command[2]) - if command[:3] == ['python3', '-B', 'Scripts/qualify_structural_acceptance.py']: - data = json.loads((ROOT / command[command.index('--output') + 1] / 'receipt.json').read_text()) - passed = passed and check_structural_acceptance(data) - if command[:3] == ['python3', '-B', 'Scripts/qualify_structural_workflows.py']: - data = json.loads((ROOT / command[command.index('--output') + 1] / 'receipt.json').read_text()) - passed = passed and check_workflow_consumer(data) - if command[:3] == ['python3', '-B', 'Scripts/qualify_store.py']: - data = json.loads((ROOT / command[command.index('--output') + 1] / 'receipt.json').read_text()) - passed = passed and check_store_consumer(data) - if command[:3] == ['python3', '-B', 'Scripts/qualify_store_preparation.py']: - data = json.loads((ROOT / command[command.index('--output') + 1] / 'receipt.json').read_text()) - probes = [x for x in data['commands'] if x.get('expected') is not None] - passed = passed and data['status'] == 'passed' and len(data['commands']) == 44 and len(probes) == 38 - passed = passed and all(x['passed'] and not x['compiler_crash'] for x in data['commands']) - if command[:3] == ['python3', '-B', 'Scripts/qualify_navigation_reducer.py']: - passed = passed and 'REDUCER_CODEGEN passed positive=2 negative=20' in log.splitlines() - if command[:3] == ['python3', '-B', 'Scripts/qualify_snapshot.py']: - passed = passed and 'SNAPSHOT_COMPILER passed configurations=2 consumer_cases=14 address_cases=14 guide_fences=3 negative_probes=18' in log.splitlines() - if command[:3] == ['python3', '-B', 'Scripts/qualify_p2_borrowing.py']: - data = json.loads((ROOT / command[command.index('--output') + 1] / 'receipt.json').read_text()) - passed = passed and check_borrow_receipt(data) - if command[:3] == ['python3', '-B', 'Scripts/verify_p2_fixtures.py']: - passed = passed and 'P2_FIXTURES passed pinned=20 matrix_shapes=9 set_oracles=4 design_only=6' in log.splitlines() - if command[:3] == ['python3', '-B', 'Scripts/qualify_exact_projections.py']: - data = json.loads((ROOT / command[command.index('--output') + 1] / 'receipt.json').read_text()) - passed = passed and check_exact_receipt(data) - if command[:3] == ['python3', '-B', 'Scripts/qualify_p2.py']: - data = json.loads((ROOT / command[command.index('--output') + 1] / 'receipt.json').read_text()) - passed = passed and check_private_receipt(data) - if command[:3] == ['python3', '-B', 'Scripts/qualify_public_projections.py']: - data = json.loads((ROOT / command[command.index('--output') + 1] / 'receipt.json').read_text()) - passed = passed and check_public_projection_receipt(data) - if command[:3] == ['python3', '-B', 'Scripts/qualify_projections.py']: - data = json.loads((ROOT / command[command.index('--output') + 1] / 'receipt.json').read_text()) - probes = [c for c in data['commands'] if c.get('expected_diagnostic')] - passed = data['status'] == 'passed' and len(data['commands']) == 24 and len(probes) == 14 - passed = passed and all(c['status'] == 'passed' and not c['compiler_crash'] for c in data['commands']) - passed = passed and all(c['exit_code'] > 0 and any(c['expected_diagnostic'].lower() in d.lower() for d in c['primary_diagnostics']) for c in probes) - if command[:3] == ['python3', '-B', 'Scripts/qualify_inspection.py']: - passed = passed and 'INSPECTION_COMPILER passed configurations=2 consumer_cases=12 address_cases=12 guide_fences=3 negative_probes=8' in log.splitlines() - if command[:3] == ['python3', '-B', 'Scripts/qualify_navigation.py']: - passed = passed and 'NAVIGATION_COMPILER passed configurations=2 consumer_cases=18 negative_probes=8' in log.splitlines() - if command[:3] == ['python3', '-B', 'Scripts/verify_html_assets.py']: - passed = passed and 'JSONHTML_ASSETS passed count=2' in log.splitlines() - if 'BrowserFixtures' in command: - passed = passed and 'JSONHTML_BROWSER_FIXTURES passed count=15' in log.splitlines() - if command[:3] == ['python3', '-B', 'Scripts/qualify_html_browser.py']: - passed = passed and 'JSONHTML_BROWSER passed engines=3 cases=76 skips=0' in log.splitlines() - if 'Scripts/test_qualification_harness.py' in command: + if command[:3] == ['python3', '-B', 'Scripts/test_qualification_harness.py']: passed = passed and 'HARNESS_TESTS passed cases=35' in log.splitlines() - if command[:3] == ['python3', '-B', 'Scripts/qualify_transform_selection.py']: - data = json.loads(log.strip().splitlines()[-1]) - passed = passed and data['status'] == 'passed' and len(data['commands']) == 20 and all(x['passed'] for x in data['commands']) and data['witnesses_per_configuration'] == 10 and data['negative_fixture_invocations'] == 14 and data['compiled_guide_fences'] == 1 - if command[:3] == ['python3', '-B', 'Scripts/qualify_transform_bindings.py']: - data = json.loads(log.strip().splitlines()[-1]) - passed = passed and data['status'] == 'passed' and len(data['commands']) == 8 and all(x['passed'] for x in data['commands']) and data['witnesses_per_configuration'] == 6 and data['negative_fixture_invocations'] == 4 and data['compiled_guide_fences'] == 1 return passed, counts def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--lane', choices=['transform-bindings', 'transform-bindings-safety', 'structural-acceptance', 'structural-acceptance-safety', 'structural-workflows', 'structural-workflows-safety', 'store', 'store-preparation', 'composition', 'navigation', 'inspection', 'native', 'safety', 'docs', 'r0', 'qa', 'linux', 'linux-safety', 'wasm', 'browser', 'export', 'cli', 'schema-examples', 'schema-examples-safety', 'schema-examples-determinism', 'schema-examples-construction', 'schema-examples-advanced', 'schema-examples-qualification', 'coverage', 'linux-coverage', 'linux-qa', 'wasm-qa', 'archive'], required=True) + parser.add_argument('--lane', choices=['transform-bindings', 'transform-bindings-safety', 'structural-acceptance', 'structural-acceptance-safety', 'structural-workflows', 'structural-workflows-safety', 'store', 'store-preparation', 'inspection', 'schema-examples-qualification', 'schema-examples-advanced', 'schema-examples-determinism', 'schema-examples-construction', 'schema-examples-safety', 'schema-examples', 'composition', 'navigation', 'archive', 'coverage', 'linux-coverage', 'wasm', 'wasm-qa', 'linux-qa', 'linux', 'linux-safety', 'qa', 'export', 'browser', 'cli', 'native', 'safety', 'r0']) parser.add_argument('--output') parser.add_argument('--skip-host-measurements', action='store_true', help='For WASI runtime qualification only: record optional host timings as NOT RUN.') args = parser.parse_args() @@ -556,8 +427,8 @@ def main(): repository = subprocess.run(['git', 'rev-parse', '--show-toplevel'], cwd=ROOT, capture_output=True, text=True) is_checkout = repository.returncode == 0 and pathlib.Path(repository.stdout.strip()).resolve() == ROOT.resolve() commit = subprocess.run(['git', 'rev-parse', 'HEAD'], cwd=ROOT, capture_output=True, text=True) if is_checkout else None - environment = collect_linux() if args.lane.startswith('linux') or (args.lane in ('transform-bindings', 'transform-bindings-safety', 'composition', 'inspection', 'structural-workflows', 'structural-workflows-safety', 'structural-acceptance', 'structural-acceptance-safety') and sys.platform == 'linux') else collect() - receipt = {'schema_version': 1, 'lane': args.lane, 'status': 'blocked', 'environment': environment, 'started_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'git_commit': commit.stdout.strip() if commit is not None and commit.returncode == 0 else None, 'github_run_id': os.environ.get('GITHUB_RUN_ID'), 'source_inputs': inputs, 'source_manifest_sha256': hashlib.sha256(json.dumps(inputs, sort_keys=True).encode()).hexdigest(), 'commands': []} + environment = collect_linux() if args.lane.startswith('linux') or (args.lane in ('transform-bindings', 'transform-bindings-safety', 'composition', 'inspection', 'structural-workflows', 'structural-workflows-safety', 'store', 'store-preparation', 'navigation', 'coverage')) else collect() + receipt = {'schema_version': 1, 'lane': args.lane, 'status': 'blocked', 'environment': environment, 'started_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'git_commit': commit.stdout.strip() if commit else None, 'commands': []} def save(): (output / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') save() @@ -595,11 +466,7 @@ def save(): except (OSError, KeyError, ValueError, TypeError) as error: passed, counts = False, [] event_error = str(error) - entry = {'command': [x.replace(str(ROOT), '.') for x in command], 'exit_code': code, 'status': 'passed' if passed else 'failed', 'log': log_path.relative_to(ROOT).as_posix(), 'log_sha256': hashlib.sha256(log_path.read_bytes()).hexdigest(), 'swift_testing_function_counts': counts, 'parameterized_case_invocations': sum(int(n) for n in re.findall(r'with (\d+) test cases passed', log))} - if events is not None: - entry['console_parameterized_case_invocations'] = entry['parameterized_case_invocations'] - entry['parameterized_case_invocations'] = events['parameter_case_invocations'] - entry['swift_testing_events'] = dict(events, path=event_path.relative_to(ROOT).as_posix(), sha256=hashlib.sha256(event_path.read_bytes()).hexdigest()) + entry = {'command': [x.replace(str(ROOT), '.') for x in command], 'exit_code': code, 'status': 'passed' if passed else 'failed', 'log': log_path.relative_to(ROOT).as_posix(), 'log_sha256': hashlib.sha256(log_path.read_bytes()).hexdigest(), 'counts': counts} if event_error is not None: entry['event_error'] = event_error receipt['commands'].append(entry) @@ -610,49 +477,6 @@ def save(): save() else: receipt['status'] = 'passed' - if receipt['status'] == 'passed' and args.lane in ('native', 'linux'): - try: - receipt['materialized_outcomes'] = materialized_outcomes(output) - except (OSError, KeyError, ValueError, TypeError) as error: - receipt['status'] = 'failed' - receipt['evidence_error'] = str(error) - print('Native evidence failed: ' + str(error), flush=True) - if receipt['status'] == 'passed' and args.lane in ('wasm', 'wasm-qa'): - try: - wasi = json.loads((output / 'probes/wasi.json').read_text()) - if wasi['status'] != 'passed' or wasi['root_counts'] != DISCOVERY['wasi_root_functions']: - raise ValueError('Missing WASI qualification boundary') - if args.lane == 'wasm-qa' and (wasi.get('scope') != 'root-qa' or wasi.get('qa_counts') != DISCOVERY['qa_functions']): - raise ValueError('Missing scoped WASI QA boundary') - receipt['materialized_wasi'] = {'path':'probes/wasi.json','sha256':hashlib.sha256((output / 'probes/wasi.json').read_bytes()).hexdigest()} - except (OSError,KeyError,ValueError,TypeError) as error: - receipt['status'] = 'failed'; receipt['evidence_error'] = str(error) - if receipt['status'] == 'passed' and args.lane in ('coverage', 'linux-coverage'): - try: - coverage = json.loads((output / 'export/coverage.json').read_text()) - if coverage['status'] != 'passed' or coverage['root_counts'] != ROOT_COUNTS or coverage['parameterized_invocations'] != PARAMETER_INVOCATIONS or coverage['omitted_binary_probe']['status'] != 'rejected' or set(coverage['modules']) != set(DISCOVERY['coverage_modules']): - raise ValueError('Missing complete current-module coverage boundary') - receipt['materialized_coverage'] = {'path': 'export/coverage.json', 'sha256': hashlib.sha256((output / 'export/coverage.json').read_bytes()).hexdigest()} - except (OSError, KeyError, ValueError, TypeError) as error: - receipt['status'] = 'failed'; receipt['evidence_error'] = str(error) - if receipt['status'] == 'passed' and args.lane == 'archive': - try: - reproduction = json.loads((output / 'reproduction/reproduction.json').read_text()) - native_path = output / 'reproduction/native/receipt.json' - native = json.loads(native_path.read_text()) - candidate = output / 'reproduction/candidate.tar.gz' - if (reproduction['status'] != 'passed' or not reproduction['outside_git_verified'] - or not reproduction['byte_equal_rebuild'] or not reproduction['byte_equal_extracted_rebuild'] - or reproduction['archive_sha256'] != hashlib.sha256(candidate.read_bytes()).hexdigest() - or reproduction['native_receipt_sha256'] != hashlib.sha256(native_path.read_bytes()).hexdigest() - or native['status'] != 'passed' or native['git_commit'] is not None - or not native['source_inputs_unchanged'] or native['source_inputs'] != inputs - or reproduction['native_commands'] != len(commands('native', output))): - raise ValueError('Missing full source-bound Git-free reproduction') - receipt['materialized_reproduction'] = {'path': 'reproduction/reproduction.json', - 'sha256': hashlib.sha256((output / 'reproduction/reproduction.json').read_bytes()).hexdigest()} - except (OSError, KeyError, ValueError, TypeError) as error: - receipt['status'] = 'failed'; receipt['evidence_error'] = str(error) receipt['source_inputs_unchanged'] = snapshot() == inputs if not receipt['source_inputs_unchanged']: receipt['status'] = 'failed' diff --git a/Scripts/test_qualification_harness.py b/Scripts/test_qualification_harness.py index b41bea0..a8d1ec4 100644 --- a/Scripts/test_qualification_harness.py +++ b/Scripts/test_qualification_harness.py @@ -1,602 +1,3 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: Apache-2.0 -"""Negative coverage/discovery harness tests; never substitutes for Swift behavior.""" -import copy -import json -import pathlib -import os -import re -import subprocess -from qualify_release_1_1 import checkout_head -from generate_cli_identity import rendered -import tempfile -import unittest -from unittest.mock import patch -import documentation_catalog -from coverage import executable_objects, shared_objects, validate_export, without_schema_objects -from test_discovery import DISCOVERY, ROOT_COUNTS, PARAMETER_INVOCATIONS, check_root, check_restored, RESTORED_FUNCTIONS -from ci import check_result, commands, SHARED_TESTS, SHARED_COUNTS -from p2_acceptance import PUBLIC_FUNCTIONS, PUBLIC_SHARED, check_exact_cases, check_exact_receipt -from p2_acceptance import EXPECTED as P2_FUNCTIONS, SHARED as P2_SHARED, BORROW_CHECKS, check_p2_cases, check_borrow_receipt, check_private_receipt -from graph_acceptance import EXPECTED as GRAPH_FUNCTIONS, check_graph_cases, check_graph_receipt, check_public_graph_receipt, PUBLIC_CHECKS as GRAPH_PUBLIC_CHECKS -from store_acceptance import FUNCTIONS as STORE_FUNCTIONS, VECTORS, check_store_tests, check_store_consumer -from projection_acceptance import EXPECTED as PROJECTION_FUNCTIONS, PUBLIC_CHECKS, check_projection_cases, check_public_projection_receipt - - -def store_markers(): - return '\n'.join(['STORE_VECTOR %s passed driver=%s phase=%s kind=%s' % - (x['id'],x['execution_kind'],x['expected']['phase'],x['expected']['kind']) - for x in VECTORS if x['execution_kind'] in ('native-build','strict-parser')]+[ - 'STORE_RUNTIME R%02d passed' % n for n in range(1,11)]) - -class HarnessTests(unittest.TestCase): - def workflow_jobs(self, filename): - text = (pathlib.Path(__file__).resolve().parents[1]/'.github/workflows'/filename).read_text() - # Inspect this repository's authored job blocks, not a general YAML parser. - jobs = dict(re.findall(r'^ (\w+):\n(.*?)(?=^ \w+:\n|\Z)', text.split('jobs:\n',1)[1], re.M|re.S)) - return text, jobs - - def test_required_workflow_contexts_wait_for_complete_execution_matrices(self): - for filename, execution, required, name in [ - ('native.yml','[native, safety, docs, archive, composition]','[native, safety, docs, archive]','Native / ${{ matrix.lane }}'), - ('linux.yml','[linux, linux-safety, composition]','[linux, linux-safety]','Linux / ${{ matrix.architecture }} / ${{ matrix.lane }}'), - ]: - with self.subTest(workflow=filename): - _, jobs = self.workflow_jobs(filename) - self.assertIn('lane: '+execution, jobs['qualification']) - self.assertIn("run: python3 -B Scripts/ci.py --lane '${{ matrix.lane }}'", jobs['qualification']) - self.assertNotIn("if: matrix.lane == 'composition'", jobs['qualification']) - self.assertNotIn('--lane composition', jobs['qualification']) - gate = jobs['required'] - self.assertIn('name: '+name+'\n', gate) - self.assertIn('needs: qualification\n', gate) - self.assertIn('if: ${{ always() }}\n', gate) - self.assertIn('lane: '+required, gate) - if filename=='linux.yml': - for block in [jobs['qualification'], gate]: - self.assertIn('architecture: [x86_64, arm64]', block) - - def test_workflow_gates_reject_failed_cancelled_skipped_and_missing_results(self): - for filename in ['native.yml','linux.yml']: - text, jobs = self.workflow_jobs(filename) - self.assertNotIn('continue-on-error', text) - gate = jobs['required'] - self.assertIn('QUALIFICATION_RESULT: ${{ needs.qualification.result }}', gate) - script = re.findall(r'^ run: (.+)$', gate, re.M) - self.assertEqual(script, ['test "$QUALIFICATION_RESULT" = success']) - for result in ['success','failure','cancelled','skipped','', 'Success','success failure']: - run = subprocess.run(['bash','-c',script[0]],env=dict(os.environ,QUALIFICATION_RESULT=result),capture_output=True) - self.assertEqual(run.returncode==0, result=='success', (filename,result)) - env = dict(os.environ); env.pop('QUALIFICATION_RESULT',None) - self.assertNotEqual(subprocess.run(['bash','-c',script[0]],env=env,capture_output=True).returncode,0) - - def test_composition_scheduling_preserves_timeouts_pins_artifacts_and_pages_guard(self): - for filename in ['native.yml','linux.yml']: - text, jobs = self.workflow_jobs(filename) - self.assertIn('timeout-minutes: 60', jobs['qualification']) - self.assertIn('fail-fast: false', jobs['qualification']) - self.assertIn('persist-credentials: false', jobs['qualification']) - self.assertIn('retention-days: 30', jobs['qualification']) - self.assertIn('if-no-files-found: error', jobs['qualification']) - self.assertIn('actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803', text) - self.assertIn('actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f', text) - self.assertIn("branches: [main, 'release/1.6']", text) - _, native = self.workflow_jobs('native.yml') - self.assertIn("if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'", native['publish']) - self.assertIn('needs: [qualification, required]', native['publish']) - _, linux = self.workflow_jobs('linux.yml') - self.assertNotIn('publish', linux) - self.assertIn('swift@sha256:31d14d727f4451f25e29bac9c42563a098c39ac66415e8bad970f48038e38aca', linux['qualification']) - - def test_selection_filter_preserves_exact_identity_and_completion_evidence(self): - ids = [n for n in DISCOVERY['transform_function_ids'] if '.selection' in n] - self.assertEqual(len(ids), 18) - command = ['swift','test','--test-product','JSONTransformTests','--filter','selection'] - events = dict(bundle_counts=[18],module_function_counts={'JSONTransformTests':18},function_names=ids,parameter_case_invocations=0) - self.assertTrue(check_result(command,0,'',events)[0]) - self.assertFalse(check_result(command,0,'',dict(events,function_names=ids[:-1]))[0]) - consumer = ['python3','-B','Scripts/qualify_transform_selection.py'] - receipt = dict(status='passed',commands=[dict(passed=True)]*20,witnesses_per_configuration=10,negative_fixture_invocations=14,compiled_guide_fences=1) - self.assertTrue(check_result(consumer,0,json.dumps(receipt))[0]) - for broken in [dict(receipt,commands=receipt['commands'][:-1]),dict(receipt,negative_fixture_invocations=0),dict(receipt,compiled_guide_fences=0)]: - self.assertFalse(check_result(consumer,0,json.dumps(broken))[0]) - - def test_transform_binding_discovery_rejects_incomplete_duplicate_and_foreign_events(self): - ids = DISCOVERY['transform_function_ids'] - command = ['swift', 'test', '--test-product', 'JSONTransformTests'] - events = dict(bundle_counts=[59], module_function_counts={'JSONTransformTests':59}, - function_names=ids, parameter_case_invocations=0) - self.assertTrue(check_result(command, 0, '', events)[0]) - for broken in [dict(events,function_names=ids[:-1]),dict(events,function_names=ids+ids[:1]), - dict(events,module_function_counts={'JSONTests':59}),dict(events,parameter_case_invocations=1)]: - self.assertFalse(check_result(command, 0, '', broken)[0]) - shared = [n for n in ids if 'bindingShared' in n or 'selectionShared' in n or 'jtxPlanIsSendableAndExecutionsHaveIndependentHealth' in n] - thread = command + ['--sanitize', 'thread'] - self.assertTrue(check_result(thread, 0, '', dict(events,bundle_counts=[4],module_function_counts={'JSONTransformTests':4},function_names=shared))[0]) - - def test_acceptance18_discovery_preserves_exact_public_pipeline_identities(self): - names = DISCOVERY['structural_acceptance_function_ids'] - self.assertEqual(len(names), 9) - self.assertEqual(len(DISCOVERY['integration_function_ids']), 43) - events = dict(bundle_counts=[9], module_function_counts={'JSONIntegrationTests':9}, - function_names=names, parameter_case_invocations=0) - command = ['swift','test','--test-product','JSONIntegrationTests','--filter','acceptance18'] - self.assertTrue(check_result(command, 0, '', events)[0]) - full = dict(events, bundle_counts=[43], module_function_counts={'JSONIntegrationTests':43}, function_names=DISCOVERY['integration_function_ids']) - full_command = ['swift','test','--test-product','JSONIntegrationTests','--filter','JSONIntegrationTests'] - self.assertTrue(check_result(full_command, 0, '', full)[0]) - self.assertFalse(check_result(full_command, 0, '', dict(full,function_names=full['function_names'][:-1]))[0]) - for broken in [dict(events, function_names=names[:-1]), dict(events, function_names=names+names[:1]), - dict(events, module_function_counts={'JSONInspectionTests':9}), dict(events, parameter_case_invocations=1)]: - self.assertFalse(check_result(command, 0, '', broken)[0]) - - def test_capacity18_does_not_convert_refusals_or_incomplete_boundaries_to_passes(self): - from qualify_structural_acceptance import MATRIX, validate_capacity, validate_fixtures - cells = [] - for identifier in MATRIX['cells']: - operation, count = identifier.rsplit('-',1); count=int(count) - cells.append(dict(id=identifier, operation=operation, requests=count, - fixture='chain-%d.json' % count if operation=='follow-chain' else 'fixed-eight.json', - outcome='completed', report=dict(work=1,nodes=1,retained=32,peak=32), - independentSingles=None if operation=='follow-chain' else dict(workSum=count,nodesSum=count,maximumInvocationRetained=32,maximumInvocationPeak=32), - requestBoundary=True,workBoundary=True,retentionBoundary=True)) - data=dict(schemaVersion=1,profile='balanced',cells=cells) - self.assertTrue(validate_capacity(data)) - refused=copy.deepcopy(data);refused['cells'][-1]=dict(refused['cells'][-1],outcome='operational_error',report=None, - requestBoundary=None,workBoundary=None,retentionBoundary=None,errorMetric='workUnits',errorLimit=1,errorAttempted=2,errorStage='construction') - self.assertTrue(validate_capacity(refused)) - for mode in ['missing','duplicate','fake-boundary','bad-single-count','required-refusal','fake-metric','refusal-as-success','missing-report']: - bad=copy.deepcopy(refused) - if mode=='missing':bad['cells'].pop() - elif mode=='duplicate':bad['cells'][-1]=bad['cells'][0] - elif mode=='fake-boundary':bad['cells'][0]['workBoundary']=False - elif mode=='bad-single-count':bad['cells'][0]['independentSingles']['nodesSum']=2 - elif mode=='required-refusal':bad['cells'][3]=dict(bad['cells'][-1],id='follow-chain-1',requests=1,fixture='chain-1.json') - elif mode=='fake-metric':bad['cells'][-1]['errorMetric']='unrelated' - elif mode=='refusal-as-success':bad['cells'][-1]['outcome']='completed' - else:bad['cells'][0]['report']=None - self.assertFalse(validate_capacity(bad)) - with tempfile.TemporaryDirectory() as temporary: - directory=pathlib.Path(temporary) - with self.assertRaisesRegex(ValueError,'inventory'):validate_fixtures(directory) - for fixture in MATRIX['fixtures']:(directory/fixture['name']).write_bytes(b'corrupt') - with self.assertRaisesRegex(ValueError,'bytes'):validate_fixtures(directory) - - def test_workflow_discovery_requires_all_exact_identities(self): - names = DISCOVERY['workflow_function_ids'] - self.assertEqual(len(names), 32) - events = dict(bundle_counts=[32], module_function_counts={'JSONInspectionTests':32}, - function_names=names, parameter_case_invocations=0) - command = ['swift','test','--test-product','JSONInspectionTests','--filter','workflow'] - self.assertTrue(check_result(command, 0, '', events)[0]) - for broken in [dict(events, function_names=names[:-1]), dict(events, function_names=names+names[:1]), - dict(events, parameter_case_invocations=1), dict(events, bundle_counts=[21])]: - self.assertFalse(check_result(command, 0, '', broken)[0]) - - def test_workflow_consumer_rejects_missing_witnesses_or_incidental_diagnostics(self): - from qualify_structural_workflows import CASES, PROBES, check_receipt - data = dict(status='passed', commands=[dict(name=c+'-'+name, passed=True, compiler_crash=False, - exit=1 if name in PROBES else 0, expected=PROBES[name][1] if name in PROBES else None, - primary_diagnostics=[PROBES[name][1]] if name in PROBES else []) - for c in ['debug','release'] for name in ['consumer','binpath','guide',*PROBES]], - consumer_results={c:dict(cases=sorted(CASES),work_units=1,logical_retained_bytes=1) for c in ['debug','release']}) - self.assertTrue(check_receipt(data)) - for mode in ['missing-command','missing-case','incidental','crash']: - bad=copy.deepcopy(data) - if mode == 'missing-command': bad['commands'].pop() - elif mode == 'missing-case': bad['consumer_results']['release']['cases'].pop() - elif mode == 'incidental': bad['commands'][3]['primary_diagnostics']=['unrelated error'] - else: bad['commands'][3]['compiler_crash']=True - self.assertFalse(check_receipt(bad)) - - def test_scoped_catalog_check_cannot_weaken_the_full_site_inventory(self): - with tempfile.TemporaryDirectory() as temporary: - root = pathlib.Path(temporary).resolve() - source = root / 'Sources/JSONNavigation/Probe.swift' - source.parent.mkdir(parents=True); source.write_text('public struct Probe {}\n') - graphs = root / 'graphs'; graphs.mkdir() - symbol = dict(location=dict(uri=source.as_uri()), kind=dict(identifier='swift.struct'), - accessLevel='public', identifier=dict(precise='probe'), names=dict(title='Probe'), - pathComponents=['Probe'], docComment=dict(lines=[dict(text='A scoped probe.')])) - graph = graphs / 'JSONNavigation.symbols.json' - graph.write_text(json.dumps(dict(module=dict(name='JSONNavigation'), symbols=[symbol]))) - with patch.object(documentation_catalog, 'ROOT', root): - with self.assertRaisesRegex(ValueError, 'No authored symbols for JSON'): - documentation_catalog.public_symbols(graphs) - with patch.object(documentation_catalog, 'CATALOGS', ('JSONNavigation',)): - symbols = documentation_catalog.public_symbols(graphs) - self.assertEqual(len(symbols), 1) - self.assertEqual(documentation_catalog.verify_comments(symbols)['errors'], []) - symbol['accessLevel'] = 'package' - graph.write_text(json.dumps(dict(module=dict(name='JSONNavigation'), symbols=[symbol]))) - with self.assertRaisesRegex(ValueError, 'Non-public symbol'): - documentation_catalog.public_symbols(graphs) - self.assertEqual(len(documentation_catalog.CATALOGS), 13) - - def test_store_requires_exact_engine_vectors_runtime_and_compiler_primary_diagnostics(self): - events = dict(function_names=sorted(STORE_FUNCTIONS), module_function_counts={'JSONInspectionTests':12}, - parameterized_functions={'storeD1NativeVector(id:)':65}, parameter_case_invocations=65) - log = store_markers() - self.assertTrue(check_store_tests(events, log, scoped=True)) - for broken in [log.replace('D001','D999'), log+'\n'+log.splitlines()[0], - log.replace('driver=native-build','driver=python-premise'), log.replace('R10 passed','R10 skipped')]: - self.assertFalse(check_store_tests(events, broken, scoped=True)) - self.assertFalse(check_store_tests(dict(events,function_names=sorted(STORE_FUNCTIONS)[:-1]),log)) - self.assertFalse(check_store_tests(dict(events,parameterized_functions={'storeD1NativeVector(id:)':64}),log,scoped=True)) - valid = dict(status='passed',commands=[dict(name=str(i),passed=True,compiler_crash=False, - expected='inaccessible' if i<16 else None,exit=1 if i<16 else 0, - primary_diagnostics=['inaccessible'] if i<16 else []) for i in range(26)], - vector_compiler_results=[dict(id=i,configuration=c,driver='profile-schema/compiler-negative',passed=True) - for i in ['D051','D053','D069'] for c in ['debug','release']]) - self.assertTrue(check_store_consumer(valid)) - for mode in ['missing','incidental','crash','zero','vector']: - bad=copy.deepcopy(valid) - if mode=='missing': bad['commands'].pop() - elif mode=='incidental': bad['commands'][0]['primary_diagnostics']=['unrelated error'] - elif mode=='crash': bad['commands'][0]['compiler_crash']=True - elif mode=='zero': bad['commands'][0]['exit']=0 - else: bad['vector_compiler_results'].pop() - self.assertFalse(check_store_consumer(bad)) - - def test_public_projection_receipt_rejects_incomplete_and_incidental_successes(self): - valid = dict(status='passed', commands=[dict(name=str(i), status='passed', - compiler_crash=False, expected_diagnostic='inaccessible' if i < 18 else None, - primary_diagnostics=['inaccessible'] if i < 18 else [], exit_code=1 if i < 18 else 0) - for i in range(37)], consumer_results={config: dict(cases=sorted(PUBLIC_CHECKS), - work_units=1, logical_retained_bytes=1, peak_logical_retained_bytes=1, admitted_nodes=1) - for config in ['debug', 'release', 'address']}) - self.assertTrue(check_public_projection_receipt(valid)) - broken = copy.deepcopy(valid); broken['commands'].pop() - self.assertFalse(check_public_projection_receipt(broken)) - broken = copy.deepcopy(valid); broken['commands'][0]['primary_diagnostics'] = ['unrelated warning'] - self.assertFalse(check_public_projection_receipt(broken)) - broken = copy.deepcopy(valid); broken['commands'][0]['compiler_crash'] = True - self.assertFalse(check_public_projection_receipt(broken)) - broken = copy.deepcopy(valid); broken['commands'][0]['exit_code'] = 0 - self.assertFalse(check_public_projection_receipt(broken)) - for config in ['debug', 'release', 'address']: - for mode in ['missing', 'duplicate', 'renamed', 'zero-resource']: - broken = copy.deepcopy(valid); row = broken['consumer_results'][config] - if mode == 'missing': row['cases'].pop() - elif mode == 'duplicate': row['cases'][0] = row['cases'][1] - elif mode == 'renamed': row['cases'][0] = 'unexpected' - else: row['work_units'] = 0 - self.assertFalse(check_public_projection_receipt(broken)) - - def test_full_thread_filter_is_not_misclassified_as_preparation_subset(self): - names = [module+'.fixture'+str(i)+'()' for module,count in SHARED_COUNTS.items() for i in range(count)] - names = [n for n in names if not n.startswith('JSONInspectionTests.')]+[ - 'JSONInspectionTests.inspectionI17CoverageSpansReplacementAndSharedInvocations()', - 'JSONInspectionTests.snapshotS19NavigationParityAndIndependentInvocations()', - 'JSONInspectionTests.snapshotV22ConcurrentLookupsUseIndependentAdmissionAndHealth()', - 'JSONInspectionTests.projectionP16SharedViewsHaveIndependentHealthAndLegacyStatistics()', - 'JSONInspectionTests.storeR06TerminalQuotasNeverPublishAndImmutableConcurrentHealth()', *sorted(P2_SHARED | PUBLIC_SHARED), 'JSONInspectionTests.graphG30ConcurrentViewsHaveIndependentHealth()', 'JSONInspectionTests.graphGP14ConcurrentQueries()'] - events=dict(function_names=names,module_function_counts=SHARED_COUNTS, - bundle_counts=sorted(SHARED_COUNTS.values()),parameter_case_invocations=0,parameterized_functions={}) - command=['swift','test','--sanitize','thread','--filter',SHARED_TESTS] - self.assertTrue(check_result(command,0,'STORE_RUNTIME R06 passed',events)[0]) - for bad in [dict(events,parameter_case_invocations=1),dict(events,function_names=names[:-1]), - dict(events,module_function_counts=dict(SHARED_COUNTS,JSONNavigationTests=99)), - dict(events,bundle_counts=sorted(SHARED_COUNTS.values())[:-1])]: - self.assertFalse(check_result(command,0,'STORE_RUNTIME R06 passed',bad)[0]) - - def test_prepared_entry_rejects_missing_renamed_and_wrong_module_cases(self): - command = ['swift', 'test', '--filter', 'navigationPreparedD0'] - names = list(DISCOVERY['prepared_entry_function_ids']) - events = dict(bundle_counts=[10], module_function_counts={'JSONNavigationTests': 10}, - parameter_case_invocations=0, function_names=names) - self.assertTrue(check_result(command, 0, '', events)[0]) - for broken in [dict(events, function_names=names[:-1]), - dict(events, function_names=names[:-1]+['Other.renamed()']), - dict(events, parameter_case_invocations=1), - dict(events, bundle_counts=[9]), - dict(events, module_function_counts={'WrongModule': 10})]: - self.assertFalse(check_result(command, 0, '', broken)[0]) - - def test_composition_discovery_and_capacity_limits_are_not_blanket_successes(self): - command = ['swift', 'test', '--package-path', 'Tools/CompositionQualification'] - events = dict(bundle_counts=[5], module_function_counts={'CompositionScaleTests': 5}, - parameter_case_invocations=0) - self.assertTrue(check_result(command, 0, '', events)[0]) - for broken in [dict(events, bundle_counts=[4]), - dict(events, module_function_counts={'WrongModule': 5}), - dict(events, parameter_case_invocations=1)]: - self.assertFalse(check_result(command, 0, '', broken)[0]) - command = ['python3', '-B', 'Scripts/qualify_composition_scale.py'] - report = dict(status='passed', cells=8, completed_imports=5, operational_refusals=3, - adoption_status='bounded_fixtures_only', simple_strategy_capacity_status='blocked', - unmet_capacity_targets=['balanced-128-128']) - self.assertTrue(check_result(command, 0, json.dumps(report))[0]) - for broken in [dict(report, completed_imports=8), dict(report, operational_refusals=0), - dict(report, unmet_capacity_targets=[]), dict(report, adoption_status='all_imports_qualified')]: - self.assertFalse(check_result(command, 0, json.dumps(broken))[0]) - - # Combined-source checks must retain both independently owned optional products. - for module, count, command in [ - ('JSONNavigationTests', 57, ['swift', 'test', '--filter', 'navigation']), - ('JSONInspectionTests', 222, ['swift', 'test', '--filter', 'inspection|snapshot|projectionP|p2|exactProjectionPUB|graphG|store|workflow']), - ('JSONSchemaExamplesTests', 98, ['swift', 'test', '--test-product', 'JSONSchemaExamplesTests']), - ]: - events = dict(bundle_counts=[count], module_function_counts={module: count}, - parameter_case_invocations=65 if module == 'JSONInspectionTests' else 0, parameterized_functions={'storeD1NativeVector(id:)':65} if module == 'JSONInspectionTests' else {}, function_names=sorted(PROJECTION_FUNCTIONS | P2_FUNCTIONS | PUBLIC_FUNCTIONS | GRAPH_FUNCTIONS | STORE_FUNCTIONS) if module == 'JSONInspectionTests' else []) - self.assertTrue(check_result(command, 0, store_markers() if module == 'JSONInspectionTests' else '', events)[0]) - for broken in [dict(events, bundle_counts=[count-1]), - dict(events, module_function_counts={'WrongModule': count}), - dict(events, parameter_case_invocations=1)]: - self.assertFalse(check_result(command, 0, store_markers() if module == 'JSONInspectionTests' else '', broken)[0]) - policy = DISCOVERY - for key in ('root_functions', 'portable_root_functions', 'wasi_root_functions'): - self.assertEqual(policy[key]['JSONNavigationTests'], 57) - self.assertEqual(policy[key]['JSONSchemaExamplesTests'], 98) - self.assertEqual(policy[key]['JSONInspectionTests'], 222) - self.assertEqual(policy[key]['JSONIntegrationTests'], 45) - for key in ('shared_state_functions', 'portable_shared_state_functions'): - self.assertEqual(policy[key]['JSONNavigationTests'], 2) - self.assertEqual(policy[key]['JSONSchemaExamplesTests'], 4) - self.assertEqual(policy[key]['JSONInspectionTests'], 11) - self.assertEqual(policy[key]['JSONIntegrationTests'], 12) - - def test_private_projection_identities_cannot_be_missing_renamed_or_duplicated(self): - names = sorted(PROJECTION_FUNCTIONS) - self.assertEqual(len(names), 32) - self.assertTrue(check_projection_cases({'function_names': names})) - for broken in [names[:-1], names + names[:1], - ['WrongModule.' + names[0].split('.', 1)[1]] + names[1:], - [names[0].replace('projectionP01', 'projectionP99')] + names[1:]]: - self.assertFalse(check_projection_cases({'function_names': broken})) - shared = ['JSONInspectionTests.projectionP16SharedViewsHaveIndependentHealthAndLegacyStatistics()'] - self.assertTrue(check_projection_cases({'function_names': shared}, shared=True)) - self.assertFalse(check_projection_cases({'function_names': []}, shared=True)) - - def test_p2_function_identities_are_exact_and_shared_is_separate(self): - names = sorted(P2_FUNCTIONS) - self.assertEqual(len(names), 46) - self.assertTrue(check_p2_cases({'function_names': names})) - for broken in [names[:-1], names + names[:1], ['WrongModule.' + names[0]] + names[1:], - [names[0] + 'renamed'] + names[1:]]: - self.assertFalse(check_p2_cases({'function_names': broken})) - self.assertTrue(check_p2_cases({'function_names': sorted(P2_SHARED)}, shared=True)) - self.assertFalse(check_p2_cases({'function_names': names}, shared=True)) - - def test_p2_borrow_receipt_rejects_crash_incidental_diagnostic_and_missing_runtime(self): - names = ['build','path','positive-codegen','escape','send','method','task','external','link','runtime'] - negatives = {'escape','send','method','task','external'} - valid = dict(status='passed', commands=[dict(name=config+'-'+name, status='passed', - compiler_crash=False, expected_diagnostic='expected error' if name in negatives else None, - primary_diagnostics=['expected error'] if name in negatives else [], exit_code=1 if name in negatives else 0) - for config in ['debug','release'] for name in names], - runtime_results={c:sorted(BORROW_CHECKS) for c in ['debug','release']}) - self.assertTrue(check_borrow_receipt(valid)) - for mode in ['missing','duplicate','crash','diagnostic','zero','runtime']: - broken = copy.deepcopy(valid) - if mode == 'missing': broken['commands'].pop() - elif mode == 'duplicate': broken['commands'][0]['name'] = broken['commands'][1]['name'] - elif mode == 'crash': broken['commands'][3]['compiler_crash'] = True - elif mode == 'diagnostic': broken['commands'][3]['primary_diagnostics'] = ['unrelated'] - elif mode == 'zero': broken['commands'][3]['exit_code'] = 0 - else: broken['runtime_results']['release'].pop() - self.assertFalse(check_borrow_receipt(broken)) - - # Build an independent synthetic *harness* fixture, not qualification evidence. - private_names = ['build','path','positive-codegen','external-set-reducer','external-matrix-reducer','external-accounting', - 'set-constructor','matrix-constructor','matrix-mutation','link','runtime'] - private = dict(status='passed', commands=[dict(name=c+'-'+n, status='passed', - compiler_crash=False, expected_diagnostic='inaccessible' if 3 <= i <= 8 else None, - primary_diagnostics=['inaccessible'] if 3 <= i <= 8 else [], exit_code=1 if 3 <= i <= 8 else 0) - for c in ['debug','release'] for i,n in enumerate(private_names)], - runtime_results={c:['set-exact','matrix-exact','zero-shape','typed-refusal','one-admission','independent-read'] - for c in ['debug','release']}) - self.assertTrue(check_private_receipt(private)) - broken = copy.deepcopy(private); broken['commands'][3]['exit_code'] = 0 - self.assertFalse(check_private_receipt(broken)) - broken = copy.deepcopy(private); broken['runtime_results']['debug'][0] = 'unknown' - self.assertFalse(check_private_receipt(broken)) - - def test_public_exact_identities_and_receipts_are_fail_closed(self): - names = sorted(PUBLIC_FUNCTIONS) - self.assertEqual(len(names), 8) - self.assertTrue(check_exact_cases({'function_names': names})) - for broken in [names[:-1], names + names[:1], ['Wrong.'+names[0]]+names[1:]]: - self.assertFalse(check_exact_cases({'function_names': broken})) - self.assertTrue(check_exact_cases({'function_names': sorted(PUBLIC_SHARED)}, shared=True)) - self.assertFalse(check_exact_receipt({'status': 'passed', 'commands': [], 'consumer_results': {}})) - def test_graph_identity_and_compiler_receipts_reject_false_success(self): - names=sorted(GRAPH_FUNCTIONS) - self.assertEqual(len(names),48) - self.assertTrue(check_graph_cases({'function_names':names})) - for broken in [names[:-1],names+names[:1],['WrongModule.'+names[0].split('.',1)[1]]+names[1:]]: - self.assertFalse(check_graph_cases({'function_names':broken})) - for context,count,negatives in [(True,18,8),(False,28,18)]: - commands=[dict(name=str(n),status='passed',compiler_crash=False,exit_code=1 if n