From b61a9b1be8681c6ceb28b098efcea79204c3d4ac Mon Sep 17 00:00:00 2001 From: Alsu Date: Mon, 31 Aug 2026 11:47:09 +0000 Subject: [PATCH 1/4] Add Stress test (amd_asan_ubsan, cas s3 storage) Stress already runs stateless tests at random in parallel; this lane installs cas_s3 as the default MergeTree policy (RustFS, not MinIO) so CAS is actually on that path. "cas s3" must win over the plain "s3" substring or install.sh would still pick the S3 default. Co-authored-by: Cursor --- .github/workflows/master.yml | 47 ++++++++++++++++++- .github/workflows/pull_request.yml | 45 +++++++++++++++++- ci/defs/job_configs.py | 5 ++ ci/jobs/scripts/clickhouse_proc.py | 2 + ci/jobs/scripts/stress/stress.py | 8 ++++ ci/jobs/stress_job.py | 7 ++- ci/tests/test_stress_job.py | 22 +++++++++ .../ContentAddressed/README.md | 5 ++ tests/docker_scripts/stress_runner.sh | 20 +++++++- tests/docker_scripts/stress_tests.lib | 12 ++++- 10 files changed, 167 insertions(+), 6 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index fc49ce2ad074..9c59166bc718 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -5391,6 +5391,50 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (amd_asan_ubsan)' --workflow "MasterCI" --ci --timestamp + stress_test_amd_asan_ubsan_cas_s3_storage: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + needs: [build_amd_asan_ubsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RyZXNzIHRlc3QgKGFtZF9hc2FuX3Vic2FuLCBjYXMgczMgc3RvcmFnZSk=') }} + name: "Stress test (amd_asan_ubsan, cas s3 storage)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stress test (amd_asan_ubsan, cas s3 storage)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + cat > ./ci/tmp/workflow_inputs.json << 'EOF' + ${{ toJson(github.event.inputs) }} + EOF + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (amd_asan_ubsan, cas s3 storage)' --workflow "MasterCI" --ci --timestamp + stress_test_amd_tsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [build_amd_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -6505,7 +6549,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_llvm_coverage_per_test, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_ubsan, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, clickbench_amd_release, clickbench_arm_release, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sign_release_amd_release, sign_release_arm_release, source_upload, sqllogic_test, sqlstorm_test, sqltest, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_1_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_2_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_3_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_4_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_5_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_6_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_7_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_8_8, stateless_tests_amd_msan_cas_s3_storage_parallel_1_3, stateless_tests_amd_msan_cas_s3_storage_parallel_2_3, stateless_tests_amd_msan_cas_s3_storage_parallel_3_3, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_arm_ubsan, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_llvm_coverage_per_test, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_ubsan, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, clickbench_amd_release, clickbench_arm_release, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sign_release_amd_release, sign_release_arm_release, source_upload, sqllogic_test, sqlstorm_test, sqltest, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_1_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_2_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_3_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_4_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_5_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_6_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_7_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_8_8, stateless_tests_amd_msan_cas_s3_storage_parallel_1_3, stateless_tests_amd_msan_cas_s3_storage_parallel_2_3, stateless_tests_amd_msan_cas_s3_storage_parallel_3_3, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_asan_ubsan_cas_s3_storage, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_arm_ubsan, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -6709,6 +6753,7 @@ jobs: - integration_tests_amd_msan_10_10 - stress_test_amd_debug - stress_test_amd_asan_ubsan + - stress_test_amd_asan_ubsan_cas_s3_storage - stress_test_amd_tsan - stress_test_amd_msan - stress_test_arm_release diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ba64d1a171e0..5a9f1c9fde17 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -5002,6 +5002,48 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (amd_asan_ubsan)' --workflow "PR" --ci --timestamp + stress_test_amd_asan_ubsan_cas_s3_storage: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RyZXNzIHRlc3QgKGFtZF9hc2FuX3Vic2FuLCBjYXMgczMgc3RvcmFnZSk=') }} + name: "Stress test (amd_asan_ubsan, cas s3 storage)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stress test (amd_asan_ubsan, cas s3 storage)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (amd_asan_ubsan, cas s3 storage)' --workflow "PR" --ci --timestamp + stress_test_amd_tsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] @@ -6035,7 +6077,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_ubsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, ci_tests, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_asan_ubsan_targeted, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, keeper_stress_tests_pr, quick_functional_tests, source_upload, sqllogic_test, sqlstorm_test, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_cas_s3_storage_parallel_1_3, stateless_tests_amd_msan_cas_s3_storage_parallel_2_3, stateless_tests_amd_msan_cas_s3_storage_parallel_3_3, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_asan_ubsan_targeted, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_arm_ubsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_ubsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, ci_tests, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_asan_ubsan_targeted, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, keeper_stress_tests_pr, quick_functional_tests, source_upload, sqllogic_test, sqlstorm_test, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_cas_s3_storage_parallel_1_3, stateless_tests_amd_msan_cas_s3_storage_parallel_2_3, stateless_tests_amd_msan_cas_s3_storage_parallel_3_3, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_asan_ubsan_targeted, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_asan_ubsan_cas_s3_storage, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_arm_ubsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -6232,6 +6274,7 @@ jobs: - compatibility_check_arm_release - stress_test_amd_debug - stress_test_amd_asan_ubsan + - stress_test_amd_asan_ubsan_cas_s3_storage - stress_test_amd_tsan - stress_test_amd_msan - stress_test_arm_release diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index 3d65c8f09b1f..21b43fee19aa 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -874,6 +874,11 @@ class JobConfigs: runs_on=RunnerLabels.FUNC_TESTER_AMD, requires=[ArtifactNames.DEB_AMD_ASAN_UBSAN], ), + Job.ParamSet( + parameter="amd_asan_ubsan, cas s3 storage", + runs_on=RunnerLabels.FUNC_TESTER_AMD, + requires=[ArtifactNames.DEB_AMD_ASAN_UBSAN], + ), Job.ParamSet( parameter="amd_tsan", runs_on=RunnerLabels.FUNC_TESTER_AMD, diff --git a/ci/jobs/scripts/clickhouse_proc.py b/ci/jobs/scripts/clickhouse_proc.py index d4ce6313bf24..abde80b862e2 100644 --- a/ci/jobs/scripts/clickhouse_proc.py +++ b/ci/jobs/scripts/clickhouse_proc.py @@ -1549,6 +1549,8 @@ def set_random_timezone(): res = ch.start_minio(param) elif command == "start_azurite": res = ch.start_azurite() + elif command == "start_rustfs": + res = ch.start_rustfs() else: raise ValueError(f"Unknown command: {command}") except Exception: diff --git a/ci/jobs/scripts/stress/stress.py b/ci/jobs/scripts/stress/stress.py index e3ad390ff141..448271322b27 100755 --- a/ci/jobs/scripts/stress/stress.py +++ b/ci/jobs/scripts/stress/stress.py @@ -128,6 +128,14 @@ def get_options(i: int, upgrade_check: bool, encrypted_storage: bool) -> str: options = [] client_options = [] + # Honour the CAS-as-default MergeTree policy so clickhouse-test skips + # `no-cas-storage` / `no-object-storage` / `no-s3-storage` tests. Without + # this flag those tests still run and fail against a CAS disk. + if os.environ.get("USE_CAS_S3_STORAGE_FOR_MERGE_TREE") == "1": + options.append("--cas-s3-storage") + elif os.environ.get("USE_CAS_STORAGE_FOR_MERGE_TREE") == "1": + options.append("--cas-storage") + if upgrade_check: # Disable settings randomization for upgrade checks to prevent test failures caused by missing settings in old version options.append("--no-random-settings") diff --git a/ci/jobs/stress_job.py b/ci/jobs/stress_job.py index df09b3f70194..943bb46b0d76 100644 --- a/ci/jobs/stress_job.py +++ b/ci/jobs/stress_job.py @@ -142,7 +142,12 @@ def get_additional_envs(info, check_name: str) -> List[str]: "AZURE_STORAGE_ACCOUNT_URL=$AZURE_STORAGE_ACCOUNT_URL", ]) - if "s3" in check_name: + # "cas s3" must win over the plain "s3" substring. Otherwise + # `USE_S3_STORAGE_FOR_MERGE_TREE=1` is also set and install.sh's + # if/elif chain installs the plain-S3 default policy instead of CAS. + if "cas s3" in check_name: + result.append("USE_CAS_S3_STORAGE_FOR_MERGE_TREE=1") + elif "s3" in check_name: result.append("USE_S3_STORAGE_FOR_MERGE_TREE=1") result.append( diff --git a/ci/tests/test_stress_job.py b/ci/tests/test_stress_job.py index 4cda92f8435b..79e1bf38668d 100644 --- a/ci/tests/test_stress_job.py +++ b/ci/tests/test_stress_job.py @@ -35,6 +35,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../..")) from ci.jobs.stress_job import ( + get_additional_envs, process_results, read_test_results, sanitize_test_result_line, @@ -245,5 +246,26 @@ def test_dpkg_progress_in_info_does_not_split_row(tmp_path): assert malformed == [] +def test_cas_s3_env_is_exclusive_of_plain_s3(monkeypatch): + monkeypatch.setattr("ci.jobs.ci_utils.is_extended_run", lambda: False) + envs = get_additional_envs(None, "Stress test (amd_asan_ubsan, cas s3 storage)") + assert "USE_CAS_S3_STORAGE_FOR_MERGE_TREE=1" in envs + assert "USE_S3_STORAGE_FOR_MERGE_TREE=1" not in envs + + +def test_plain_s3_env_is_unchanged(monkeypatch): + monkeypatch.setattr("ci.jobs.ci_utils.is_extended_run", lambda: False) + envs = get_additional_envs(None, "Stress test (arm_asan_ubsan, s3)") + assert "USE_S3_STORAGE_FOR_MERGE_TREE=1" in envs + assert "USE_CAS_S3_STORAGE_FOR_MERGE_TREE=1" not in envs + + +def test_default_stress_env_has_neither_s3_flag(monkeypatch): + monkeypatch.setattr("ci.jobs.ci_utils.is_extended_run", lambda: False) + envs = get_additional_envs(None, "Stress test (amd_asan_ubsan)") + assert "USE_S3_STORAGE_FOR_MERGE_TREE=1" not in envs + assert "USE_CAS_S3_STORAGE_FOR_MERGE_TREE=1" not in envs + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/README.md b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/README.md index 305ffe1b1ff9..d60b56b92fd7 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/README.md +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/README.md @@ -190,6 +190,11 @@ not to multipart-capable blob publication. (local object storage) and "`cas s3 storage`" run the whole stateless suite with `MergeTree` defaulting to a CAS disk. Tests that legitimately cannot run there carry the `no-cas-storage` tag. +- **Stress**: `Stress test (amd_asan_ubsan, cas s3 storage)` runs the + same `clickhouse-test --stress-tests` loop as the other stress jobs + (stateless tests picked at random, several clients in parallel, no + result validation) with `cas_s3` as the default `MergeTree` policy + and RustFS backing the pool. - **Soak / chaos**: `utils/ca-soak/` — multi-replica docker-compose harnesses (fault proxies, GC sharding variants, AWS S3/GCS backends) and adversarial scenarios. diff --git a/tests/docker_scripts/stress_runner.sh b/tests/docker_scripts/stress_runner.sh index 7d62e06683e1..c68c154e3797 100755 --- a/tests/docker_scripts/stress_runner.sh +++ b/tests/docker_scripts/stress_runner.sh @@ -52,6 +52,11 @@ cd /repo && python3 /repo/ci/jobs/scripts/clickhouse_proc.py logs_export_config cd /repo && python3 /repo/ci/jobs/scripts/clickhouse_proc.py start_minio stateless || { echo "Failed to start minio"; exit 1; } cd /repo && python3 /repo/ci/jobs/scripts/clickhouse_proc.py start_azurite || { echo "Failed to start azurite"; exit 1; } +if [[ "$USE_CAS_S3_STORAGE_FOR_MERGE_TREE" == "1" ]]; then + # CAS-over-S3 needs RustFS (enforced If-Match deletes). MinIO stays up for + # the non-CAS s3 disks that EXPORT_S3_STORAGE_POLICIES still installs. + cd /repo && python3 /repo/ci/jobs/scripts/clickhouse_proc.py start_rustfs || { echo "Failed to start rustfs"; exit 1; } +fi # Start Redpanda (Kafka-compatible broker) so that Kafka engine tests work and # do not leave behind broken StorageKafka tables whose background threads cause @@ -100,7 +105,10 @@ clickhouse-client --query "SHOW TABLES FROM tpcds" clickhouse-client --query "SHOW TABLES FROM tpch" clickhouse-client --query "SHOW TABLES FROM test" -if [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" == "1" ]]; then +if [[ "$USE_CAS_S3_STORAGE_FOR_MERGE_TREE" == "1" ]]; then + TEMP_POLICY="cas_s3" + echo "Using cas_s3 storage policy" +elif [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" == "1" ]]; then TEMP_POLICY="s3_cache" elif [[ "$USE_AZURE_STORAGE_FOR_MERGE_TREE" == "1" ]]; then TEMP_POLICY="azure_cache" @@ -315,6 +323,12 @@ start_server 10 || { echo "Failed to start server"; exit 1; } check_server_start +# clickhouse-local cannot claim a CAS server-root (its uuid is all-zero), so +# dump system logs via the still-running server that owns the root. +if [[ "$USE_CAS_S3_STORAGE_FOR_MERGE_TREE" == "1" || "$USE_CAS_STORAGE_FOR_MERGE_TREE" == "1" ]]; then + collect_query_and_trace_logs --client +fi + stop_server [ -f /var/log/clickhouse-server/clickhouse-server.log ] || echo -e "Server log does not exist\tFAIL" @@ -327,6 +341,8 @@ check_logs_for_critical_errors tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||: -collect_query_and_trace_logs +if [[ "$USE_CAS_S3_STORAGE_FOR_MERGE_TREE" != "1" && "$USE_CAS_STORAGE_FOR_MERGE_TREE" != "1" ]]; then + collect_query_and_trace_logs +fi mv /var/log/clickhouse-server/stderr.log /test_output/ diff --git a/tests/docker_scripts/stress_tests.lib b/tests/docker_scripts/stress_tests.lib index ae961a4a431d..6a6d3a156a88 100644 --- a/tests/docker_scripts/stress_tests.lib +++ b/tests/docker_scripts/stress_tests.lib @@ -350,10 +350,20 @@ function check_logs_for_critical_errors() function collect_query_and_trace_logs() { + # `--client`: dump via the running server. Required on a CAS disk because + # clickhouse-local starts with uuid 0 and refuses to claim the server-root. + local use_client=0 + if [[ "${1:-}" == "--client" ]]; then + use_client=1 + fi for table in query_log trace_log metric_log aggregated_zookeeper_log do # Don't ignore errors here, it leads to ignore sanitizer reports when running clickhouse-local - clickhouse-local --config-file=/etc/clickhouse-server/config.xml --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst + if [[ "$use_client" == "1" ]]; then + clickhouse-client -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst + else + clickhouse-local --config-file=/etc/clickhouse-server/config.xml --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst + fi done } From 07aec1c8a23a8b572ebacec6258eb12421dd1ef8 Mon Sep 17 00:00:00 2001 From: Alsu Date: Mon, 31 Aug 2026 12:01:37 +0000 Subject: [PATCH 2/4] Add CAS-S3 stress jobs for debug, TSan, and MSan Match the AMD stress sanitizer set plus the non-sanitizer debug package, not only ASan. Co-authored-by: Cursor --- .github/workflows/master.yml | 137 +++++++++++++++++- .github/workflows/pull_request.yml | 131 ++++++++++++++++- ci/defs/job_configs.py | 15 ++ ci/tests/test_stress_job.py | 13 +- .../ContentAddressed/README.md | 9 +- 5 files changed, 297 insertions(+), 8 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 9c59166bc718..7641a87e603b 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -5347,6 +5347,50 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (amd_debug)' --workflow "MasterCI" --ci --timestamp + stress_test_amd_debug_cas_s3_storage: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + needs: [build_amd_debug, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RyZXNzIHRlc3QgKGFtZF9kZWJ1ZywgY2FzIHMzIHN0b3JhZ2Up') }} + name: "Stress test (amd_debug, cas s3 storage)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stress test (amd_debug, cas s3 storage)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + cat > ./ci/tmp/workflow_inputs.json << 'EOF' + ${{ toJson(github.event.inputs) }} + EOF + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (amd_debug, cas s3 storage)' --workflow "MasterCI" --ci --timestamp + stress_test_amd_asan_ubsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [build_amd_asan_ubsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -5479,6 +5523,50 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (amd_tsan)' --workflow "MasterCI" --ci --timestamp + stress_test_amd_tsan_cas_s3_storage: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + needs: [build_amd_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RyZXNzIHRlc3QgKGFtZF90c2FuLCBjYXMgczMgc3RvcmFnZSk=') }} + name: "Stress test (amd_tsan, cas s3 storage)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stress test (amd_tsan, cas s3 storage)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + cat > ./ci/tmp/workflow_inputs.json << 'EOF' + ${{ toJson(github.event.inputs) }} + EOF + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (amd_tsan, cas s3 storage)' --workflow "MasterCI" --ci --timestamp + stress_test_amd_msan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -5523,6 +5611,50 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (amd_msan)' --workflow "MasterCI" --ci --timestamp + stress_test_amd_msan_cas_s3_storage: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RyZXNzIHRlc3QgKGFtZF9tc2FuLCBjYXMgczMgc3RvcmFnZSk=') }} + name: "Stress test (amd_msan, cas s3 storage)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stress test (amd_msan, cas s3 storage)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + cat > ./ci/tmp/workflow_inputs.json << 'EOF' + ${{ toJson(github.event.inputs) }} + EOF + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (amd_msan, cas s3 storage)' --workflow "MasterCI" --ci --timestamp + stress_test_arm_release: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64] needs: [build_arm_release, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] @@ -6549,7 +6681,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_llvm_coverage_per_test, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_ubsan, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, clickbench_amd_release, clickbench_arm_release, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sign_release_amd_release, sign_release_arm_release, source_upload, sqllogic_test, sqlstorm_test, sqltest, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_1_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_2_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_3_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_4_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_5_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_6_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_7_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_8_8, stateless_tests_amd_msan_cas_s3_storage_parallel_1_3, stateless_tests_amd_msan_cas_s3_storage_parallel_2_3, stateless_tests_amd_msan_cas_s3_storage_parallel_3_3, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_asan_ubsan_cas_s3_storage, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_arm_ubsan, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_llvm_coverage_per_test, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_ubsan, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, clickbench_amd_release, clickbench_arm_release, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sign_release_amd_release, sign_release_arm_release, source_upload, sqllogic_test, sqlstorm_test, sqltest, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_1_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_2_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_3_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_4_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_5_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_6_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_7_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_8_8, stateless_tests_amd_msan_cas_s3_storage_parallel_1_3, stateless_tests_amd_msan_cas_s3_storage_parallel_2_3, stateless_tests_amd_msan_cas_s3_storage_parallel_3_3, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_asan_ubsan_cas_s3_storage, stress_test_amd_debug, stress_test_amd_debug_cas_s3_storage, stress_test_amd_msan, stress_test_amd_msan_cas_s3_storage, stress_test_amd_tsan, stress_test_amd_tsan_cas_s3_storage, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_arm_ubsan, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -6752,10 +6884,13 @@ jobs: - integration_tests_amd_msan_9_10 - integration_tests_amd_msan_10_10 - stress_test_amd_debug + - stress_test_amd_debug_cas_s3_storage - stress_test_amd_asan_ubsan - stress_test_amd_asan_ubsan_cas_s3_storage - stress_test_amd_tsan + - stress_test_amd_tsan_cas_s3_storage - stress_test_amd_msan + - stress_test_amd_msan_cas_s3_storage - stress_test_arm_release - stress_test_arm_debug - stress_test_arm_asan_ubsan diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 5a9f1c9fde17..5fb334321137 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -4960,6 +4960,48 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (amd_debug)' --workflow "PR" --ci --timestamp + stress_test_amd_debug_cas_s3_storage: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RyZXNzIHRlc3QgKGFtZF9kZWJ1ZywgY2FzIHMzIHN0b3JhZ2Up') }} + name: "Stress test (amd_debug, cas s3 storage)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stress test (amd_debug, cas s3 storage)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (amd_debug, cas s3 storage)' --workflow "PR" --ci --timestamp + stress_test_amd_asan_ubsan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] @@ -5086,6 +5128,48 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (amd_tsan)' --workflow "PR" --ci --timestamp + stress_test_amd_tsan_cas_s3_storage: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RyZXNzIHRlc3QgKGFtZF90c2FuLCBjYXMgczMgc3RvcmFnZSk=') }} + name: "Stress test (amd_tsan, cas s3 storage)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stress test (amd_tsan, cas s3 storage)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (amd_tsan, cas s3 storage)' --workflow "PR" --ci --timestamp + stress_test_amd_msan: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] @@ -5128,6 +5212,48 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (amd_msan)' --workflow "PR" --ci --timestamp + stress_test_amd_msan_cas_s3_storage: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RyZXNzIHRlc3QgKGFtZF9tc2FuLCBjYXMgczMgc3RvcmFnZSk=') }} + name: "Stress test (amd_msan, cas s3 storage)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Stress test (amd_msan, cas s3 storage)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (amd_msan, cas s3 storage)' --workflow "PR" --ci --timestamp + stress_test_arm_release: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_tsan, build_arm_binary, build_arm_release, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_arm_binary_parallel] @@ -6077,7 +6203,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_ubsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, ci_tests, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_asan_ubsan_targeted, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, keeper_stress_tests_pr, quick_functional_tests, source_upload, sqllogic_test, sqlstorm_test, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_cas_s3_storage_parallel_1_3, stateless_tests_amd_msan_cas_s3_storage_parallel_2_3, stateless_tests_amd_msan_cas_s3_storage_parallel_3_3, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_asan_ubsan_targeted, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_asan_ubsan_cas_s3_storage, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_arm_ubsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_ubsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, ci_tests, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_asan_ubsan_targeted, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, keeper_stress_tests_pr, quick_functional_tests, source_upload, sqllogic_test, sqlstorm_test, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_asan_ubsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_2, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_binary_cas_s3_storage_parallel, stateless_tests_amd_binary_cas_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_cas_s3_storage_parallel_1_3, stateless_tests_amd_msan_cas_s3_storage_parallel_2_3, stateless_tests_amd_msan_cas_s3_storage_parallel_3_3, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_1_2, stateless_tests_amd_tsan_cas_s3_storage_parallel_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_asan_ubsan_targeted, stateless_tests_arm_binary_cas_s3_storage_parallel, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_asan_ubsan_cas_s3_storage, stress_test_amd_debug, stress_test_amd_debug_cas_s3_storage, stress_test_amd_msan, stress_test_amd_msan_cas_s3_storage, stress_test_amd_tsan, stress_test_amd_tsan_cas_s3_storage, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_arm_ubsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -6273,10 +6399,13 @@ jobs: - compatibility_check_amd_release - compatibility_check_arm_release - stress_test_amd_debug + - stress_test_amd_debug_cas_s3_storage - stress_test_amd_asan_ubsan - stress_test_amd_asan_ubsan_cas_s3_storage - stress_test_amd_tsan + - stress_test_amd_tsan_cas_s3_storage - stress_test_amd_msan + - stress_test_amd_msan_cas_s3_storage - stress_test_arm_release - stress_test_arm_debug - stress_test_arm_asan_ubsan diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index 21b43fee19aa..9def1e5a7fe2 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -869,6 +869,11 @@ class JobConfigs: runs_on=RunnerLabels.FUNC_TESTER_AMD, requires=[ArtifactNames.DEB_AMD_DEBUG], ), + Job.ParamSet( + parameter="amd_debug, cas s3 storage", + runs_on=RunnerLabels.FUNC_TESTER_AMD, + requires=[ArtifactNames.DEB_AMD_DEBUG], + ), Job.ParamSet( parameter="amd_asan_ubsan", runs_on=RunnerLabels.FUNC_TESTER_AMD, @@ -884,11 +889,21 @@ class JobConfigs: runs_on=RunnerLabels.FUNC_TESTER_AMD, requires=[ArtifactNames.DEB_AMD_TSAN], ), + Job.ParamSet( + parameter="amd_tsan, cas s3 storage", + runs_on=RunnerLabels.FUNC_TESTER_AMD, + requires=[ArtifactNames.DEB_AMD_TSAN], + ), Job.ParamSet( parameter="amd_msan", runs_on=RunnerLabels.FUNC_TESTER_AMD, requires=[ArtifactNames.DEB_AMD_MSAN], ), + Job.ParamSet( + parameter="amd_msan, cas s3 storage", + runs_on=RunnerLabels.FUNC_TESTER_AMD, + requires=[ArtifactNames.DEB_AMD_MSAN], + ), Job.ParamSet( parameter="arm_release", runs_on=RunnerLabels.FUNC_TESTER_ARM, diff --git a/ci/tests/test_stress_job.py b/ci/tests/test_stress_job.py index 79e1bf38668d..35cb34e6513b 100644 --- a/ci/tests/test_stress_job.py +++ b/ci/tests/test_stress_job.py @@ -246,9 +246,18 @@ def test_dpkg_progress_in_info_does_not_split_row(tmp_path): assert malformed == [] -def test_cas_s3_env_is_exclusive_of_plain_s3(monkeypatch): +@pytest.mark.parametrize( + "check_name", + [ + "Stress test (amd_debug, cas s3 storage)", + "Stress test (amd_asan_ubsan, cas s3 storage)", + "Stress test (amd_tsan, cas s3 storage)", + "Stress test (amd_msan, cas s3 storage)", + ], +) +def test_cas_s3_env_is_exclusive_of_plain_s3(monkeypatch, check_name): monkeypatch.setattr("ci.jobs.ci_utils.is_extended_run", lambda: False) - envs = get_additional_envs(None, "Stress test (amd_asan_ubsan, cas s3 storage)") + envs = get_additional_envs(None, check_name) assert "USE_CAS_S3_STORAGE_FOR_MERGE_TREE=1" in envs assert "USE_S3_STORAGE_FOR_MERGE_TREE=1" not in envs diff --git a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/README.md b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/README.md index d60b56b92fd7..d631d53d021d 100644 --- a/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/README.md +++ b/src/Disks/DiskObjectStorage/MetadataStorages/ContentAddressed/README.md @@ -190,11 +190,12 @@ not to multipart-capable blob publication. (local object storage) and "`cas s3 storage`" run the whole stateless suite with `MergeTree` defaulting to a CAS disk. Tests that legitimately cannot run there carry the `no-cas-storage` tag. -- **Stress**: `Stress test (amd_asan_ubsan, cas s3 storage)` runs the - same `clickhouse-test --stress-tests` loop as the other stress jobs - (stateless tests picked at random, several clients in parallel, no +- **Stress**: `Stress test (amd_{debug,asan_ubsan,tsan,msan}, cas s3 storage)` + run the same `clickhouse-test --stress-tests` loop as the other stress + jobs (stateless tests picked at random, several clients in parallel, no result validation) with `cas_s3` as the default `MergeTree` policy - and RustFS backing the pool. + and RustFS backing the pool. One job per AMD sanitizer build plus + debug (the non-sanitizer stress package). - **Soak / chaos**: `utils/ca-soak/` — multi-replica docker-compose harnesses (fault proxies, GC sharding variants, AWS S3/GCS backends) and adversarial scenarios. From 6e29e68339280964eaa27f4a956b295ed55507f4 Mon Sep 17 00:00:00 2001 From: Alsu Date: Mon, 31 Aug 2026 12:15:01 +0000 Subject: [PATCH 3/4] Collect rustfs.log in CAS-S3 stress jobs The stress job uploads only files found under `/test_output` and the server log directory, but `start_rustfs` writes to `ci/tmp/rustfs.log`. RustFS is the storage backend for the `cas_s3` policy, so its log is the only record of pool-side failures such as `If-Match` mismatches or request saturation. Copy it into `/test_output` compressed so it reaches the CI report. Co-authored-by: Cursor --- tests/docker_scripts/stress_runner.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/docker_scripts/stress_runner.sh b/tests/docker_scripts/stress_runner.sh index c68c154e3797..d039b8e375aa 100755 --- a/tests/docker_scripts/stress_runner.sh +++ b/tests/docker_scripts/stress_runner.sh @@ -346,3 +346,9 @@ if [[ "$USE_CAS_S3_STORAGE_FOR_MERGE_TREE" != "1" && "$USE_CAS_STORAGE_FOR_MERGE fi mv /var/log/clickhouse-server/stderr.log /test_output/ + +# The job uploads /test_output only, and rustfs.log is the sole record of +# pool-side S3 failures (412 mismatches, 503 saturation) behind a CAS disk. +if [[ -f /repo/ci/tmp/rustfs.log ]]; then + zstd --threads=0 -o /test_output/rustfs.log.zst /repo/ci/tmp/rustfs.log ||: +fi From 894658a37c8d879e8ad26b48800dc16e80cea5be Mon Sep 17 00:00:00 2001 From: CarlosFelipeOR Date: Mon, 7 Sep 2026 13:28:27 -0300 Subject: [PATCH 4/4] Scrape system logs with a read-only CAS disk Opening the disk read-only skips the server-root claim that made `clickhouse-local` unusable, so every stress lane keeps one post-shutdown dump instead of forking to `clickhouse-client`. Flush the logs before the final stop, collect the helper logs host-side in `stress_job.py` so an early abort still publishes rustfs.log, and drop the unreachable `USE_CAS_STORAGE_FOR_MERGE_TREE`. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: CarlosFelipeOR --- ci/jobs/scripts/stress/stress.py | 2 -- ci/jobs/stress_job.py | 9 +++++++++ tests/docker_scripts/stress_runner.sh | 17 +++-------------- tests/docker_scripts/stress_tests.lib | 22 ++++++++++++---------- 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/ci/jobs/scripts/stress/stress.py b/ci/jobs/scripts/stress/stress.py index 448271322b27..0f0e30ce815c 100755 --- a/ci/jobs/scripts/stress/stress.py +++ b/ci/jobs/scripts/stress/stress.py @@ -133,8 +133,6 @@ def get_options(i: int, upgrade_check: bool, encrypted_storage: bool) -> str: # this flag those tests still run and fail against a CAS disk. if os.environ.get("USE_CAS_S3_STORAGE_FOR_MERGE_TREE") == "1": options.append("--cas-s3-storage") - elif os.environ.get("USE_CAS_STORAGE_FOR_MERGE_TREE") == "1": - options.append("--cas-storage") if upgrade_check: # Disable settings randomization for upgrade checks to prevent test failures caused by missing settings in old version diff --git a/ci/jobs/stress_job.py b/ci/jobs/stress_job.py index 943bb46b0d76..7b2ba415c184 100644 --- a/ci/jobs/stress_job.py +++ b/ci/jobs/stress_job.py @@ -1,6 +1,7 @@ import csv import logging import os +import shutil import socket import sys from pathlib import Path @@ -313,6 +314,14 @@ def run_stress_test(upgrade_check: bool = False) -> None: Utils.fix_ownership_after_docker(temp_path, docker_image) + # ci/tmp is not uploaded. Copying here rather than from stress_runner.sh runs + # whatever the container's exit code, so a job that aborts early still publishes + # rustfs.log -- the only record of what the pool side did behind a CAS disk. + for helper_log in ("rustfs.log", "minio.log", "azurite.log", "kafka.log"): + src = temp_path / helper_log + if src.is_file(): + shutil.copy(src, result_path / helper_log) + core_files = ClickHouseService.collect_cores(cores_path) is_oom = False diff --git a/tests/docker_scripts/stress_runner.sh b/tests/docker_scripts/stress_runner.sh index d039b8e375aa..42fbd294a835 100755 --- a/tests/docker_scripts/stress_runner.sh +++ b/tests/docker_scripts/stress_runner.sh @@ -323,11 +323,8 @@ start_server 10 || { echo "Failed to start server"; exit 1; } check_server_start -# clickhouse-local cannot claim a CAS server-root (its uuid is all-zero), so -# dump system logs via the still-running server that owns the root. -if [[ "$USE_CAS_S3_STORAGE_FOR_MERGE_TREE" == "1" || "$USE_CAS_STORAGE_FOR_MERGE_TREE" == "1" ]]; then - collect_query_and_trace_logs --client -fi +# The server may be killed rather than shut down, so don't rely on the shutdown flush. +clickhouse-client --receive_timeout 30 -q "SYSTEM FLUSH LOGS" ||: stop_server @@ -341,14 +338,6 @@ check_logs_for_critical_errors tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||: -if [[ "$USE_CAS_S3_STORAGE_FOR_MERGE_TREE" != "1" && "$USE_CAS_STORAGE_FOR_MERGE_TREE" != "1" ]]; then - collect_query_and_trace_logs -fi +collect_query_and_trace_logs mv /var/log/clickhouse-server/stderr.log /test_output/ - -# The job uploads /test_output only, and rustfs.log is the sole record of -# pool-side S3 failures (412 mismatches, 503 saturation) behind a CAS disk. -if [[ -f /repo/ci/tmp/rustfs.log ]]; then - zstd --threads=0 -o /test_output/rustfs.log.zst /repo/ci/tmp/rustfs.log ||: -fi diff --git a/tests/docker_scripts/stress_tests.lib b/tests/docker_scripts/stress_tests.lib index 6a6d3a156a88..c8729b86b02a 100644 --- a/tests/docker_scripts/stress_tests.lib +++ b/tests/docker_scripts/stress_tests.lib @@ -350,20 +350,22 @@ function check_logs_for_critical_errors() function collect_query_and_trace_logs() { - # `--client`: dump via the running server. Required on a CAS disk because - # clickhouse-local starts with uuid 0 and refuses to claim the server-root. - local use_client=0 - if [[ "${1:-}" == "--client" ]]; then - use_client=1 + # A writable open of a CAS disk claims the server-root and fails closed against the + # real server's owner uuid; read-only skips the claim, which is all a dump needs. + # `--follow-symlinks`: install.sh symlinks these configs. See dump_system_tables. + grep -Rl 'cas' /etc/clickhouse-server/ 2>/dev/null \ + | xargs -r sed -i --follow-symlinks 's|cas|castrue|g' + + # A declared but not read-only CAS disk means this scrape is about to die on ownership. + if grep -Rlq 'cas' /etc/clickhouse-server/ 2>/dev/null \ + && ! grep -Rlq 'castrue' /etc/clickhouse-server/ 2>/dev/null; then + echo "WARNING: a CAS disk is declared but the read-only marker was not inserted -- clickhouse-local will claim server-root ownership and this scrape will fail" fi + for table in query_log trace_log metric_log aggregated_zookeeper_log do # Don't ignore errors here, it leads to ignore sanitizer reports when running clickhouse-local - if [[ "$use_client" == "1" ]]; then - clickhouse-client -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst - else - clickhouse-local --config-file=/etc/clickhouse-server/config.xml --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst - fi + clickhouse-local --config-file=/etc/clickhouse-server/config.xml --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst done }