diff --git a/.github/workflows/build_deploy.yml b/.github/workflows/build_deploy.yml index ffdb1981793..9af68dfbd72 100644 --- a/.github/workflows/build_deploy.yml +++ b/.github/workflows/build_deploy.yml @@ -85,7 +85,7 @@ jobs: with: persist-credentials: false fetch-depth: 0 - - uses: actions-rust-lang/setup-rust-toolchain@9d7e65c320fdb52dcd45ffaa68deb6c02c8754d9 # v1.12.0 + - uses: actions-rust-lang/setup-rust-toolchain@1780873c7b576612439a134613cc4cc74ce5538c # v1.15.2 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 name: Install Python with: diff --git a/.github/workflows/build_python_3.yml b/.github/workflows/build_python_3.yml index bb80dd7e85e..b9bfd2d5a0b 100644 --- a/.github/workflows/build_python_3.yml +++ b/.github/workflows/build_python_3.yml @@ -56,7 +56,7 @@ jobs: - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.13" - - run: pip install cibuildwheel==3.1.4 + - run: pip install cibuildwheel==3.4.0 - id: set-matrix env: CIBW_BUILD: ${{ inputs.cibw_build }} @@ -141,7 +141,7 @@ jobs: platforms: all - name: Build wheels - uses: pypa/cibuildwheel@c923d83ad9c1bc00211c5041d0c3f73294ff88f6 # v3.1.4 + uses: pypa/cibuildwheel@ee02a1537ce3071a004a6b08c41e72f0fdc42d9a # v3.4.0 with: only: ${{ matrix.only }} diff --git a/.github/workflows/encoders-profile.yml b/.github/workflows/encoders-profile.yml deleted file mode 100644 index 6963600f668..00000000000 --- a/.github/workflows/encoders-profile.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: encoders-profile -on: - push: - branches: - - main - - 'mq-working-branch**' - pull_request: - paths: - - 'ddtrace/internal/_encoding.pyx' - - 'scripts/profiles/encoders/**' - - '.github/workflows/encoders-profile.yml' - -jobs: - encoders-profile: - runs-on: ubuntu-latest - env: - PREFIX: ${{ github.workspace }}/prefix - defaults: - run: - working-directory: ddtrace - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - path: ddtrace - - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.10" - - - name: Setup - run: | - bash scripts/profiles/encoders/setup.sh ${PREFIX} - - - name: Run - run: | - bash scripts/profiles/encoders/run.sh ${PREFIX} - for a in `ls ${PREFIX}/artifacts` - do - sed -i 's|${{ github.workspace }}/ddtrace/||g' ${PREFIX}/artifacts/$a - done - - - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: encoders-profile - path: ${{ github.workspace }}/prefix/artifacts diff --git a/.github/workflows/pytorch_gpu_tests.yml b/.github/workflows/pytorch_gpu_tests.yml index 229a63cbefa..42d2587287f 100644 --- a/.github/workflows/pytorch_gpu_tests.yml +++ b/.github/workflows/pytorch_gpu_tests.yml @@ -27,7 +27,7 @@ jobs: with: python-version: '3.12' - - uses: actions-rust-lang/setup-rust-toolchain@9d7e65c320fdb52dcd45ffaa68deb6c02c8754d9 # v1.12.0 + - uses: actions-rust-lang/setup-rust-toolchain@1780873c7b576612439a134613cc4cc74ce5538c # v1.15.2 - name: Install latest stable toolchain and rustfmt run: rustup update stable && rustup default stable && rustup component add rustfmt clippy diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 7451f923229..41aaf0cf246 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -25,7 +25,7 @@ jobs: persist-credentials: false fetch-depth: 0 - - uses: actions-rust-lang/setup-rust-toolchain@9d7e65c320fdb52dcd45ffaa68deb6c02c8754d9 # v1.12.0 + - uses: actions-rust-lang/setup-rust-toolchain@1780873c7b576612439a134613cc4cc74ce5538c # v1.15.2 - name: Install latest stable toolchain and rustfmt run: rustup update stable && rustup default stable && rustup component add rustfmt clippy diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index db882d67353..e93a1e977ec 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,9 +1,12 @@ stages: - package + - shared-pipeline-build - tests - shared-pipeline + - shared-pipeline-test - benchmarks - release + - shared-pipeline-publish variables: REPO_LANG: python # "python" is used everywhere rather than "py" @@ -14,7 +17,7 @@ variables: DD_VPA_TEMPLATE: "vpa-template-cpu-p70-10percent-2x-oom-min-cap" # CI_DEBUG_SERVICES: "true" # Automatically managed, use scripts/update-system-tests-version to update - SYSTEM_TESTS_REF: "9daccd578f30ad15d66011b3550f4f35b2e9be52" + SYSTEM_TESTS_REF: "f5a10b7a61f4ef7ff99765335c2989e211e10af6" default: interruptible: true @@ -187,6 +190,15 @@ publish-lib-init-pinned-tags: - job: generate-lib-init-pinned-tag-values artifacts: true +# Keep the existing 3.19 publishing flow without mutable-tag rollouts. +rollout-mutable-tags-staging: + rules: + - when: never + +rollout-mutable-tags-prod: + rules: + - when: never + configure_system_tests: variables: SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,docker-ssi,lib-injection" diff --git a/.gitlab/benchmarks/bp-runner.microbenchmarks.fail-on-breach.yml b/.gitlab/benchmarks/bp-runner.microbenchmarks.fail-on-breach.yml index 0d98eba7b4a..d7136601c5d 100644 --- a/.gitlab/benchmarks/bp-runner.microbenchmarks.fail-on-breach.yml +++ b/.gitlab/benchmarks/bp-runner.microbenchmarks.fail-on-breach.yml @@ -10,717 +10,544 @@ experiments: - name: coreapiscenario-context_with_data_listeners thresholds: - execution_time < 0.02 ms - - max_rss_usage < 38.00 MB - name: coreapiscenario-context_with_data_no_listeners thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: coreapiscenario-get_item_exists thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: coreapiscenario-get_item_missing thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: coreapiscenario-set_item thresholds: - execution_time < 0.03 ms - - max_rss_usage < 38.00 MB # djangosimple - name: djangosimple-appsec thresholds: - execution_time < 22.30 ms - - max_rss_usage < 70.50 MB - name: djangosimple-exception-replay-enabled thresholds: - execution_time < 1.45 ms - - max_rss_usage < 67.50 MB - name: djangosimple-iast thresholds: - execution_time < 22.25 ms - - max_rss_usage < 70.00 MB - name: djangosimple-profiler thresholds: - execution_time < 16.55 ms - - max_rss_usage < 57.50 MB - name: djangosimple-span-code-origin thresholds: - execution_time < 28.20 ms - - max_rss_usage < 71.00 MB - name: djangosimple-tracer thresholds: - execution_time < 21.75 ms - - max_rss_usage < 70.00 MB - name: djangosimple-tracer-minimal thresholds: - execution_time < 17.50 ms - - max_rss_usage < 70.00 MB - name: djangosimple-tracer-native thresholds: - execution_time < 21.75 ms - - max_rss_usage < 72.50 MB - name: djangosimple-tracer-and-profiler thresholds: - execution_time < 23.50 ms - - max_rss_usage < 71.00 MB - name: djangosimple-tracer-no-caches thresholds: - execution_time < 19.65 ms - - max_rss_usage < 70.00 MB - name: djangosimple-tracer-no-databases thresholds: - execution_time < 20.10 ms - - max_rss_usage < 70.00 MB - name: djangosimple-tracer-dont-create-db-spans thresholds: - execution_time < 21.50 ms - - max_rss_usage < 70.00 MB - name: djangosimple-tracer-no-middleware thresholds: - execution_time < 21.50 ms - - max_rss_usage < 70.00 MB - name: djangosimple-tracer-no-templates thresholds: - execution_time < 22.00 ms - - max_rss_usage < 70.50 MB - name: djangosimple-resource-renaming thresholds: - execution_time < 21.75 ms - - max_rss_usage < 70.50 MB # errortrackingdjangosimple - name: errortrackingdjangosimple-errortracking-enabled-all thresholds: - execution_time < 19.85 ms - - max_rss_usage < 70.00 MB - name: errortrackingdjangosimple-errortracking-enabled-user thresholds: - execution_time < 19.40 ms - - max_rss_usage < 70.00 MB - name: errortrackingdjangosimple-tracer-enabled thresholds: - execution_time < 19.45 ms - - max_rss_usage < 70.00 MB # errortrackingflasksqli - name: errortrackingflasksqli-errortracking-enabled-all thresholds: - execution_time < 2.30 ms - - max_rss_usage < 56.50 MB - name: errortrackingflasksqli-errortracking-enabled-user thresholds: - execution_time < 2.25 ms - - max_rss_usage < 56.50 MB - name: errortrackingflasksqli-tracer-enabled thresholds: - execution_time < 2.30 ms - - max_rss_usage < 56.50 MB # flask_simple - name: flasksimple-tracer thresholds: - execution_time < 3.65 ms - - max_rss_usage < 56.50 MB - name: flasksimple-tracer-native thresholds: - execution_time < 3.65 ms - - max_rss_usage < 60.00 MB - name: flasksimple-profiler thresholds: - execution_time < 2.10 ms - - max_rss_usage < 50.00 MB - name: flasksimple-debugger thresholds: - execution_time < 2.00 ms - - max_rss_usage < 49.50 MB - name: flasksimple-iast-get thresholds: - execution_time < 2.00 ms - - max_rss_usage < 49.00 MB - name: flasksimple-appsec-get thresholds: - execution_time < 4.75 ms - - max_rss_usage < 66.50 MB - name: flasksimple-appsec-post thresholds: - execution_time < 6.75 ms - - max_rss_usage < 66.50 MB - name: flasksimple-appsec-telemetry thresholds: - execution_time < 4.75 ms - - max_rss_usage < 66.50 MB - name: flasksimple-resource-renaming thresholds: - execution_time < 3.65 ms - - max_rss_usage < 56.00 MB # flasksqli - name: flasksqli-appsec-enabled thresholds: - execution_time < 4.20 ms - - max_rss_usage < 66.00 MB - name: flasksqli-iast-enabled thresholds: - execution_time < 2.80 ms - - max_rss_usage < 62.50 MB - name: flasksqli-tracer-enabled thresholds: - execution_time < 2.25 ms - - max_rss_usage < 56.50 MB # httppropagationextract - name: httppropagationextract-all_styles_all_headers thresholds: - execution_time < 0.10 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-b3_headers thresholds: - execution_time < 0.02 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-b3_single_headers thresholds: - execution_time < 0.02 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-datadog_tracecontext_tracestate_not_propagated_on_trace_id_no_match thresholds: - execution_time < 0.08 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-datadog_tracecontext_tracestate_propagated_on_trace_id_match thresholds: - execution_time < 0.08 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-empty_headers thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-full_t_id_datadog_headers thresholds: - execution_time < 0.03 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-invalid_priority_header thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-invalid_span_id_header thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-invalid_tags_header thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-invalid_trace_id_header thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-large_header_no_matches thresholds: - execution_time < 0.03 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-large_valid_headers_all thresholds: - execution_time < 0.04 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-medium_header_no_matches thresholds: - execution_time < 0.02 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-medium_valid_headers_all thresholds: - execution_time < 0.02 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-none_propagation_style thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-tracecontext_headers thresholds: - execution_time < 0.04 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-valid_headers_all thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-valid_headers_basic thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-wsgi_empty_headers thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-wsgi_invalid_priority_header thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-wsgi_invalid_span_id_header thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-wsgi_invalid_tags_header thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-wsgi_invalid_trace_id_header thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-wsgi_large_header_no_matches thresholds: - execution_time < 0.04 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-wsgi_large_valid_headers_all thresholds: - execution_time < 0.04 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-wsgi_medium_header_no_matches thresholds: - execution_time < 0.02 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-wsgi_medium_valid_headers_all thresholds: - execution_time < 0.02 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-wsgi_valid_headers_all thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: httppropagationextract-wsgi_valid_headers_basic thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB # httppropagationinject - name: httppropagationinject-ids_only thresholds: - execution_time < 0.03 ms - - max_rss_usage < 38.00 MB - name: httppropagationinject-with_all thresholds: - execution_time < 0.04 ms - - max_rss_usage < 38.00 MB - name: httppropagationinject-with_dd_origin thresholds: - execution_time < 0.03 ms - - max_rss_usage < 38.00 MB - name: httppropagationinject-with_priority_and_origin thresholds: - execution_time < 0.04 ms - - max_rss_usage < 38.00 MB - name: httppropagationinject-with_sampling_priority thresholds: - execution_time < 0.03 ms - - max_rss_usage < 38.00 MB - name: httppropagationinject-with_tags thresholds: - execution_time < 0.04 ms - - max_rss_usage < 38.00 MB - name: httppropagationinject-with_tags_invalid thresholds: - execution_time < 0.04 ms - - max_rss_usage < 38.00 MB - name: httppropagationinject-with_tags_max_size thresholds: - execution_time < 0.04 ms - - max_rss_usage < 38.00 MB # iast_aspects - name: iast_aspects-re_expand_aspect thresholds: - execution_time < 0.04 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_expand_noaspect thresholds: - execution_time < 0.04 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_findall_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_findall_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_finditer_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_finditer_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_fullmatch_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_fullmatch_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_group_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_group_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_groups_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_groups_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_match_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_match_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_search_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_search_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_sub_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_sub_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_subn_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iast_aspects-re_subn_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB # iastaspects - name: iastaspects-add_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-add_inplace_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-add_inplace_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-add_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-bytearray_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-bytearray_extend_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-bytearray_extend_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-bytearray_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-bytes_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-bytes_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-bytesio_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-bytesio_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-capitalize_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-capitalize_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-casefold_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-casefold_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-decode_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-decode_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-encode_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-encode_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-format_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-format_map_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-format_map_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-format_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-index_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-index_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-join_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-join_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-ljust_aspect thresholds: - execution_time < 0.02 ms - - max_rss_usage < 41.50 MB - name: iastaspects-ljust_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-lower_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-lower_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-lstrip_aspect thresholds: - execution_time < 0.02 ms - - max_rss_usage < 41.50 MB - name: iastaspects-lstrip_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-modulo_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-modulo_aspect_for_bytearray_bytearray thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-modulo_aspect_for_bytes thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-modulo_aspect_for_bytes_bytearray thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-modulo_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-replace_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-replace_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-repr_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-repr_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-rstrip_aspect thresholds: - execution_time < 0.02 ms - - max_rss_usage < 41.50 MB - name: iastaspects-rstrip_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-slice_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-slice_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-stringio_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-stringio_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-strip_aspect thresholds: - execution_time < 0.02 ms - - max_rss_usage < 41.50 MB - name: iastaspects-strip_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-swapcase_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-swapcase_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-title_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-title_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-translate_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-translate_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-upper_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB - name: iastaspects-upper_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.50 MB # iastaspectsospath - name: iastaspectsospath-ospathbasename_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iastaspectsospath-ospathbasename_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iastaspectsospath-ospathjoin_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iastaspectsospath-ospathjoin_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iastaspectsospath-ospathnormcase_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iastaspectsospath-ospathnormcase_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iastaspectsospath-ospathsplit_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iastaspectsospath-ospathsplit_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iastaspectsospath-ospathsplitdrive_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iastaspectsospath-ospathsplitdrive_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iastaspectsospath-ospathsplitext_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iastaspectsospath-ospathsplitext_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB # iastaspectssplit - name: iastaspectssplit-rsplit_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iastaspectssplit-rsplit_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iastaspectssplit-split_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iastaspectssplit-split_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iastaspectssplit-splitlines_aspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: iastaspectssplit-splitlines_noaspect thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB # iastpropagation # - name: iastpropagation-no-propagation @@ -744,411 +571,313 @@ experiments: - name: otelsdkspan-add-event thresholds: - execution_time < 42.00 ms - - max_rss_usage < 39.00 MB - name: otelsdkspan-add-link thresholds: - execution_time < 38.55 ms - - max_rss_usage < 39.00 MB - name: otelsdkspan-add-metrics thresholds: - execution_time < 232.00 ms - - max_rss_usage < 39.00 MB - name: otelsdkspan-add-tags thresholds: - execution_time < 221.60 ms - - max_rss_usage < 39.00 MB - name: otelsdkspan-get-context thresholds: - execution_time < 31.30 ms - - max_rss_usage < 39.00 MB - name: otelsdkspan-is-recording thresholds: - execution_time < 31.00 ms - - max_rss_usage < 39.00 MB - name: otelsdkspan-record-exception thresholds: - execution_time < 65.85 ms - - max_rss_usage < 39.00 MB - name: otelsdkspan-set-status thresholds: - execution_time < 34.15 ms - - max_rss_usage < 39.00 MB - name: otelsdkspan-start thresholds: - execution_time < 30.15 ms - - max_rss_usage < 39.00 MB - name: otelsdkspan-start-finish thresholds: - execution_time < 35.35 ms - - max_rss_usage < 39.00 MB - name: otelsdkspan-start-finish-telemetry thresholds: - execution_time < 35.45 ms - - max_rss_usage < 39.00 MB - name: otelsdkspan-update-name thresholds: - execution_time < 33.40 ms - - max_rss_usage < 39.00 MB # otelspan - name: otelspan-add-event thresholds: - execution_time < 47.15 ms - - max_rss_usage < 47.00 MB - name: otelspan-add-metrics thresholds: - execution_time < 344.80 ms - - max_rss_usage < 47.50 MB - name: otelspan-add-tags thresholds: - execution_time < 314.00 ms - - max_rss_usage < 47.50 MB - name: otelspan-get-context thresholds: - execution_time < 92.35 ms - - max_rss_usage < 46.50 MB - name: otelspan-is-recording thresholds: - execution_time < 44.50 ms - - max_rss_usage < 47.50 MB - name: otelspan-record-exception thresholds: - execution_time < 67.65 ms - - max_rss_usage < 47.00 MB - name: otelspan-set-status thresholds: - execution_time < 50.40 ms - - max_rss_usage < 47.00 MB - name: otelspan-start thresholds: - execution_time < 43.45 ms - - max_rss_usage < 47.00 MB - name: otelspan-start-finish thresholds: - execution_time < 88.00 ms - - max_rss_usage < 46.50 MB - name: otelspan-start-finish-telemetry thresholds: - execution_time < 89.00 ms - - max_rss_usage < 46.50 MB - name: otelspan-update-name thresholds: - execution_time < 45.15 ms - - max_rss_usage < 47.00 MB # packagespackageforrootmodulemapping - name: packagespackageforrootmodulemapping-cache_off thresholds: - execution_time < 354.30 ms - - max_rss_usage < 41.50 MB - name: packagespackageforrootmodulemapping-cache_on thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB # packagesupdateimporteddependencies - name: packagesupdateimporteddependencies-import_many thresholds: - execution_time < 0.17 ms - - max_rss_usage < 41.00 MB - name: packagesupdateimporteddependencies-import_many_cached thresholds: - execution_time < 0.13 ms - - max_rss_usage < 41.00 MB - name: packagesupdateimporteddependencies-import_many_stdlib thresholds: - execution_time < 1.75 ms - - max_rss_usage < 41.00 MB - name: packagesupdateimporteddependencies-import_many_stdlib_cached thresholds: - execution_time < 1.10 ms - - max_rss_usage < 41.00 MB - name: packagesupdateimporteddependencies-import_many_unknown thresholds: - execution_time < 0.89 ms - - max_rss_usage < 41.00 MB - name: packagesupdateimporteddependencies-import_many_unknown_cached thresholds: - execution_time < 0.87 ms - - max_rss_usage < 41.00 MB - name: packagesupdateimporteddependencies-import_one thresholds: - execution_time < 0.03 ms - - max_rss_usage < 41.00 MB - name: packagesupdateimporteddependencies-import_one_cache thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: packagesupdateimporteddependencies-import_one_stdlib thresholds: - execution_time < 0.02 ms - - max_rss_usage < 41.00 MB - name: packagesupdateimporteddependencies-import_one_stdlib_cache thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB - name: packagesupdateimporteddependencies-import_one_unknown thresholds: - execution_time < 0.05 ms - - max_rss_usage < 41.00 MB - name: packagesupdateimporteddependencies-import_one_unknown_cache thresholds: - execution_time < 0.01 ms - - max_rss_usage < 41.00 MB # ratelimiter - name: ratelimiter-defaults thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: ratelimiter-high_rate_limit thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: ratelimiter-long_window thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: ratelimiter-low_rate_limit thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: ratelimiter-no_rate_limit thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB - name: ratelimiter-short_window thresholds: - execution_time < 0.01 ms - - max_rss_usage < 38.00 MB # recursivecomputation - name: recursivecomputation-deep thresholds: - execution_time < 320.95 ms - - max_rss_usage < 46.00 MB - name: recursivecomputation-deep-profiled thresholds: - execution_time < 359.15 ms - - max_rss_usage < 40.50 MB - name: recursivecomputation-medium thresholds: - execution_time < 7.40 ms - - max_rss_usage < 46.00 MB - name: recursivecomputation-shallow thresholds: - execution_time < 1.05 ms - - max_rss_usage < 46.00 MB # samplingrules - name: samplingrules-average_match thresholds: - execution_time < 0.29 ms - - max_rss_usage < 38.00 MB - name: samplingrules-high_match thresholds: - execution_time < 0.48 ms - - max_rss_usage < 38.00 MB - name: samplingrules-low_match thresholds: - execution_time < 0.12 ms - - max_rss_usage < 700.00 MB - name: samplingrules-very_low_match thresholds: - execution_time < 8.50 ms - - max_rss_usage < 75.00 MB # sethttpmeta - name: sethttpmeta-all-disabled thresholds: - execution_time < 0.02 ms - - max_rss_usage < 42.00 MB - name: sethttpmeta-all-enabled thresholds: - execution_time < 0.05 ms - - max_rss_usage < 42.00 MB - name: sethttpmeta-collectipvariant_exists thresholds: - execution_time < 0.05 ms - - max_rss_usage < 42.00 MB - name: sethttpmeta-no-collectipvariant thresholds: - execution_time < 0.05 ms - - max_rss_usage < 42.00 MB - name: sethttpmeta-no-useragentvariant thresholds: - execution_time < 0.05 ms - - max_rss_usage < 42.00 MB - name: sethttpmeta-obfuscation-no-query thresholds: - execution_time < 0.05 ms - - max_rss_usage < 42.00 MB - name: sethttpmeta-obfuscation-regular-case-explicit-query thresholds: - execution_time < 0.09 ms - - max_rss_usage < 42.00 MB - name: sethttpmeta-obfuscation-regular-case-implicit-query thresholds: - execution_time < 0.09 ms - - max_rss_usage < 42.00 MB - name: sethttpmeta-obfuscation-send-querystring-disabled thresholds: - - execution_time < 0.17 ms - - max_rss_usage < 42.00 MB + - execution_time < 0.19 ms - name: sethttpmeta-obfuscation-worst-case-explicit-query thresholds: - - execution_time < 0.16 ms - - max_rss_usage < 42.00 MB + - execution_time < 0.19 ms - name: sethttpmeta-obfuscation-worst-case-implicit-query thresholds: - - execution_time < 0.17 ms - - max_rss_usage < 42.00 MB + - execution_time < 0.19 ms - name: sethttpmeta-useragentvariant_exists_1 thresholds: - execution_time < 0.05 ms - - max_rss_usage < 42.00 MB - name: sethttpmeta-useragentvariant_exists_2 thresholds: - execution_time < 0.05 ms - - max_rss_usage < 42.00 MB - name: sethttpmeta-useragentvariant_exists_3 thresholds: - execution_time < 0.05 ms - - max_rss_usage < 42.00 MB - name: sethttpmeta-useragentvariant_not_exists_1 thresholds: - execution_time < 0.05 ms - - max_rss_usage < 42.00 MB - name: sethttpmeta-useragentvariant_not_exists_2 thresholds: - execution_time < 0.05 ms - - max_rss_usage < 42.00 MB # span - name: span-add-event thresholds: - execution_time < 22.50 ms - - max_rss_usage < 53.00 MB - name: span-add-metrics thresholds: - execution_time < 93.50 ms - - max_rss_usage < 53.00 MB - name: span-add-tags thresholds: - execution_time < 155.00 ms - - max_rss_usage < 53.00 MB - name: span-get-context thresholds: - execution_time < 20.50 ms - - max_rss_usage < 53.00 MB - name: span-is-recording thresholds: - execution_time < 20.50 ms - - max_rss_usage < 53.00 MB - name: span-record-exception thresholds: - execution_time < 40.00 ms - - max_rss_usage < 53.00 MB - name: span-set-status thresholds: - execution_time < 22.00 ms - - max_rss_usage < 53.00 MB - name: span-start thresholds: - execution_time < 20.50 ms - - max_rss_usage < 53.00 MB - name: span-start-finish thresholds: - execution_time < 52.50 ms - - max_rss_usage < 53.00 MB - name: span-start-finish-telemetry thresholds: - execution_time < 54.50 ms - - max_rss_usage < 53.00 MB - name: span-start-finish-traceid128 thresholds: - execution_time < 57.00 ms - - max_rss_usage < 53.00 MB - name: span-start-traceid128 thresholds: - execution_time < 22.50 ms - - max_rss_usage < 53.00 MB - name: span-update-name thresholds: - execution_time < 22.00 ms - - max_rss_usage < 53.00 MB # telemetryaddmetric - name: telemetryaddmetric-1-count-metric-1-times thresholds: - execution_time < 0.02 ms - - max_rss_usage < 38.00 MB - name: telemetryaddmetric-1-count-metrics-100-times thresholds: - execution_time < 0.35 ms - - max_rss_usage < 38.00 MB - name: telemetryaddmetric-1-distribution-metric-1-times thresholds: - execution_time < 0.02 ms - - max_rss_usage < 38.00 MB - name: telemetryaddmetric-1-distribution-metrics-100-times thresholds: - execution_time < 0.36 ms - - max_rss_usage < 38.00 MB - name: telemetryaddmetric-1-gauge-metric-1-times thresholds: - execution_time < 0.02 ms - - max_rss_usage < 38.00 MB - name: telemetryaddmetric-1-gauge-metrics-100-times thresholds: - execution_time < 0.15 ms - - max_rss_usage < 38.00 MB - name: telemetryaddmetric-1-rate-metric-1-times thresholds: - execution_time < 0.02 ms - - max_rss_usage < 38.00 MB - name: telemetryaddmetric-1-rate-metrics-100-times thresholds: - execution_time < 0.36 ms - - max_rss_usage < 38.00 MB - name: telemetryaddmetric-100-count-metrics-100-times thresholds: - execution_time < 33.86 ms - - max_rss_usage < 38.00 MB - name: telemetryaddmetric-100-distribution-metrics-100-times thresholds: - execution_time < 3.59 ms - - max_rss_usage < 38.00 MB - name: telemetryaddmetric-100-gauge-metrics-100-times thresholds: - execution_time < 1.55 ms - - max_rss_usage < 38.00 MB - name: telemetryaddmetric-100-rate-metrics-100-times thresholds: - execution_time < 3.61 ms - - max_rss_usage < 38.00 MB - name: telemetryaddmetric-flush-1-metric thresholds: - execution_time < 0.02 ms - - max_rss_usage < 38.00 MB - name: telemetryaddmetric-flush-100-metrics thresholds: - execution_time < 0.25 ms - - max_rss_usage < 40.00 MB - name: telemetryaddmetric-flush-1000-metrics thresholds: - execution_time < 2.50 ms - - max_rss_usage < 41.00 MB # tracer - name: tracer-large thresholds: - execution_time < 32.95 ms - - max_rss_usage < 39.25 MB - name: tracer-medium thresholds: - execution_time < 3.20 ms - - max_rss_usage < 38.75 MB - name: tracer-small thresholds: - execution_time < 0.37 ms - - max_rss_usage < 38.75 MB diff --git a/.gitlab/one-pipeline.locked.yml b/.gitlab/one-pipeline.locked.yml index ccf6f04825e..d0dad3eaba2 100644 --- a/.gitlab/one-pipeline.locked.yml +++ b/.gitlab/one-pipeline.locked.yml @@ -1,4 +1,4 @@ # DO NOT EDIT THIS FILE MANUALLY # This file is auto-generated by automation. include: - - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/f9ee2fa697e8b0c440dc9762cc522cafbe295a9671d49067f38258fbee985c74/one-pipeline.yml + - remote: https://gitlab-templates.ddbuild.io/libdatadog/include/versions/1.6.0/one-pipeline.yml diff --git a/ddtrace/contrib/internal/ray/patch.py b/ddtrace/contrib/internal/ray/patch.py index 6e607286d16..5682b454c3a 100644 --- a/ddtrace/contrib/internal/ray/patch.py +++ b/ddtrace/contrib/internal/ray/patch.py @@ -193,7 +193,7 @@ def traced_submit_job(wrapped, instance, args, kwargs): of creating the root span. It will also inject _RAY_SUBMISSION_ID and _RAY_JOB_NAME in the env variable as some spans will not have access to them - trough ray_ctx + through ray_ctx """ from ray.dashboard.modules.job.job_manager import generate_job_id diff --git a/ddtrace/contrib/internal/ray/utils.py b/ddtrace/contrib/internal/ray/utils.py index d6405e45320..64448d412e1 100644 --- a/ddtrace/contrib/internal/ray/utils.py +++ b/ddtrace/contrib/internal/ray/utils.py @@ -237,7 +237,7 @@ def get_signature(func: Any) -> inspect.Signature: it is relatively stable. Future versions of Python may allow overloading the inspect 'isfunction' and 'ismethod' functions / create ABC for Python functions. Until then, it appears that Cython won't do anything about - compatability with the inspect module. + compatibility with the inspect module. Args: func: The function whose signature should be checked. diff --git a/ddtrace/contrib/internal/trace_utils_base.py b/ddtrace/contrib/internal/trace_utils_base.py index 3d784b37325..a31c8a22f2e 100644 --- a/ddtrace/contrib/internal/trace_utils_base.py +++ b/ddtrace/contrib/internal/trace_utils_base.py @@ -154,4 +154,12 @@ def _set_url_tag(integration_config: IntegrationConfig, span: Span, url: str, qu # obfuscation is disabled when DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP="" span._set_tag_str(http.URL, strip_query_string(url)) else: - span._set_tag_str(http.URL, redact_url(url, config._obfuscation_query_string_pattern, query)) + span._set_tag_str( + http.URL, + redact_url( + url, + config._obfuscation_query_string_pattern, + query, + preserve_delimiter=config._query_string_obfuscation_preserve_delimiter, + ), + ) diff --git a/ddtrace/internal/_encoding.pyx b/ddtrace/internal/_encoding.pyx index 6b11aaf94d1..20d90ff86f5 100644 --- a/ddtrace/internal/_encoding.pyx +++ b/ddtrace/internal/_encoding.pyx @@ -21,7 +21,6 @@ from ._utils cimport PyBytesLike_Check from ..constants import _ORIGIN_KEY as ORIGIN_KEY from .constants import SPAN_LINKS_KEY from .constants import SPAN_EVENTS_KEY -from .constants import MAX_UINT_64BITS from .._trace._limits import MAX_SPAN_META_VALUE_LEN from .._trace._limits import TRUNCATED_SPAN_ATTRIBUTE_LEN from ..settings._agent import config as agent_config @@ -1190,7 +1189,6 @@ cdef class Packer(object): cdef int _pack(self, object o) except -1: cdef long long llval cdef unsigned long long ullval - cdef long longval cdef double dval cdef char* rawval cdef int ret diff --git a/ddtrace/internal/dist_computing/utils.py b/ddtrace/internal/dist_computing/utils.py index 8498456f720..28b0bf185c5 100644 --- a/ddtrace/internal/dist_computing/utils.py +++ b/ddtrace/internal/dist_computing/utils.py @@ -3,7 +3,7 @@ def in_ray_job(): # type: () -> bool - """Returns whether we are in a ray environemt. + """Returns whether we are in a ray environment. This is accomplished by checking if the _RAY_SUBMISSION_ID environment variable is defined which means a job has been submitted and is traced """ diff --git a/ddtrace/internal/utils/http.py b/ddtrace/internal/utils/http.py index ea1aaa13bf2..cb72aa07bf7 100644 --- a/ddtrace/internal/utils/http.py +++ b/ddtrace/internal/utils/http.py @@ -76,21 +76,32 @@ def strip_query_string(url): return h + fs + f -def redact_query_string(query_string, query_string_obfuscation_pattern): - # type: (str, re.Pattern) -> Union[bytes, str] +def _redact_query_string_match(match): + # type: (re.Match) -> bytes + return (match.group(1) or b"") + b"" + + +def redact_query_string(query_string, query_string_obfuscation_pattern, *, preserve_delimiter=False): + # type: (str, re.Pattern, bool) -> Union[bytes, str] bytes_query = query_string if isinstance(query_string, bytes) else query_string.encode("utf-8") - return query_string_obfuscation_pattern.sub(b"", bytes_query) + return query_string_obfuscation_pattern.sub( + _redact_query_string_match if preserve_delimiter else b"", bytes_query + ) -def redact_url(url, query_string_obfuscation_pattern, query_string=None): - # type: (str, re.Pattern, Optional[str]) -> Union[str,bytes] +def redact_url(url, query_string_obfuscation_pattern, query_string=None, *, preserve_delimiter=False): + # type: (str, re.Pattern, Optional[str], bool) -> Union[str,bytes] parts = parse.urlparse(url) redacted_query = None if query_string: - redacted_query = redact_query_string(query_string, query_string_obfuscation_pattern) + redacted_query = redact_query_string( + query_string, query_string_obfuscation_pattern, preserve_delimiter=preserve_delimiter + ) elif parts.query: - redacted_query = redact_query_string(parts.query, query_string_obfuscation_pattern) + redacted_query = redact_query_string( + parts.query, query_string_obfuscation_pattern, preserve_delimiter=preserve_delimiter + ) if redacted_query is not None and len(parts) >= 5: redacted_parts = parts[:4] + (redacted_query,) + parts[5:] # type: Tuple[Union[str, bytes], ...] diff --git a/ddtrace/settings/_config.py b/ddtrace/settings/_config.py index 81588a4e84e..d25853c7615 100644 --- a/ddtrace/settings/_config.py +++ b/ddtrace/settings/_config.py @@ -48,7 +48,7 @@ ENDPOINT_FETCHED_CONFIG = fetch_config_from_endpoint() DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP_DEFAULT = ( - r"(?ix)" + r"(?i)" r"(?:" # JSON-ish leading quote r'(?:"|%22)?' r")" @@ -81,10 +81,11 @@ r'(?:"|%22)' # closing '"' at end of value r")" r"|(?:" # other common secret values - r" bearer(?:\s|%20)+[a-z0-9._\-]+" + r"bearer(?:\s|%20)+[a-z0-9._\-]+" r"|token(?::|%3A)[a-z0-9]{13}" r"|gh[opsu]_[0-9a-zA-Z]{36}" - r"|ey[I-L](?:[\w=-]|%3D)+\.ey[I-L](?:[\w=-]|%3D)+(?:\.(?:[\w.+/=-]|%3D|%2F|%2B)+)?" + r"|(^|[^\w%-]|%[0-9a-f]{2})ey[I-L][\w-]+(?:=|%3D)*\.ey[I-L][\w-]+(?:=|%3D)*" + r"(?:\.(?:[\w.+/=-]|%3D|%2F|%2B)+)?" r"|-{5}BEGIN(?:[a-z\s]|%20)+PRIVATE(?:\s|%20)KEY-{5}[^\-]+-{5}END" r"(?:[a-z\s]|%20)+PRIVATE(?:\s|%20)KEY(?:-{5})?(?:\n|%0A)?" r"|(?:ssh-(?:rsa|dss)|ecdsa-[a-z0-9]+-[a-z0-9]+)(?:\s|%20|%09)+(?:[a-z0-9/.+]" @@ -615,6 +616,9 @@ def __init__(self): dd_trace_obfuscation_query_string_regexp = _get_config( "DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP", DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP_DEFAULT ) + self._query_string_obfuscation_preserve_delimiter = ( + dd_trace_obfuscation_query_string_regexp == DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP_DEFAULT + ) self._global_query_string_obfuscation_disabled = dd_trace_obfuscation_query_string_regexp == "" self._obfuscation_query_string_pattern = None self._http_tag_query_string = True # Default behaviour of query string tagging in http.url diff --git a/docs/configuration.rst b/docs/configuration.rst index a729c222c54..a75167bdc5c 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -239,7 +239,7 @@ Traces DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP: default: | - ``'(?ix)(?:(?:"|%22)?)(?:(?:old[-_]?|new[-_]?)?p(?:ass)?w(?:or)?d(?:1|2)?|pass(?:[-_]?phrase)?|secret|(?:api[-_]?|private[-_]?|public[-_]?|access[-_]?|secret[-_]?)key(?:[-_]?id)?|token|consumer[-_]?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:(?:\\s|%20)*(?:=|%3D)[^&]+|(?:"|%22)(?:\\s|%20)*(?::|%3A)(?:\\s|%20)*(?:"|%22)(?:%2[^2]|%[^2]|[^"%])+(?:"|%22))|(?: bearer(?:\\s|%20)+[a-z0-9._\\-]+|token(?::|%3A)[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L](?:[\\w=-]|%3D)+\\.ey[I-L](?:[\\w=-]|%3D)+(?:\\.(?:[\\w.+/=-]|%3D|%2F|%2B)+)?|-{5}BEGIN(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY-{5}[^\\-]+-{5}END(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY(?:-{5})?(?:\\n|%0A)?|(?:ssh-(?:rsa|dss)|ecdsa-[a-z0-9]+-[a-z0-9]+)(?:\\s|%20|%09)+(?:[a-z0-9/.+]|%2F|%5C|%2B){100,}(?:=|%3D)*(?:(?:\\s|%20|%09)+[a-z0-9._-]+)?)'`` + ``'(?i)(?:(?:"|%22)?)(?:(?:old[-_]?|new[-_]?)?p(?:ass)?w(?:or)?d(?:1|2)?|pass(?:[-_]?phrase)?|secret|(?:api[-_]?|private[-_]?|public[-_]?|access[-_]?|secret[-_]?|app(?:lication)?[-_]?)key(?:[-_]?id)?|token|consumer[-_]?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:(?:\\s|%20)*(?:=|%3D)[^&]+|(?:"|%22)(?:\\s|%20)*(?::|%3A)(?:\\s|%20)*(?:"|%22)(?:%2[^2]|%[^2]|[^"%])+(?:"|%22))|(?:bearer(?:\\s|%20)+[a-z0-9._\\-]+|token(?::|%3A)[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|(^|[^\\w%-]|%[0-9a-f]{2})ey[I-L][\\w-]+(?:=|%3D)*\\.ey[I-L][\\w-]+(?:=|%3D)*(?:\\.(?:[\\w.+/=-]|%3D|%2F|%2B)+)?|-{5}BEGIN(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY-{5}[^\\-]+-{5}END(?:[a-z\\s]|%20)+PRIVATE(?:\\s|%20)KEY(?:-{5})?(?:\\n|%0A)?|(?:ssh-(?:rsa|dss)|ecdsa-[a-z0-9]+-[a-z0-9]+)(?:\\s|%20|%09)+(?:[a-z0-9/.+]|%2F|%5C|%2B){100,}(?:=|%3D)*(?:(?:\\s|%20|%09)+[a-z0-9._-]+)?)'`` description: A regexp to redact sensitive query strings. Obfuscation disabled if set to empty string diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 5161021023d..74c90b131cf 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -169,6 +169,7 @@ langchain langchain_community libdatadog libddwaf +lication lifecycle linters litellm diff --git a/hatch.toml b/hatch.toml index e46a08acacc..84791b8d796 100644 --- a/hatch.toml +++ b/hatch.toml @@ -54,7 +54,7 @@ checks = [ "spelling", "riot", "security", - "test", + "hatch run scripts:test", "suitespec-check", "error-log-check", "sg", diff --git a/releasenotes/notes/fix-query-string-obfuscation-ca8726c83a8d9667.yaml b/releasenotes/notes/fix-query-string-obfuscation-ca8726c83a8d9667.yaml new file mode 100644 index 00000000000..0cadef478f6 --- /dev/null +++ b/releasenotes/notes/fix-query-string-obfuscation-ca8726c83a8d9667.yaml @@ -0,0 +1,4 @@ +security: + - | + tracing: Fixes a security vulnerability, as described in + `GHSA-3633-r77q-h3v3 `_. diff --git a/releasenotes/notes/fix-serverless-service-representation-b1f81fe0553a7a45.yaml b/releasenotes/notes/fix-serverless-service-representation-b1f81fe0553a7a45.yaml index 0535ee9e8cd..5e1ad3c5800 100644 --- a/releasenotes/notes/fix-serverless-service-representation-b1f81fe0553a7a45.yaml +++ b/releasenotes/notes/fix-serverless-service-representation-b1f81fe0553a7a45.yaml @@ -1,7 +1,7 @@ --- features: - | - aws: Set peer.service explictly to improve the accuracy of serverless + aws: Set peer.service explicitly to improve the accuracy of serverless service representation. Base_service defaults to unhelpful value "runtime" in serverless spans. Remove base_service to prevent unwanted service overrides in Lambda spans. diff --git a/scripts/gen_gitlab_config.py b/scripts/gen_gitlab_config.py index c078b15f24b..78f355f71a6 100644 --- a/scripts/gen_gitlab_config.py +++ b/scripts/gen_gitlab_config.py @@ -13,6 +13,18 @@ import typing as t +# Keep VCR-backed and network-behavior tests off the proxy so their behavior stays deterministic. +NO_PROXY_ADDITIONS = ( + "icanhazdadjoke.com", + "doesnotexist.google.com", + "api.stripe.com", + "us-central1-aiplatform.googleapis.com", + "github.com", + "api.github.com", + ".amazonaws.com", +) + + @dataclass class JobSpec: name: str @@ -29,6 +41,7 @@ class JobSpec: allow_failure: bool = False paths: t.Optional[t.Set[str]] = None # ignored only: t.Optional[t.Set[str]] = None # ignored + no_proxy: bool = False def __str__(self) -> str: lines = [] @@ -68,6 +81,14 @@ def __str__(self) -> str: lines.append(" before_script:") lines.append(f" - !reference [{base}, before_script]") lines.append(" - pip cache info") + if self.no_proxy: + no_proxy_additions = ",".join(NO_PROXY_ADDITIONS) + lines.append(" - |") + lines.append(f' no_proxy_additions="{no_proxy_additions}"') + lines.append(' no_proxy_existing="${NO_PROXY:-${no_proxy:-}}"') + lines.append(' export NO_PROXY="${no_proxy_existing:+${no_proxy_existing},}${no_proxy_additions}"') + lines.append(' export no_proxy="${no_proxy_existing:+${no_proxy_existing},}${no_proxy_additions}"') + lines.append(' echo "NO_PROXY=${NO_PROXY}"') if wait_for: if self.runner == "riot" and wait_for: lines.append(f" - riot -v run -s --pass-env wait -- {' '.join(wait_for)}") @@ -269,6 +290,7 @@ def check(name: str, command: str, paths: t.Set[str]) -> None: if not checks: return + no_proxy_additions = ",".join(NO_PROXY_ADDITIONS) with TESTS_GEN.open("a") as f: f.write( """ @@ -276,6 +298,15 @@ def check(name: str, command: str, paths: t.Set[str]) -> None: extends: .testrunner stage: setup needs: [] + before_script: + - !reference [.testrunner, before_script] + - | + no_proxy_additions=""" + + no_proxy_additions + + """ + no_proxy_existing="${NO_PROXY:-${no_proxy:-}}" + export NO_PROXY="${no_proxy_existing:+${no_proxy_existing},}${no_proxy_additions}" + export no_proxy="${no_proxy_existing:+${no_proxy_existing},}${no_proxy_additions}" variables: PIP_CACHE_DIR: '${CI_PROJECT_DIR}/.cache/pip' script: diff --git a/tests/README.md b/tests/README.md index 8dcf61e07f3..03fb5763c56 100644 --- a/tests/README.md +++ b/tests/README.md @@ -45,6 +45,7 @@ The suite schema is as follows: pattern: # The pattern/environment name (if different from the suite name) paths: # The paths/components that trigger the job services: # The services to start before running the suite, defined in .gitlab/services.yml + no_proxy: # Add the suite's external test domains to NO_PROXY and no_proxy ``` For example @@ -72,3 +73,7 @@ Components do not need to be declared within the same `suitespec.yml` file. They can be declared in any file within the `/tests` sub-tree. The CI configuration generator will aggregate all the components and suites to build the full test suite specification and resolve the components after that. + +Set `no_proxy: true` only for suites whose VCR-backed or network-behavior tests need to bypass the +CI proxy. The generated job preserves existing `NO_PROXY` and `no_proxy` values and appends the +standard test-domain exclusions. diff --git a/tests/appsec/appsec/api_security/test_api_security_manager.py b/tests/appsec/appsec/api_security/test_api_security_manager.py index f611ac22540..3a011f2f081 100644 --- a/tests/appsec/appsec/api_security/test_api_security_manager.py +++ b/tests/appsec/appsec/api_security/test_api_security_manager.py @@ -277,7 +277,7 @@ def test_should_collect_schema_route_fallbacks_to_endpoint(self, mock_environmen # First request should collect assert manager._should_collect_schema(mock_environment, USER_KEEP) - # Sencond one should discarded + # Second one should discarded assert not manager._should_collect_schema(mock_environment, USER_KEEP) def test_should_collect_schema_route_missing_computes_endpoint(self, mock_environment): @@ -307,7 +307,7 @@ def get_tag(name): # First request should collect assert manager._should_collect_schema(mock_environment, USER_KEEP) - # Sencond one should be discarded + # Second one should be discarded assert not manager._should_collect_schema(mock_environment, USER_KEEP) @pytest.mark.parametrize("status_code", [404, "404"]) diff --git a/tests/contrib/litellm/cassettes/completion_command-r.yaml b/tests/contrib/litellm/cassettes/completion_cohere_chat.yaml similarity index 100% rename from tests/contrib/litellm/cassettes/completion_command-r.yaml rename to tests/contrib/litellm/cassettes/completion_cohere_chat.yaml diff --git a/tests/contrib/litellm/test_litellm.py b/tests/contrib/litellm/test_litellm.py index 958bd6d64c8..38bdfa96b62 100644 --- a/tests/contrib/litellm/test_litellm.py +++ b/tests/contrib/litellm/test_litellm.py @@ -90,7 +90,7 @@ async def test_litellm_atext_completion(litellm, snapshot_context, request_vcr, pass -@pytest.mark.parametrize("model", ["command-r", "anthropic/claude-3-5-sonnet-20240620"]) +@pytest.mark.parametrize("model", ["cohere_chat/command-r", "anthropic/claude-3-5-sonnet-20240620"]) def test_litellm_completion_different_models(litellm, snapshot_context, request_vcr, model): with snapshot_context( token="tests.contrib.litellm.test_litellm.test_litellm_completion", ignores=["meta.litellm.request.model"] diff --git a/tests/contrib/suitespec.yml b/tests/contrib/suitespec.yml index 055c21cbc09..4ac634d36c5 100644 --- a/tests/contrib/suitespec.yml +++ b/tests/contrib/suitespec.yml @@ -431,6 +431,7 @@ suites: - azureservicebusemulator botocore: parallelism: 11 + no_proxy: true paths: - '@bootstrap' - '@core' @@ -770,12 +771,14 @@ suites: - '@tracing' - '@httplib' - tests/contrib/httplib/* + no_proxy: true runner: riot services: - httpbin snapshot: true httpx: parallelism: 3 + no_proxy: true paths: - '@bootstrap' - '@core' @@ -1108,6 +1111,7 @@ suites: snapshot: true requests: parallelism: 1 + no_proxy: true paths: - '@bootstrap' - '@core' diff --git a/tests/llmobs/suitespec.yml b/tests/llmobs/suitespec.yml index d3741825585..89494659ecd 100644 --- a/tests/llmobs/suitespec.yml +++ b/tests/llmobs/suitespec.yml @@ -127,6 +127,7 @@ suites: - '@tracing' - '@llmobs' - tests/llmobs/* + no_proxy: true runner: riot snapshot: true mcp: diff --git a/tests/profiling_v2/test_code_provenance.py b/tests/profiling_v2/test_code_provenance.py index 88b4feab3ee..74711249329 100644 --- a/tests/profiling_v2/test_code_provenance.py +++ b/tests/profiling_v2/test_code_provenance.py @@ -128,7 +128,7 @@ def test_main_package_my_code(self): json_str = json_str_to_export() json_obj = json.loads(json_str) # Really what we're checking is that whatever package the user calls the - # "main" pacakge isn't called a library. Normally this would matter if + # "main" package isn't called a library. Normally this would matter if # the user installs their package as a dependency and then runs it as # the main package. But for the sake of this test, just call ddtrace the # "main" package. diff --git a/tests/tracer/test_http.py b/tests/tracer/test_http.py index 864abdb49e1..e5872fedb63 100644 --- a/tests/tracer/test_http.py +++ b/tests/tracer/test_http.py @@ -7,6 +7,7 @@ from ddtrace.internal.utils.http import normalize_header_name from ddtrace.internal.utils.http import redact_url from ddtrace.internal.utils.http import strip_query_string +from ddtrace.settings._config import DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP_DEFAULT def _url_fixtures(): @@ -75,7 +76,25 @@ def test_redact_url_not_redacts_with_param(url): ("://?x", re.compile(b"x"), None, b"://?"), ("://x", re.compile(b"x"), "x", b"://x?"), ("://y", re.compile(b"x"), "x", b"://y?"), + ( + "://?jwt=eyJa.eyJb&eyJa.eyJb&x=%22eyJa.eyJb", + None, + None, + b"://?jwt=&&x=%22", + ), + ("://x", None, "eyJa.eyJb", b"://x?"), + ("://?x=%3DeyJa%3D.eyJb%3D.signature", None, None, b"://?x=%3D"), + ( + "://?x=abceyJa.eyJb&x=-eyJa.eyJb&heyJude.eyJoe", + None, + None, + b"://?x=abceyJa.eyJb&x=-eyJa.eyJb&heyJude.eyJoe", + ), + ("://?password=secret&b=2", re.compile(rb"(password)=([^&]+)"), None, b"://?&b=2"), + ("://?password=secret&b=2", re.compile(rb"(?Ppassword=[^&]+)"), None, b"://?&b=2"), ), ) def test_redact_url_does_redact(url, regex, query_string, expected): - assert redact_url(url, regex, query_string) == expected + preserve_delimiter = regex is None + regex = regex or re.compile(DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP_DEFAULT.encode("ascii")) + assert redact_url(url, regex, query_string, preserve_delimiter=preserve_delimiter) == expected diff --git a/tests/tracer/test_trace_utils.py b/tests/tracer/test_trace_utils.py index 65b79767706..e8acdb45d7b 100644 --- a/tests/tracer/test_trace_utils.py +++ b/tests/tracer/test_trace_utils.py @@ -24,6 +24,7 @@ from ddtrace.propagation.http import HTTP_HEADER_PARENT_ID from ddtrace.propagation.http import HTTP_HEADER_TRACE_ID from ddtrace.propagation.http import HTTPPropagator +from ddtrace.settings._config import DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP_DEFAULT from ddtrace.settings._config import Config from ddtrace.settings.integration import IntegrationConfig from ddtrace.trace import Context @@ -1166,9 +1167,25 @@ def test_url_in_http_with_obfuscation_enabled_and_empty_regex(): assert span.get_tag(http.URL) == "http://weblog:7777/", span._meta -def test_url_in_http_meta(span, int_config): - SENSITIVE_QS_URL = "http://example.com/search?token=03cb9f67dbbc4cb8b963629951e10934&q=query#frag?ment" - REDACTED_URL = "http://example.com/search?&q=query#frag?ment" +@pytest.mark.parametrize( + "regex, query, redacted_query", + ( + (None, "token=03cb9f67dbbc4cb8b963629951e10934&q=query", "&q=query"), + (None, "jwt=eyJa.eyJb", "jwt="), + (DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP_DEFAULT, "jwt=eyJa.eyJb", "jwt="), + ("(?Ppassword=[^&]+)", "password=secret", ""), + ), +) +def test_url_in_http_meta(span, int_config, monkeypatch, regex, query, redacted_query): + if regex is not None: + monkeypatch.setenv("DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP", regex) + cfg = Config() + monkeypatch.setattr(config, "_obfuscation_query_string_pattern", cfg._obfuscation_query_string_pattern) + monkeypatch.setattr( + config, "_query_string_obfuscation_preserve_delimiter", cfg._query_string_obfuscation_preserve_delimiter + ) + SENSITIVE_QS_URL = "http://example.com/search?" + query + "#frag?ment" + REDACTED_URL = "http://example.com/search?" + redacted_query + "#frag?ment" STRIPPED_URL = "http://example.com/search#frag?ment" int_config.http_tag_query_string = True diff --git a/tests/utils.py b/tests/utils.py index 79b1b6043fe..8be9888a3ea 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -49,6 +49,7 @@ from ddtrace.propagation._database_monitoring import unlisten as dbm_config_unlisten from ddtrace.propagation.http import _DatadogMultiHeader from ddtrace.settings._agent import config as agent_config +from ddtrace.settings._config import DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP_DEFAULT from ddtrace.settings._database_monitoring import dbm_config from ddtrace.settings.asm import config as asm_config from ddtrace.settings.openfeature import config as ffe_config @@ -145,6 +146,7 @@ def override_global_config(values): "_trace_compute_stats", "_trace_resource_renaming_always_simplified_endpoint", "_obfuscation_query_string_pattern", + "_query_string_obfuscation_preserve_delimiter", "_global_query_string_obfuscation_disabled", "_ci_visibility_agentless_url", "_ci_visibility_agentless_enabled", @@ -191,6 +193,11 @@ def override_global_config(values): for key, value in values.items(): if key in global_config_keys: setattr(ddtrace.config, key, value) + if "_obfuscation_query_string_pattern" in values and "_query_string_obfuscation_preserve_delimiter" not in values: + pattern = values["_obfuscation_query_string_pattern"] + ddtrace.config._query_string_obfuscation_preserve_delimiter = ( + pattern is not None and pattern.pattern == DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP_DEFAULT.encode("ascii") + ) # rebuild asm config from env vars and global config for key, value in values.items(): if key in asm_config_keys: