From 9569b8e43e176e04488027cee9e7c50023539c4d Mon Sep 17 00:00:00 2001 From: Michael Commey Date: Fri, 17 Jul 2026 15:46:10 +0000 Subject: [PATCH 1/5] Add test for otlp publishing components --- generator/test_case_generator.go | 868 +++++++++--------- .../agent_configs/host_metrics_config.json | 10 - .../agent_configs/host_metrics_config.json | 13 + .../{ => linux}/host_metrics_test.go | 0 .../host_metrics/windows/agent_config.json | 13 + .../host_metrics/windows/parameters.yml | 23 + .../otlp/agent_configs/otlp_config.json | 1 - .../otlp/linux/agent_configs/otlp_config.json | 14 + .../otlp/{ => linux}/otlp_test.go | 0 .../otlp/windows/agent_config.json | 17 + test/otel_collect/otlp/windows/parameters.yml | 24 + .../agent_configs/prometheus_config.json | 1 - .../agent_configs/prometheus_config.json | 13 + .../prometheus/{ => linux}/prometheus_test.go | 0 .../{ => linux}/resources/prometheus_metrics | 0 .../resources/prometheus_scrape_config.yaml | 13 + .../resources/prometheus_scrape_config.yaml | 4 - .../prometheus/windows/agent_config.json | 13 + .../prometheus/windows/parameters.yml | 45 + .../prometheus/windows/prometheus_metrics | 13 + .../windows/prometheus_scrape_config.yaml | 13 + 21 files changed, 657 insertions(+), 441 deletions(-) delete mode 100644 test/otel_collect/host_metrics/agent_configs/host_metrics_config.json create mode 100644 test/otel_collect/host_metrics/linux/agent_configs/host_metrics_config.json rename test/otel_collect/host_metrics/{ => linux}/host_metrics_test.go (100%) create mode 100644 test/otel_collect/host_metrics/windows/agent_config.json create mode 100644 test/otel_collect/host_metrics/windows/parameters.yml delete mode 100644 test/otel_collect/otlp/agent_configs/otlp_config.json create mode 100644 test/otel_collect/otlp/linux/agent_configs/otlp_config.json rename test/otel_collect/otlp/{ => linux}/otlp_test.go (100%) create mode 100644 test/otel_collect/otlp/windows/agent_config.json create mode 100644 test/otel_collect/otlp/windows/parameters.yml delete mode 100644 test/otel_collect/prometheus/agent_configs/prometheus_config.json create mode 100644 test/otel_collect/prometheus/linux/agent_configs/prometheus_config.json rename test/otel_collect/prometheus/{ => linux}/prometheus_test.go (100%) rename test/otel_collect/prometheus/{ => linux}/resources/prometheus_metrics (100%) create mode 100644 test/otel_collect/prometheus/linux/resources/prometheus_scrape_config.yaml delete mode 100644 test/otel_collect/prometheus/resources/prometheus_scrape_config.yaml create mode 100644 test/otel_collect/prometheus/windows/agent_config.json create mode 100644 test/otel_collect/prometheus/windows/parameters.yml create mode 100644 test/otel_collect/prometheus/windows/prometheus_metrics create mode 100644 test/otel_collect/prometheus/windows/prometheus_scrape_config.yaml diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index 344d9690..f9751dc8 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -75,451 +75,469 @@ const ( // you can't have a const map in golang var testTypeToTestConfig = map[string][]testConfig{ - "ec2_gpu": { - {testDir: "./test/nvidia_gpu"}, - }, - "ec2_efa": { - {testDir: "./test/efa_ec2", terraformDir: "terraform/ec2/efa"}, - }, - "ec2_linux_wd": { - {testDir: "./test/workload_discovery"}, - }, - "ec2_linux_wd_nvidia": { - {testDir: "./test/workload_discovery"}, - }, + // "ec2_gpu": { + // {testDir: "./test/nvidia_gpu"}, + // }, + // "ec2_efa": { + // {testDir: "./test/efa_ec2", terraformDir: "terraform/ec2/efa"}, + // }, + // "ec2_linux_wd": { + // {testDir: "./test/workload_discovery"}, + // }, + // "ec2_linux_wd_nvidia": { + // {testDir: "./test/workload_discovery"}, + // }, "ec2_linux_onprem": { {testDir: "./test/cloudwatchlogs"}, + {testDir: "./test/otel_collect/otlp/linux"}, + {testDir: "./test/otel_collect/host_metrics/linux"}, + {testDir: "./test/otel_collect/prometheus/linux"}, }, testTypeKeyEc2Linux: { - {testDir: "./test/ca_bundle"}, - {testDir: "./test/cloudwatchlogs"}, - { - testDir: "./test/log_state/logfile", - targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - }, - { - testDir: "./test/log_state/journald", - targets: map[string]map[string]struct{}{"os": {"al2": {}, "al2023": {}}}, - }, - { - testDir: "./test/feature/linux/journald_logs", - targets: map[string]map[string]struct{}{"os": {"al2": {}, "al2023": {}}}, - }, - { - testDir: "./test/metrics_number_dimension", - targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - }, - { - testDir: "./test/emf_concurrent", - targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - maxAttempts: 1, - }, - { - testDir: "./test/emf_prometheus", - targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - maxAttempts: 2, - }, - {testDir: "./test/entity_metrics_benchmark"}, - { - testDir: "./test/metric_value_benchmark", - instanceTypeByArch: map[string]string{ - "amd64": "i3en.large", - "arm64": "i4g.large", - }, - }, - {testDir: "./test/run_as_user"}, - {testDir: "./test/collection_interval"}, - {testDir: "./test/metric_dimension"}, - {testDir: "./test/restart"}, - {testDir: "./test/xray"}, - {testDir: "./test/otlp"}, - { - testDir: "./test/otel_collect/database_insights", - excludedOs: map[string]struct{}{"ol8": {}, "ubuntu-25": {}}, - }, - {testDir: "./test/otel_collect/host_metrics"}, - {testDir: "./test/otel_collect/otlp"}, - { - testDir: "./test/otel_collect/prometheus", + // {testDir: "./test/ca_bundle"}, + // {testDir: "./test/cloudwatchlogs"}, + // { + // testDir: "./test/log_state/logfile", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + // }, + // { + // testDir: "./test/log_state/journald", + // targets: map[string]map[string]struct{}{"os": {"al2": {}, "al2023": {}}}, + // }, + // { + // testDir: "./test/feature/linux/journald_logs", + // targets: map[string]map[string]struct{}{"os": {"al2": {}, "al2023": {}}}, + // }, + // { + // testDir: "./test/metrics_number_dimension", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + // }, + // { + // testDir: "./test/emf_concurrent", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + // maxAttempts: 1, + // }, + // { + // testDir: "./test/emf_prometheus", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + // maxAttempts: 2, + // }, + // {testDir: "./test/entity_metrics_benchmark"}, + // { + // testDir: "./test/metric_value_benchmark", + // instanceTypeByArch: map[string]string{ + // "amd64": "i3en.large", + // "arm64": "i4g.large", + // }, + // }, + // {testDir: "./test/run_as_user"}, + // {testDir: "./test/collection_interval"}, + // {testDir: "./test/metric_dimension"}, + // {testDir: "./test/restart"}, + // {testDir: "./test/xray"}, + // {testDir: "./test/otlp"}, + // { + // testDir: "./test/otel_collect/database_insights", + // excludedOs: map[string]struct{}{"ol8": {}, "ubuntu-25": {}}, + // }, + {testDir: "./test/otel_collect/host_metrics/linux"}, + {testDir: "./test/otel_collect/otlp/linux"}, + { + testDir: "./test/otel_collect/prometheus/linux", excludedOs: map[string]struct{}{"rhel8": {}, "ol8": {}, "sles-15": {}}, }, - { - testDir: "./test/acceptance", - targets: map[string]map[string]struct{}{"os": {"ubuntu-20.04": {}}}, - }, - // skipping FIPS test as the test cannot be verified - // neither ssh nor SSM works after a reboot once FIPS is enabled - //{ - // testDir: "./test/fips", - // targets: map[string]map[string]struct{}{"os": {"rhel8": {}}}, - //}, - { - testDir: "./test/lvm", - targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - }, - { - testDir: "./test/proxy", - targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - }, - { - testDir: "./test/ssl_cert", - targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - }, - { - testDir: "./test/userdata", - terraformDir: "terraform/ec2/userdata", - targets: map[string]map[string]struct{}{"os": {"ol9": {}}}, - }, - { - testDir: "./test/credentials_file", - terraformDir: "terraform/ec2/creds", - targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - }, - { - testDir: "./test/amp", - targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - }, - { - testDir: "./test/histograms", - targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - }, - { - testDir: "./test/agent_otel_merging", - targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - }, - { - testDir: "./test/assume_role", - terraformDir: "terraform/ec2/assume_role", - targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - }, - { - testDir: "./test/credential_chain", - terraformDir: "terraform/ec2/assume_role", - targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - }, - { - testDir: "./test/detailed_metrics", - targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - }, - { - testDir: "./test/otlp", - targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - }, - { - testDir: "./test/dualstack_endpoint", - targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - }, - {testDir: "./test/ssm_document"}, - { - testDir: "./test/system_metrics/enabled", - targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - wip: true, - }, - { - testDir: "./test/system_metrics/disabled", - targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - }, - { - testDir: "./test/app_signals_service_events", - targets: map[string]map[string]struct{}{"os": {"al2023": {}}, "arc": {"amd64": {}}}, - }, - }, - testTypeKeyEc2SELinux: { - {testDir: "./test/ca_bundle"}, - {testDir: "./test/cloudwatchlogs"}, - { - testDir: "./test/log_state/journald", - targets: map[string]map[string]struct{}{"os": {"al2": {}, "al2023": {}}}, - }, - { - testDir: "./test/feature/linux/journald_logs", - targets: map[string]map[string]struct{}{"os": {"al2": {}, "al2023": {}}}, - }, - { - testDir: "./test/metrics_number_dimension", - targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - }, - { - testDir: "./test/emf_concurrent", - targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - maxAttempts: 1, - }, - { - testDir: "./test/emf_prometheus", - maxAttempts: 2, - }, - //{testDir: "./test/metric_value_benchmark"}, // Skipping test until it is fixed! - {testDir: "./test/run_as_user"}, - {testDir: "./test/collection_interval"}, - {testDir: "./test/metric_dimension"}, - {testDir: "./test/restart"}, - {testDir: "./test/xray"}, - {testDir: "./test/selinux_negative_test"}, - //{testDir: "./test/otlp"}, // Skipping test until it is fixed! - { - testDir: "./test/lvm", - targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - }, - { - testDir: "./test/proxy", - targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - }, - { - testDir: "./test/ssl_cert", - targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - }, - { - testDir: "./test/credentials_file", - terraformDir: "terraform/ec2/creds", - targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - }, - { - testDir: "./test/amp", - targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - }, - { - testDir: "./test/agent_otel_merging", - targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - }, - { - testDir: "./test/assume_role", - terraformDir: "terraform/ec2/assume_role", - targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - }, - { - testDir: "./test/dualstack_endpoint", - targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - }, + // { + // testDir: "./test/acceptance", + // targets: map[string]map[string]struct{}{"os": {"ubuntu-20.04": {}}}, + // }, + // // skipping FIPS test as the test cannot be verified + // // neither ssh nor SSM works after a reboot once FIPS is enabled + // //{ + // // testDir: "./test/fips", + // // targets: map[string]map[string]struct{}{"os": {"rhel8": {}}}, + // //}, + // { + // testDir: "./test/lvm", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + // }, + // { + // testDir: "./test/proxy", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + // }, + // { + // testDir: "./test/ssl_cert", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + // }, + // { + // testDir: "./test/userdata", + // terraformDir: "terraform/ec2/userdata", + // targets: map[string]map[string]struct{}{"os": {"ol9": {}}}, + // }, + // { + // testDir: "./test/credentials_file", + // terraformDir: "terraform/ec2/creds", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + // }, + // { + // testDir: "./test/amp", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + // }, + // { + // testDir: "./test/histograms", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + // }, + // { + // testDir: "./test/agent_otel_merging", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + // }, + // { + // testDir: "./test/assume_role", + // terraformDir: "terraform/ec2/assume_role", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + // }, + // { + // testDir: "./test/credential_chain", + // terraformDir: "terraform/ec2/assume_role", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + // }, + // { + // testDir: "./test/detailed_metrics", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + // }, + // { + // testDir: "./test/otlp", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + // }, + // { + // testDir: "./test/dualstack_endpoint", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + // }, + // {testDir: "./test/ssm_document"}, + // { + // testDir: "./test/system_metrics/enabled", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + // wip: true, + // }, + // { + // testDir: "./test/system_metrics/disabled", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + // }, + // { + // testDir: "./test/app_signals_service_events", + // targets: map[string]map[string]struct{}{"os": {"al2023": {}}, "arc": {"amd64": {}}}, + // }, }, + // testTypeKeyEc2SELinux: { + // {testDir: "./test/ca_bundle"}, + // {testDir: "./test/cloudwatchlogs"}, + // { + // testDir: "./test/log_state/journald", + // targets: map[string]map[string]struct{}{"os": {"al2": {}, "al2023": {}}}, + // }, + // { + // testDir: "./test/feature/linux/journald_logs", + // targets: map[string]map[string]struct{}{"os": {"al2": {}, "al2023": {}}}, + // }, + // { + // testDir: "./test/metrics_number_dimension", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + // }, + // { + // testDir: "./test/emf_concurrent", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + // maxAttempts: 1, + // }, + // { + // testDir: "./test/emf_prometheus", + // maxAttempts: 2, + // }, + // //{testDir: "./test/metric_value_benchmark"}, // Skipping test until it is fixed! + // {testDir: "./test/run_as_user"}, + // {testDir: "./test/collection_interval"}, + // {testDir: "./test/metric_dimension"}, + // {testDir: "./test/restart"}, + // {testDir: "./test/xray"}, + // {testDir: "./test/selinux_negative_test"}, + // //{testDir: "./test/otlp"}, // Skipping test until it is fixed! + // { + // testDir: "./test/lvm", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + // }, + // { + // testDir: "./test/proxy", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + // }, + // { + // testDir: "./test/ssl_cert", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + // }, + // { + // testDir: "./test/credentials_file", + // terraformDir: "terraform/ec2/creds", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + // }, + // { + // testDir: "./test/amp", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + // }, + // { + // testDir: "./test/agent_otel_merging", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + // }, + // { + // testDir: "./test/assume_role", + // terraformDir: "terraform/ec2/assume_role", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + // }, + // { + // testDir: "./test/dualstack_endpoint", + // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + // }, + // }, /* You can only place 1 mac instance on a dedicate host a single time. Therefore, limit down the scope for testing in Mac since EC2 can be done with Linux and Mac under the hood share similar plugins with Linux */ - "ec2_mac": { - {testDir: "../../../test/feature/mac"}, - }, - "ec2_windows_wd": { - {testDir: "../../../test/workload_discovery"}, - }, - "ec2_windows_wd_nvidia": { - {testDir: "../../../test/workload_discovery"}, - }, + // "ec2_mac": { + // {testDir: "../../../test/feature/mac"}, + // }, + // "ec2_windows_wd": { + // {testDir: "../../../test/workload_discovery"}, + // }, + // "ec2_windows_wd_nvidia": { + // {testDir: "../../../test/workload_discovery"}, + // }, "ec2_windows": { - {testDir: "../../../test/feature/windows"}, - {testDir: "../../../test/restart"}, - {testDir: "../../../test/acceptance"}, - {testDir: "../../../test/feature/windows/event_logs"}, - {testDir: "../../../test/feature/windows/eventid_logs"}, - {testDir: "../../../test/feature/windows/event_regex_logs"}, - {testDir: "../../../test/log_state/logfile"}, - {testDir: "../../../test/log_state/windows_event_log"}, - { - testDir: "../../../test/feature/windows/custom_start/userdata", - targets: map[string]map[string]struct{}{"os": {"win-2019": {}}}, - }, - { - testDir: "../../../test/feature/windows/custom_start/ssm_start", - targets: map[string]map[string]struct{}{"os": {"win-2019": {}}}, - }, - {testDir: "../../../test/ssm_document"}, + // {testDir: "../../../test/feature/windows"}, + // {testDir: "../../../test/restart"}, + // {testDir: "../../../test/acceptance"}, + // {testDir: "../../../test/feature/windows/event_logs"}, + // {testDir: "../../../test/feature/windows/eventid_logs"}, + // {testDir: "../../../test/feature/windows/event_regex_logs"}, + // {testDir: "../../../test/log_state/logfile"}, + // {testDir: "../../../test/log_state/windows_event_log"}, + // { + // testDir: "../../../test/feature/windows/custom_start/userdata", + // targets: map[string]map[string]struct{}{"os": {"win-2019": {}}}, + // }, + // { + // testDir: "../../../test/feature/windows/custom_start/ssm_start", + // targets: map[string]map[string]struct{}{"os": {"win-2019": {}}}, + // }, + // {testDir: "../../../test/ssm_document"}, + {testDir: "../../../test/otel_collect/otlp/windows"}, + {testDir: "../../../test/otel_collect/host_metrics/windows"}, + {testDir: "../../../test/otel_collect/prometheus/windows"}, // assume role test doesn't add much value, and it already being tested with linux //{testDir: "../../../test/assume_role"}, }, - "ec2_performance": { - {testDir: "../../test/performance/emf"}, - {testDir: "../../test/performance/logs"}, - {testDir: "../../test/performance/system"}, - {testDir: "../../test/performance/statsd"}, - {testDir: "../../test/performance/collectd"}, - {testDir: "../../test/performance/trace/xray", runMockServer: true}, - }, - "ec2_windows_performance": { - {testDir: "../../test/performance/windows/logs"}, - {testDir: "../../test/performance/windows/system"}, - }, - "ec2_stress": { - {testDir: "../../test/stress/emf"}, - {testDir: "../../test/stress/logs"}, - {testDir: "../../test/stress/system"}, - {testDir: "../../test/stress/statsd"}, - {testDir: "../../test/stress/collectd"}, - {testDir: "../../test/stress/prometheus"}, - }, - "ec2_windows_stress": { - {testDir: "../../test/stress/windows/logs"}, - {testDir: "../../test/stress/windows/system"}, - }, - "ecs_fargate": { - {testDir: "./test/ecs/service_discovery"}, - }, + // "ec2_performance": { + // {testDir: "../../test/performance/emf"}, + // {testDir: "../../test/performance/logs"}, + // {testDir: "../../test/performance/system"}, + // {testDir: "../../test/performance/statsd"}, + // {testDir: "../../test/performance/collectd"}, + // {testDir: "../../test/performance/trace/xray", runMockServer: true}, + // }, + // "ec2_windows_performance": { + // {testDir: "../../test/performance/windows/logs"}, + // {testDir: "../../test/performance/windows/system"}, + // }, + // "ec2_stress": { + // {testDir: "../../test/stress/emf"}, + // {testDir: "../../test/stress/logs"}, + // {testDir: "../../test/stress/system"}, + // {testDir: "../../test/stress/statsd"}, + // {testDir: "../../test/stress/collectd"}, + // {testDir: "../../test/stress/prometheus"}, + // }, + // "ec2_windows_stress": { + // {testDir: "../../test/stress/windows/logs"}, + // {testDir: "../../test/stress/windows/system"}, + // }, + // "ecs_fargate": { + // {testDir: "./test/ecs/service_discovery"}, + // }, "ecs_ec2_daemon": { - { - testDir: "./test/metric_value_benchmark", - targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, - }, - { - testDir: "./test/statsd", + // { + // testDir: "./test/metric_value_benchmark", + // targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, + // }, + // { + // testDir: "./test/statsd", + // targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, + // }, + // { + // testDir: "./test/emf", + // targets: map[string]map[string]struct{}{"metadataEnabled": {"disabled": {}}}, + // }, + // { + // testDir: "./test/emf", + // targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, + // }, + // { + // testDir: "./test/ecs/service_discovery", + // targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, + // }, + { + testDir: "./test/otel_collect/otlp/linux", targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, }, { - testDir: "./test/emf", - targets: map[string]map[string]struct{}{"metadataEnabled": {"disabled": {}}}, - }, - { - testDir: "./test/emf", + testDir: "./test/otel_collect/host_metrics/linux", targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, }, { - testDir: "./test/ecs/service_discovery", + testDir: "./test/otel_collect/prometheus/linux", targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, }, }, - "eks_addon": { - { - testDir: "./test/gpu", - terraformDir: "terraform/eks/addon/gpu", - }, - }, - "eks_daemon": { - { - testDir: "./test/metric_value_benchmark", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - instanceType: "g4dn.xlarge", - ami: "AL2_x86_64_GPU", - }, - { - testDir: "./test/metric_value_benchmark", - terraformDir: "terraform/eks/daemon/windows/2019", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - { - testDir: "./test/metric_value_benchmark", - terraformDir: "terraform/eks/daemon/windows/2022", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - { - testDir: "./test/statsd", terraformDir: "terraform/eks/daemon/statsd", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - { - testDir: "./test/emf", terraformDir: "terraform/eks/daemon/emf", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - { - testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/d", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - {testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/bit"}, - {testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/windows/2022"}, - { - testDir: "./test/gpu", terraformDir: "terraform/eks/daemon/gpu", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - instanceType: "g4dn.xlarge", - ami: "AL2_x86_64_GPU", - }, - { - testDir: "./test/gpu_high_frequency_metrics", terraformDir: "terraform/eks/daemon/gpu", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - instanceType: "g4dn.xlarge", - ami: "AL2_x86_64_GPU", - }, - { - testDir: "./test/awsneuron", terraformDir: "terraform/eks/daemon/awsneuron", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - wip: true, - }, - { - testDir: "./test/entity", terraformDir: "terraform/eks/daemon/entity", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - { - testDir: "./test/efa", terraformDir: "terraform/eks/daemon/efa", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - instanceType: "c6in.32xlarge", - }, - { - testDir: "./test/metric_value_benchmark", terraformDir: "terraform/eks/daemon/credentials/pod_identity", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - { - testDir: "./test/ebscsi", - terraformDir: "terraform/eks/daemon/ebs", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - }, - { - testDir: "./test/liscsi", - terraformDir: "terraform/eks/daemon/liscsi", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - instanceType: "i7i.xlarge", - ami: "AL2023_x86_64_STANDARD", - k8sVersion: "1.35", - }, - { - testDir: "./test/otel/standard", - terraformDir: "terraform/eks/daemon/otel", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - ami: "AL2023_x86_64_STANDARD", - k8sVersion: "1.35", - }, - { - testDir: "./test/otel/attr_limit", - terraformDir: "terraform/eks/daemon/otel-attr-limit", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - ami: "AL2023_x86_64_STANDARD", - k8sVersion: "1.35", - }, - { - testDir: "./test/otel/ebs_csi", - terraformDir: "terraform/eks/daemon/otel-ebs-csi", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - ami: "AL2023_x86_64_STANDARD", - k8sVersion: "1.35", - }, - { - testDir: "./test/otel/efa", - terraformDir: "terraform/eks/daemon/otel-efa", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - instanceType: "c5n.9xlarge", - ami: "AL2023_x86_64_STANDARD", - k8sVersion: "1.35", - }, - { - testDir: "./test/otel/gpu", - terraformDir: "terraform/eks/daemon/otel-gpu", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - instanceType: "g4dn.xlarge", - ami: "AL2023_x86_64_NVIDIA", - k8sVersion: "1.35", - }, - { - testDir: "./test/otel/lis_csi", - terraformDir: "terraform/eks/daemon/otel-lis-csi", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - instanceType: "i7i.xlarge", - ami: "AL2023_x86_64_STANDARD", - k8sVersion: "1.35", - }, - { - testDir: "./test/otel/multi_efa", - terraformDir: "terraform/eks/daemon/otel-multi-efa", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - instanceType: "c6in.32xlarge", - ami: "AL2023_x86_64_STANDARD", - k8sVersion: "1.35", - }, - { - testDir: "./test/otel/neuron", - terraformDir: "terraform/eks/daemon/otel-neuron", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - instanceType: "inf2.xlarge", - ami: "AL2023_x86_64_NEURON", - k8sVersion: "1.35", - }, - }, - "eks_deployment": { - {testDir: "./test/metric_value_benchmark"}, - }, + // "eks_addon": { + // { + // testDir: "./test/gpu", + // terraformDir: "terraform/eks/addon/gpu", + // }, + // }, + "eks_daemon": { + // { + // testDir: "./test/metric_value_benchmark", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // instanceType: "g4dn.xlarge", + // ami: "AL2_x86_64_GPU", + // }, + // { + // testDir: "./test/metric_value_benchmark", + // terraformDir: "terraform/eks/daemon/windows/2019", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // }, + // { + // testDir: "./test/metric_value_benchmark", + // terraformDir: "terraform/eks/daemon/windows/2022", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // }, + // { + // testDir: "./test/statsd", terraformDir: "terraform/eks/daemon/statsd", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // }, + // { + // testDir: "./test/emf", terraformDir: "terraform/eks/daemon/emf", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // }, + // { + // testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/d", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // }, + // {testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/bit"}, + // {testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/windows/2022"}, + // { + // testDir: "./test/gpu", terraformDir: "terraform/eks/daemon/gpu", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // instanceType: "g4dn.xlarge", + // ami: "AL2_x86_64_GPU", + // }, + // { + // testDir: "./test/gpu_high_frequency_metrics", terraformDir: "terraform/eks/daemon/gpu", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // instanceType: "g4dn.xlarge", + // ami: "AL2_x86_64_GPU", + // }, + // { + // testDir: "./test/awsneuron", terraformDir: "terraform/eks/daemon/awsneuron", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // wip: true, + // }, + // { + // testDir: "./test/entity", terraformDir: "terraform/eks/daemon/entity", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // }, + // { + // testDir: "./test/efa", terraformDir: "terraform/eks/daemon/efa", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // instanceType: "c6in.32xlarge", + // }, + // { + // testDir: "./test/metric_value_benchmark", terraformDir: "terraform/eks/daemon/credentials/pod_identity", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // }, + // { + // testDir: "./test/ebscsi", + // terraformDir: "terraform/eks/daemon/ebs", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // }, + // { + // testDir: "./test/liscsi", + // terraformDir: "terraform/eks/daemon/liscsi", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // instanceType: "i7i.xlarge", + // ami: "AL2023_x86_64_STANDARD", + // k8sVersion: "1.35", + // }, + { + testDir: "./test/otel/standard", + terraformDir: "terraform/eks/daemon/otel", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + ami: "AL2023_x86_64_STANDARD", + k8sVersion: "1.35", + }, + // { + // testDir: "./test/otel/attr_limit", + // terraformDir: "terraform/eks/daemon/otel-attr-limit", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // ami: "AL2023_x86_64_STANDARD", + // k8sVersion: "1.35", + // }, + // { + // testDir: "./test/otel/ebs_csi", + // terraformDir: "terraform/eks/daemon/otel-ebs-csi", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // ami: "AL2023_x86_64_STANDARD", + // k8sVersion: "1.35", + // }, + // { + // testDir: "./test/otel/efa", + // terraformDir: "terraform/eks/daemon/otel-efa", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // instanceType: "c5n.9xlarge", + // ami: "AL2023_x86_64_STANDARD", + // k8sVersion: "1.35", + // }, + // { + // testDir: "./test/otel/gpu", + // terraformDir: "terraform/eks/daemon/otel-gpu", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // instanceType: "g4dn.xlarge", + // ami: "AL2023_x86_64_NVIDIA", + // k8sVersion: "1.35", + // }, + // { + // testDir: "./test/otel/lis_csi", + // terraformDir: "terraform/eks/daemon/otel-lis-csi", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // instanceType: "i7i.xlarge", + // ami: "AL2023_x86_64_STANDARD", + // k8sVersion: "1.35", + // }, + // { + // testDir: "./test/otel/multi_efa", + // terraformDir: "terraform/eks/daemon/otel-multi-efa", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // instanceType: "c6in.32xlarge", + // ami: "AL2023_x86_64_STANDARD", + // k8sVersion: "1.35", + // }, + // { + // testDir: "./test/otel/neuron", + // terraformDir: "terraform/eks/daemon/otel-neuron", + // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + // instanceType: "inf2.xlarge", + // ami: "AL2023_x86_64_NEURON", + // k8sVersion: "1.35", + // }, + // }, + // "eks_deployment": { + // {testDir: "./test/metric_value_benchmark"}, + }, } var testTypeToTestConfigE2E = map[string][]testConfig{ @@ -554,9 +572,9 @@ var partitionTests = map[string]partition{ ami: []string{"cloudwatch-agent-integration-test-aarch64-al2023*"}, excludedTestDirs: map[string]struct{}{ "./test/otel_collect/database_insights": {}, - "./test/otel_collect/host_metrics": {}, - "./test/otel_collect/otlp": {}, - "./test/otel_collect/prometheus": {}, + "./test/otel_collect/host_metrics/linux": {}, + "./test/otel_collect/otlp/linux": {}, + "./test/otel_collect/prometheus/linux": {}, }, testConfigOverrides: map[string]testConfig{ "./test/metric_value_benchmark": { @@ -575,9 +593,9 @@ var partitionTests = map[string]partition{ ami: []string{"cloudwatch-agent-integration-test-aarch64-al2023*"}, excludedTestDirs: map[string]struct{}{ "./test/otel_collect/database_insights": {}, - "./test/otel_collect/host_metrics": {}, - "./test/otel_collect/otlp": {}, - "./test/otel_collect/prometheus": {}, + "./test/otel_collect/host_metrics/linux": {}, + "./test/otel_collect/otlp/linux": {}, + "./test/otel_collect/prometheus/linux": {}, }, testConfigOverrides: map[string]testConfig{ "./test/metric_value_benchmark": { diff --git a/test/otel_collect/host_metrics/agent_configs/host_metrics_config.json b/test/otel_collect/host_metrics/agent_configs/host_metrics_config.json deleted file mode 100644 index 7f02358b..00000000 --- a/test/otel_collect/host_metrics/agent_configs/host_metrics_config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "agent": { - "region": "us-west-2" - }, - "opentelemetry": { - "collect": { - "host_metrics": {} - } - } -} diff --git a/test/otel_collect/host_metrics/linux/agent_configs/host_metrics_config.json b/test/otel_collect/host_metrics/linux/agent_configs/host_metrics_config.json new file mode 100644 index 00000000..13b50217 --- /dev/null +++ b/test/otel_collect/host_metrics/linux/agent_configs/host_metrics_config.json @@ -0,0 +1,13 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "cluster_name": "integ-test-cluster", + "collect": { + "host_metrics": { + "collection_interval": 30 + } + } + } +} diff --git a/test/otel_collect/host_metrics/host_metrics_test.go b/test/otel_collect/host_metrics/linux/host_metrics_test.go similarity index 100% rename from test/otel_collect/host_metrics/host_metrics_test.go rename to test/otel_collect/host_metrics/linux/host_metrics_test.go diff --git a/test/otel_collect/host_metrics/windows/agent_config.json b/test/otel_collect/host_metrics/windows/agent_config.json new file mode 100644 index 00000000..13b50217 --- /dev/null +++ b/test/otel_collect/host_metrics/windows/agent_config.json @@ -0,0 +1,13 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "cluster_name": "integ-test-cluster", + "collect": { + "host_metrics": { + "collection_interval": 30 + } + } + } +} diff --git a/test/otel_collect/host_metrics/windows/parameters.yml b/test/otel_collect/host_metrics/windows/parameters.yml new file mode 100644 index 00000000..85eaae5f --- /dev/null +++ b/test/otel_collect/host_metrics/windows/parameters.yml @@ -0,0 +1,23 @@ +# Windows Host Metrics OTLP integration test +# Tests that OTel host_metrics collects system metrics on Windows +# and publishes to the OTLP monitoring endpoint with cluster_name + +receivers: [] + +test_case: "win_otlp_host_metrics" +validate_type: "feature" +data_type: "metrics" + +# Number of seconds the agent should run and collect metrics (3 minutes) +agent_collection_period: 180 + +cloudwatch_agent_config: "agent_config.json" + +# Validate host metrics are published via OTLP +metric_namespace: "CWAgent/System" +metric_validation: + - metric_name: "cpu_time" + metric_sample_count: 1 + + - metric_name: "memory_usage" + metric_sample_count: 1 diff --git a/test/otel_collect/otlp/agent_configs/otlp_config.json b/test/otel_collect/otlp/agent_configs/otlp_config.json deleted file mode 100644 index 85a83933..00000000 --- a/test/otel_collect/otlp/agent_configs/otlp_config.json +++ /dev/null @@ -1 +0,0 @@ -{"agent":{"region":"us-west-2"},"opentelemetry":{"collect":{"otlp":{"grpc_endpoint":"127.0.0.1:4317","http_endpoint":"127.0.0.1:4318"}}}} diff --git a/test/otel_collect/otlp/linux/agent_configs/otlp_config.json b/test/otel_collect/otlp/linux/agent_configs/otlp_config.json new file mode 100644 index 00000000..3edc5c3b --- /dev/null +++ b/test/otel_collect/otlp/linux/agent_configs/otlp_config.json @@ -0,0 +1,14 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "cluster_name": "integ-test-cluster", + "collect": { + "otlp": { + "grpc_endpoint": "127.0.0.1:4317", + "http_endpoint": "127.0.0.1:4318" + } + } + } +} diff --git a/test/otel_collect/otlp/otlp_test.go b/test/otel_collect/otlp/linux/otlp_test.go similarity index 100% rename from test/otel_collect/otlp/otlp_test.go rename to test/otel_collect/otlp/linux/otlp_test.go diff --git a/test/otel_collect/otlp/windows/agent_config.json b/test/otel_collect/otlp/windows/agent_config.json new file mode 100644 index 00000000..9eb57b0e --- /dev/null +++ b/test/otel_collect/otlp/windows/agent_config.json @@ -0,0 +1,17 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "cluster_name": "integ-test-cluster", + "collect": { + "otlp": { + "grpc_endpoint": "127.0.0.1:4317", + "http_endpoint": "127.0.0.1:4318" + }, + "host_metrics": { + "collection_interval": 30 + } + } + } +} diff --git a/test/otel_collect/otlp/windows/parameters.yml b/test/otel_collect/otlp/windows/parameters.yml new file mode 100644 index 00000000..f1b36542 --- /dev/null +++ b/test/otel_collect/otlp/windows/parameters.yml @@ -0,0 +1,24 @@ +# Windows OTLP integration test +# Tests that OTel host_metrics + OTLP receiver work on Windows +# and publish to the OTLP monitoring endpoint + +receivers: [] + +test_case: "win_otlp_host_metrics" +validate_type: "feature" +data_type: "metrics" + +# Number of seconds the agent should run and collect metrics (3 minutes) +agent_collection_period: 180 + +cloudwatch_agent_config: "agent_config.json" + +# Validate host metrics are published via OTLP +# Host metrics on Windows use system.cpu.time, system.memory.usage, etc. +metric_namespace: "CWAgent/System" +metric_validation: + - metric_name: "cpu_time" + metric_sample_count: 1 + + - metric_name: "memory_usage" + metric_sample_count: 1 diff --git a/test/otel_collect/prometheus/agent_configs/prometheus_config.json b/test/otel_collect/prometheus/agent_configs/prometheus_config.json deleted file mode 100644 index 1de36ecf..00000000 --- a/test/otel_collect/prometheus/agent_configs/prometheus_config.json +++ /dev/null @@ -1 +0,0 @@ -{"agent":{"region":"us-west-2"},"opentelemetry":{"collect":{"prometheus":{"config_path":"/opt/aws/prometheus.yml"}}}} diff --git a/test/otel_collect/prometheus/linux/agent_configs/prometheus_config.json b/test/otel_collect/prometheus/linux/agent_configs/prometheus_config.json new file mode 100644 index 00000000..725f8452 --- /dev/null +++ b/test/otel_collect/prometheus/linux/agent_configs/prometheus_config.json @@ -0,0 +1,13 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "cluster_name": "integ-test-cluster", + "collect": { + "prometheus": { + "config_path": "/opt/aws/prometheus.yml" + } + } + } +} diff --git a/test/otel_collect/prometheus/prometheus_test.go b/test/otel_collect/prometheus/linux/prometheus_test.go similarity index 100% rename from test/otel_collect/prometheus/prometheus_test.go rename to test/otel_collect/prometheus/linux/prometheus_test.go diff --git a/test/otel_collect/prometheus/resources/prometheus_metrics b/test/otel_collect/prometheus/linux/resources/prometheus_metrics similarity index 100% rename from test/otel_collect/prometheus/resources/prometheus_metrics rename to test/otel_collect/prometheus/linux/resources/prometheus_metrics diff --git a/test/otel_collect/prometheus/linux/resources/prometheus_scrape_config.yaml b/test/otel_collect/prometheus/linux/resources/prometheus_scrape_config.yaml new file mode 100644 index 00000000..9d7a3578 --- /dev/null +++ b/test/otel_collect/prometheus/linux/resources/prometheus_scrape_config.yaml @@ -0,0 +1,13 @@ +scrape_configs: + - job_name: node + static_configs: + - targets: ['localhost:9100'] + relabel_configs: + - source_labels: [job] + regex: (.*) + target_label: renamed_job + replacement: $1 + - source_labels: [instance] + regex: (.*) + target_label: host + replacement: ${1} diff --git a/test/otel_collect/prometheus/resources/prometheus_scrape_config.yaml b/test/otel_collect/prometheus/resources/prometheus_scrape_config.yaml deleted file mode 100644 index 088eade5..00000000 --- a/test/otel_collect/prometheus/resources/prometheus_scrape_config.yaml +++ /dev/null @@ -1,4 +0,0 @@ -scrape_configs: - - job_name: node - static_configs: - - targets: ['localhost:9100'] diff --git a/test/otel_collect/prometheus/windows/agent_config.json b/test/otel_collect/prometheus/windows/agent_config.json new file mode 100644 index 00000000..1f2bf690 --- /dev/null +++ b/test/otel_collect/prometheus/windows/agent_config.json @@ -0,0 +1,13 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "cluster_name": "integ-test-cluster", + "collect": { + "prometheus": { + "config_path": "C:\\prometheus.yml" + } + } + } +} diff --git a/test/otel_collect/prometheus/windows/parameters.yml b/test/otel_collect/prometheus/windows/parameters.yml new file mode 100644 index 00000000..42ddc1a8 --- /dev/null +++ b/test/otel_collect/prometheus/windows/parameters.yml @@ -0,0 +1,45 @@ +# Windows Prometheus OTLP integration test +# Tests that OTel Prometheus receiver scrapes metrics on Windows +# and publishes to the OTLP monitoring endpoint + +receivers: [] + +test_case: "win_otlp_prometheus" +validate_type: "feature" +data_type: "metrics" + +# Number of seconds the agent should run and collect metrics (3 minutes) +agent_collection_period: 180 + +cloudwatch_agent_config: "agent_config.json" + +# Preparation script: writes prometheus scrape config and starts a fake metrics server +# The validator will run this PowerShell before starting the agent +preparation_cmd: | + Set-Content -Path "C:\prometheus.yml" -Value (Get-Content "C:\prometheus_scrape_config.yaml") + $listener = [System.Net.HttpListener]::new() + $listener.Prefixes.Add("http://+:9100/") + $listener.Start() + $job = Start-Job -ScriptBlock { + param($metrics) + $l = [System.Net.HttpListener]::new() + $l.Prefixes.Add("http://+:9100/") + $l.Start() + while ($l.IsListening) { + $ctx = $l.GetContext() + $response = $ctx.Response + $buffer = [System.Text.Encoding]::UTF8.GetBytes($metrics) + $response.ContentLength64 = $buffer.Length + $response.OutputStream.Write($buffer, 0, $buffer.Length) + $response.Close() + } + } -ArgumentList (Get-Content "C:\prometheus_metrics" -Raw) + +# Validate prometheus metrics are published via OTLP +metric_namespace: "CWAgent" +metric_validation: + - metric_name: "node_cpu_seconds_total" + metric_sample_count: 1 + + - metric_name: "node_memory_MemAvailable_bytes" + metric_sample_count: 1 diff --git a/test/otel_collect/prometheus/windows/prometheus_metrics b/test/otel_collect/prometheus/windows/prometheus_metrics new file mode 100644 index 00000000..1e158f2a --- /dev/null +++ b/test/otel_collect/prometheus/windows/prometheus_metrics @@ -0,0 +1,13 @@ +# HELP node_cpu_seconds_total Seconds the CPUs spent in each mode. +# TYPE node_cpu_seconds_total counter +node_cpu_seconds_total{cpu="0",mode="idle"} 12345.67 +node_cpu_seconds_total{cpu="0",mode="system"} 234.56 +# HELP node_memory_MemAvailable_bytes Memory information field MemAvailable_bytes. +# TYPE node_memory_MemAvailable_bytes gauge +node_memory_MemAvailable_bytes 4123456789 +# HELP node_filesystem_avail_bytes Filesystem space available to non-root users in bytes. +# TYPE node_filesystem_avail_bytes gauge +node_filesystem_avail_bytes{device="C:",fstype="ntfs",mountpoint="C:"} 50000000000 +# HELP node_network_receive_bytes_total Network device statistic receive_bytes. +# TYPE node_network_receive_bytes_total counter +node_network_receive_bytes_total{device="Ethernet"} 987654321 diff --git a/test/otel_collect/prometheus/windows/prometheus_scrape_config.yaml b/test/otel_collect/prometheus/windows/prometheus_scrape_config.yaml new file mode 100644 index 00000000..9d7a3578 --- /dev/null +++ b/test/otel_collect/prometheus/windows/prometheus_scrape_config.yaml @@ -0,0 +1,13 @@ +scrape_configs: + - job_name: node + static_configs: + - targets: ['localhost:9100'] + relabel_configs: + - source_labels: [job] + regex: (.*) + target_label: renamed_job + replacement: $1 + - source_labels: [instance] + regex: (.*) + target_label: host + replacement: ${1} From d442e36ca89f69d0a4a5f5f99e67c83c7a0e8ab1 Mon Sep 17 00:00:00 2001 From: Michael Commey Date: Tue, 21 Jul 2026 20:53:31 +0000 Subject: [PATCH 2/5] fix windows, ecs and on-prem tests --- generator/test_case_generator.go | 40 ++- .../ecs/host_metrics/host_metrics_ecs_test.go | 81 ++++++ .../ecs/host_metrics/resources/config.json | 14 + .../agent_configs/host_metrics_config.json | 0 .../host_metrics}/host_metrics_test.go | 5 +- .../otlp}/agent_configs/otlp_config.json | 0 test/otel_collect/linux/otlp/otlp_test.go | 241 ++++++++++++++++++ .../agent_configs/prometheus_config.json | 0 .../prometheus}/prometheus_test.go | 5 +- .../prometheus}/resources/prometheus_metrics | 0 .../resources/prometheus_scrape_config.yaml | 0 test/otel_collect/otlp/linux/otlp_test.go | 118 --------- test/otel_collect/otlp/windows/parameters.yml | 24 -- test/otel_collect/otlpvalidation/validate.go | 18 ++ .../host_metrics}/agent_config.json | 0 .../host_metrics}/parameters.yml | 0 test/otel_collect/windows/otlp/otlp_unix.go | 13 + .../otel_collect/windows/otlp/otlp_windows.go | 61 +++++ .../otlp/resources/config.json} | 5 +- .../prometheus}/agent_config.json | 0 .../prometheus}/parameters.yml | 0 .../prometheus}/prometheus_metrics | 0 .../prometheus}/prometheus_scrape_config.yaml | 0 util/common/metrics.go | 3 + util/common/otlp.go | 96 +++++++ util/otelmetrics/validate.go | 73 ++++++ validator/main.go | 3 + validator/models/validation_config.go | 2 +- 28 files changed, 625 insertions(+), 177 deletions(-) create mode 100644 test/otel_collect/ecs/host_metrics/host_metrics_ecs_test.go create mode 100644 test/otel_collect/ecs/host_metrics/resources/config.json rename test/otel_collect/{host_metrics/linux => linux/host_metrics}/agent_configs/host_metrics_config.json (100%) rename test/otel_collect/{host_metrics/linux => linux/host_metrics}/host_metrics_test.go (93%) rename test/otel_collect/{otlp/linux => linux/otlp}/agent_configs/otlp_config.json (100%) create mode 100644 test/otel_collect/linux/otlp/otlp_test.go rename test/otel_collect/{prometheus/linux => linux/prometheus}/agent_configs/prometheus_config.json (100%) rename test/otel_collect/{prometheus/linux => linux/prometheus}/prometheus_test.go (96%) rename test/otel_collect/{prometheus/linux => linux/prometheus}/resources/prometheus_metrics (100%) rename test/otel_collect/{prometheus/linux => linux/prometheus}/resources/prometheus_scrape_config.yaml (100%) delete mode 100644 test/otel_collect/otlp/linux/otlp_test.go delete mode 100644 test/otel_collect/otlp/windows/parameters.yml rename test/otel_collect/{host_metrics/windows => windows/host_metrics}/agent_config.json (100%) rename test/otel_collect/{host_metrics/windows => windows/host_metrics}/parameters.yml (100%) create mode 100644 test/otel_collect/windows/otlp/otlp_unix.go create mode 100644 test/otel_collect/windows/otlp/otlp_windows.go rename test/otel_collect/{otlp/windows/agent_config.json => windows/otlp/resources/config.json} (65%) rename test/otel_collect/{prometheus/windows => windows/prometheus}/agent_config.json (100%) rename test/otel_collect/{prometheus/windows => windows/prometheus}/parameters.yml (100%) rename test/otel_collect/{prometheus/windows => windows/prometheus}/prometheus_metrics (100%) rename test/otel_collect/{prometheus/windows => windows/prometheus}/prometheus_scrape_config.yaml (100%) create mode 100644 util/common/otlp.go create mode 100644 util/otelmetrics/validate.go diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index f9751dc8..3f646854 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -89,9 +89,9 @@ var testTypeToTestConfig = map[string][]testConfig{ // }, "ec2_linux_onprem": { {testDir: "./test/cloudwatchlogs"}, - {testDir: "./test/otel_collect/otlp/linux"}, - {testDir: "./test/otel_collect/host_metrics/linux"}, - {testDir: "./test/otel_collect/prometheus/linux"}, + {testDir: "./test/otel_collect/linux/otlp"}, + {testDir: "./test/otel_collect/linux/host_metrics"}, + {testDir: "./test/otel_collect/linux/prometheus"}, }, testTypeKeyEc2Linux: { // {testDir: "./test/ca_bundle"}, @@ -140,10 +140,10 @@ var testTypeToTestConfig = map[string][]testConfig{ // testDir: "./test/otel_collect/database_insights", // excludedOs: map[string]struct{}{"ol8": {}, "ubuntu-25": {}}, // }, - {testDir: "./test/otel_collect/host_metrics/linux"}, - {testDir: "./test/otel_collect/otlp/linux"}, + {testDir: "./test/otel_collect/linux/host_metrics"}, + {testDir: "./test/otel_collect/linux/otlp"}, { - testDir: "./test/otel_collect/prometheus/linux", + testDir: "./test/otel_collect/linux/prometheus", excludedOs: map[string]struct{}{"rhel8": {}, "ol8": {}, "sles-15": {}}, }, // { @@ -326,9 +326,9 @@ var testTypeToTestConfig = map[string][]testConfig{ // targets: map[string]map[string]struct{}{"os": {"win-2019": {}}}, // }, // {testDir: "../../../test/ssm_document"}, - {testDir: "../../../test/otel_collect/otlp/windows"}, - {testDir: "../../../test/otel_collect/host_metrics/windows"}, - {testDir: "../../../test/otel_collect/prometheus/windows"}, + {testDir: "../../../test/otel_collect/windows/otlp"}, + {testDir: "../../../test/otel_collect/windows/host_metrics"}, + {testDir: "../../../test/otel_collect/windows/prometheus"}, // assume role test doesn't add much value, and it already being tested with linux //{testDir: "../../../test/assume_role"}, }, @@ -381,15 +381,7 @@ var testTypeToTestConfig = map[string][]testConfig{ // targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, // }, { - testDir: "./test/otel_collect/otlp/linux", - targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, - }, - { - testDir: "./test/otel_collect/host_metrics/linux", - targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, - }, - { - testDir: "./test/otel_collect/prometheus/linux", + testDir: "./test/otel_collect/ecs/host_metrics", targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, }, }, @@ -572,9 +564,9 @@ var partitionTests = map[string]partition{ ami: []string{"cloudwatch-agent-integration-test-aarch64-al2023*"}, excludedTestDirs: map[string]struct{}{ "./test/otel_collect/database_insights": {}, - "./test/otel_collect/host_metrics/linux": {}, - "./test/otel_collect/otlp/linux": {}, - "./test/otel_collect/prometheus/linux": {}, + "./test/otel_collect/linux/host_metrics": {}, + "./test/otel_collect/linux/otlp": {}, + "./test/otel_collect/linux/prometheus": {}, }, testConfigOverrides: map[string]testConfig{ "./test/metric_value_benchmark": { @@ -593,9 +585,9 @@ var partitionTests = map[string]partition{ ami: []string{"cloudwatch-agent-integration-test-aarch64-al2023*"}, excludedTestDirs: map[string]struct{}{ "./test/otel_collect/database_insights": {}, - "./test/otel_collect/host_metrics/linux": {}, - "./test/otel_collect/otlp/linux": {}, - "./test/otel_collect/prometheus/linux": {}, + "./test/otel_collect/linux/host_metrics": {}, + "./test/otel_collect/linux/otlp": {}, + "./test/otel_collect/linux/prometheus": {}, }, testConfigOverrides: map[string]testConfig{ "./test/metric_value_benchmark": { diff --git a/test/otel_collect/ecs/host_metrics/host_metrics_ecs_test.go b/test/otel_collect/ecs/host_metrics/host_metrics_ecs_test.go new file mode 100644 index 00000000..7f755b7d --- /dev/null +++ b/test/otel_collect/ecs/host_metrics/host_metrics_ecs_test.go @@ -0,0 +1,81 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package ecs + +import ( + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/otlpvalidation" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" +) + +func init() { + environment.RegisterEnvironmentMetaDataFlags() +} + +// ECSHostMetricsTestRunner validates that the OpenTelemetry (V2) host_metrics +// component, running as the CloudWatch Agent ECS daemon, publishes metrics to +// the OTLP endpoint (CloudWatch). The agent config is applied by terraform via +// the CW_CONFIG_CONTENT SSM parameter, so GetAgentConfigFileName returns "" and +// the ECS harness validates the already-running daemon without a restart. +type ECSHostMetricsTestRunner struct { + test_runner.BaseTestRunner +} + +var _ test_runner.ITestRunner = (*ECSHostMetricsTestRunner)(nil) + +func (t *ECSHostMetricsTestRunner) GetTestName() string { return "ecs_otlp_host_metrics" } + +// GetAgentConfigFileName returns "" so the harness uses the config terraform +// already loaded from resources/config.json (no daemon restart needed). +func (t *ECSHostMetricsTestRunner) GetAgentConfigFileName() string { return "" } + +func (t *ECSHostMetricsTestRunner) GetMeasuredMetrics() []string { + return []string{ + "system.cpu.utilization", + "system.memory.utilization", + "system.network.io", + "system.disk.operations", + } +} + +func (t *ECSHostMetricsTestRunner) Validate() status.TestGroupResult { + env := environment.GetEnvironmentMetaData() + // Isolate this run's metrics by the ECS cluster ARN, which the agent's + // resourcedetection processor stamps as a resource attribute on ECS. + labels := map[string]string{ + "@resource.aws.ecs.cluster.arn": env.EcsClusterArn, + } + return otlpvalidation.ValidateOtlpMetricsWithLabels(t.GetTestName(), env.Region, t.GetMeasuredMetrics(), labels) +} + +func TestECSOtelHostMetricsSuite(t *testing.T) { + suite.Run(t, new(ECSHostMetricsTestSuite)) +} + +type ECSHostMetricsTestSuite struct { + suite.Suite + test_runner.TestSuite +} + +func (suite *ECSHostMetricsTestSuite) GetSuiteName() string { + return "ECSOtelHostMetrics" +} + +func (suite *ECSHostMetricsTestSuite) TestAllInSuite() { + env := environment.GetEnvironmentMetaData() + ecsTestRunner := &test_runner.ECSTestRunner{ + Runner: &ECSHostMetricsTestRunner{}, + RunStrategy: &test_runner.ECSAgentRunStrategy{}, + Env: *env, + } + ecsTestRunner.Run(suite, env) + suite.Assert().Equal(status.SUCCESSFUL, suite.Result.GetStatus(), "ECS OTLP Host Metrics Test Suite Failed") +} diff --git a/test/otel_collect/ecs/host_metrics/resources/config.json b/test/otel_collect/ecs/host_metrics/resources/config.json new file mode 100644 index 00000000..a2e4ea3d --- /dev/null +++ b/test/otel_collect/ecs/host_metrics/resources/config.json @@ -0,0 +1,14 @@ +{ + "agent": { + "run_as_user": "root", + "debug": true + }, + "opentelemetry": { + "cluster_name": "cwagent-ecs-integ-test", + "collect": { + "host_metrics": { + "collection_interval": 30 + } + } + } +} diff --git a/test/otel_collect/host_metrics/linux/agent_configs/host_metrics_config.json b/test/otel_collect/linux/host_metrics/agent_configs/host_metrics_config.json similarity index 100% rename from test/otel_collect/host_metrics/linux/agent_configs/host_metrics_config.json rename to test/otel_collect/linux/host_metrics/agent_configs/host_metrics_config.json diff --git a/test/otel_collect/host_metrics/linux/host_metrics_test.go b/test/otel_collect/linux/host_metrics/host_metrics_test.go similarity index 93% rename from test/otel_collect/host_metrics/linux/host_metrics_test.go rename to test/otel_collect/linux/host_metrics/host_metrics_test.go index 747c5e2b..ed3da6ec 100644 --- a/test/otel_collect/host_metrics/linux/host_metrics_test.go +++ b/test/otel_collect/linux/host_metrics/host_metrics_test.go @@ -31,9 +31,8 @@ type HostMetricsTestRunner struct { var _ test_runner.ITestRunner = (*HostMetricsTestRunner)(nil) func (t *HostMetricsTestRunner) Validate() status.TestGroupResult { - return otlpvalidation.ValidateOtlpMetricsWithLabels(t.GetTestName(), t.env.Region, t.GetMeasuredMetrics(), map[string]string{ - "@resource.host.id": t.env.InstanceId, - }) + return otlpvalidation.ValidateOtlpMetricsWithLabels(t.GetTestName(), t.env.Region, t.GetMeasuredMetrics(), + otlpvalidation.ResourceHostIDLabels(t.env.AgentStartCommand, t.env.InstanceId)) } func (t *HostMetricsTestRunner) GetTestName() string { return "HostMetrics" } diff --git a/test/otel_collect/otlp/linux/agent_configs/otlp_config.json b/test/otel_collect/linux/otlp/agent_configs/otlp_config.json similarity index 100% rename from test/otel_collect/otlp/linux/agent_configs/otlp_config.json rename to test/otel_collect/linux/otlp/agent_configs/otlp_config.json diff --git a/test/otel_collect/linux/otlp/otlp_test.go b/test/otel_collect/linux/otlp/otlp_test.go new file mode 100644 index 00000000..1f148b13 --- /dev/null +++ b/test/otel_collect/linux/otlp/otlp_test.go @@ -0,0 +1,241 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package otlp + +import ( + "bytes" + "context" + "fmt" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/otlpvalidation" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" + "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" + "github.com/aws/amazon-cloudwatch-agent-test/util/common/traces/base" + "github.com/aws/amazon-cloudwatch-agent-test/util/common/traces/otlp" +) + +func init() { + environment.RegisterEnvironmentMetaDataFlags() +} + +const ( + otlpRuntime = 3 * time.Minute + otlpEndpoint = "http://127.0.0.1:4318" + + // traceTestType is used as an X-Ray annotation value so the trace validator + // can isolate this run's traces. It is set by the test (not derived from + // IMDS), so it works in both EC2 and on-prem mode. + traceTestType = "otel_collect_otlp_traces" + + // otlpLogGroup is the CloudWatch Logs group where OTLP logs published via the + // V2 opentelemetry pipeline land. NOTE: confirm this matches the group the + // agent actually writes to on a real run; adjust if the V2 OTLP logs path + // uses a different group/stream. + otlpLogGroup = "/aws/cwagent" +) + +type OtlpCollectTestRunner struct { + test_runner.BaseTestRunner + env *environment.MetaData +} + +var _ test_runner.ITestRunner = (*OtlpCollectTestRunner)(nil) + +func (t *OtlpCollectTestRunner) Validate() status.TestGroupResult { + var results []status.TestResult + + // Metrics + metricResult := otlpvalidation.ValidateOtlpMetricsWithLabels(t.GetTestName(), t.env.Region, t.GetMeasuredMetrics(), + otlpvalidation.ResourceHostIDLabels(t.env.AgentStartCommand, t.env.InstanceId)) + results = append(results, metricResult.TestResults...) + + // Traces + results = append(results, t.validateTraces()) + + // Logs + results = append(results, t.validateLogs()) + + return status.TestGroupResult{Name: t.GetTestName(), TestResults: results} +} + +// validateTraces confirms the OTLP traces this run generated are queryable in +// X-Ray, filtered by the instance_id annotation for isolation. +func (t *OtlpCollectTestRunner) validateTraces() status.TestResult { + annotations := map[string]interface{}{ + "test_type": traceTestType, + "instance_id": t.env.InstanceId, + } + err := base.ValidateTraceSegments(time.Now().Add(-otlpRuntime), time.Now(), annotations, nil) + if err != nil { + return status.TestResult{Name: "OTLP_Traces", Status: status.FAILED, Reason: err} + } + return status.TestResult{Name: "OTLP_Traces", Status: status.SUCCESSFUL} +} + +// validateLogs confirms the OTLP logs this run published are present in +// CloudWatch Logs for this instance. +func (t *OtlpCollectTestRunner) validateLogs() status.TestResult { + since := time.Now().Add(-otlpRuntime) + until := time.Now() + + if len(awsservice.GetLogStreams(otlpLogGroup)) == 0 { + return status.TestResult{Name: "OTLP_Logs", Status: status.FAILED, + Reason: fmt.Errorf("no log streams found in log group %s", otlpLogGroup)} + } + + err := awsservice.ValidateLogs( + otlpLogGroup, + t.env.InstanceId, + &since, + &until, + awsservice.AssertLogsNotEmpty(), + awsservice.AssertPerLog( + awsservice.AssertLogContainsSubstring(fmt.Sprintf("\"InstanceId\":\"%s\"", t.env.InstanceId)), + ), + ) + if err != nil { + return status.TestResult{Name: "OTLP_Logs", Status: status.FAILED, Reason: err} + } + return status.TestResult{Name: "OTLP_Logs", Status: status.SUCCESSFUL} +} + +func (t *OtlpCollectTestRunner) GetTestName() string { return "OtelCollectOTLP" } +func (t *OtlpCollectTestRunner) GetAgentRunDuration() time.Duration { return otlpRuntime } +func (t *OtlpCollectTestRunner) GetAgentConfigFileName() string { return "otlp_config.json" } +func (t *OtlpCollectTestRunner) GetMeasuredMetrics() []string { + return []string{"otlp_test_counter", "otlp_test_gauge"} +} + +func (t *OtlpCollectTestRunner) SetupAfterAgentRun() error { + go t.sendTestMetrics() + go t.generateTraces() + go t.sendTestLogs() + return nil +} + +// generateTraces pushes OTLP traces to the agent's OTLP gRPC receiver +func (t *OtlpCollectTestRunner) generateTraces() { + generator := otlp.NewLoadGenerator(&base.TraceGeneratorConfig{ + Interval: 10 * time.Second, + Annotations: map[string]interface{}{ + "test_type": traceTestType, + "instance_id": t.env.InstanceId, + }, + Attributes: []attribute.KeyValue{ + attribute.String("test_type", traceTestType), + attribute.String("instance_id", t.env.InstanceId), + }, + }) + generator.StartSendingTraces(context.Background()) +} + +// sendTestLogs pushes OTLP logs to the agent's OTLP HTTP receiver (/v1/logs). +func (t *OtlpCollectTestRunner) sendTestLogs() { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + timeout := time.After(otlpRuntime - 30*time.Second) + for { + select { + case <-timeout: + return + case <-ticker.C: + payload := buildOtlpLogsPayload(t.env.InstanceId) + req, _ := http.NewRequest("POST", otlpEndpoint+"/v1/logs", bytes.NewReader(payload)) + req.Header.Set("Content-Type", "application/json") + http.DefaultClient.Do(req) + } + } +} + +func (t *OtlpCollectTestRunner) sendTestMetrics() { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + timeout := time.After(otlpRuntime - 30*time.Second) + for { + select { + case <-timeout: + return + case <-ticker.C: + payload := buildOtlpMetricsPayload(t.env.InstanceId) + req, _ := http.NewRequest("POST", otlpEndpoint+"/v1/metrics", bytes.NewReader(payload)) + req.Header.Set("Content-Type", "application/json") + http.DefaultClient.Do(req) //nolint:errcheck + } + } +} + +var startTime = time.Now().UnixNano() + +func buildOtlpMetricsPayload(instanceId string) []byte { + now := time.Now().UnixNano() + payload := fmt.Sprintf(`{ + "resourceMetrics": [{ + "resource": {"attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}, + "scopeMetrics": [{ + "metrics": [ + { + "name": "otlp_test_counter", + "sum": { + "dataPoints": [{"asInt": "1", "startTimeUnixNano": "%d", "timeUnixNano": "%d", "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}], + "isMonotonic": true, + "aggregationTemporality": 2 + } + }, + { + "name": "otlp_test_gauge", + "gauge": { + "dataPoints": [{"asDouble": 42.0, "timeUnixNano": "%d", "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}] + } + } + ] + }] + }] +}`, instanceId, startTime, now, instanceId, now, instanceId) + return []byte(payload) +} + +func buildOtlpLogsPayload(instanceId string) []byte { + now := time.Now().UnixNano() + payload := fmt.Sprintf(`{ + "resourceLogs": [{ + "resource": {"attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}, + "scopeLogs": [{ + "logRecords": [{ + "timeUnixNano": "%d", + "severityText": "INFO", + "body": {"stringValue": "otlp integration test log"}, + "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}] + }] + }] + }] +}`, instanceId, now, instanceId) + return []byte(payload) +} + +func TestOTLPCollect(t *testing.T) { + env := environment.GetEnvironmentMetaData() + + testRunner := &OtlpCollectTestRunner{ + BaseTestRunner: test_runner.BaseTestRunner{}, + env: env, + } + runner := &test_runner.TestRunner{TestRunner: testRunner} + result := runner.Run() + + for _, r := range result.TestResults { + require.Equal(t, status.SUCCESSFUL, r.Status, "%s failed: %v", r.Name, r.Reason) + } +} diff --git a/test/otel_collect/prometheus/linux/agent_configs/prometheus_config.json b/test/otel_collect/linux/prometheus/agent_configs/prometheus_config.json similarity index 100% rename from test/otel_collect/prometheus/linux/agent_configs/prometheus_config.json rename to test/otel_collect/linux/prometheus/agent_configs/prometheus_config.json diff --git a/test/otel_collect/prometheus/linux/prometheus_test.go b/test/otel_collect/linux/prometheus/prometheus_test.go similarity index 96% rename from test/otel_collect/prometheus/linux/prometheus_test.go rename to test/otel_collect/linux/prometheus/prometheus_test.go index 20ec9efb..04ecd271 100644 --- a/test/otel_collect/prometheus/linux/prometheus_test.go +++ b/test/otel_collect/linux/prometheus/prometheus_test.go @@ -41,9 +41,8 @@ type PrometheusOtelTestRunner struct { var _ test_runner.ITestRunner = (*PrometheusOtelTestRunner)(nil) func (t *PrometheusOtelTestRunner) Validate() status.TestGroupResult { - return otlpvalidation.ValidateOtlpMetricsWithLabels(t.GetTestName(), t.env.Region, t.GetMeasuredMetrics(), map[string]string{ - "@resource.host.id": t.env.InstanceId, - }) + return otlpvalidation.ValidateOtlpMetricsWithLabels(t.GetTestName(), t.env.Region, t.GetMeasuredMetrics(), + otlpvalidation.ResourceHostIDLabels(t.env.AgentStartCommand, t.env.InstanceId)) } func (t *PrometheusOtelTestRunner) GetTestName() string { return "OtelCollectPrometheus" } diff --git a/test/otel_collect/prometheus/linux/resources/prometheus_metrics b/test/otel_collect/linux/prometheus/resources/prometheus_metrics similarity index 100% rename from test/otel_collect/prometheus/linux/resources/prometheus_metrics rename to test/otel_collect/linux/prometheus/resources/prometheus_metrics diff --git a/test/otel_collect/prometheus/linux/resources/prometheus_scrape_config.yaml b/test/otel_collect/linux/prometheus/resources/prometheus_scrape_config.yaml similarity index 100% rename from test/otel_collect/prometheus/linux/resources/prometheus_scrape_config.yaml rename to test/otel_collect/linux/prometheus/resources/prometheus_scrape_config.yaml diff --git a/test/otel_collect/otlp/linux/otlp_test.go b/test/otel_collect/otlp/linux/otlp_test.go deleted file mode 100644 index bb7dd73a..00000000 --- a/test/otel_collect/otlp/linux/otlp_test.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -//go:build !windows - -package otlp - -import ( - "bytes" - "fmt" - "net/http" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/aws/amazon-cloudwatch-agent-test/environment" - "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/otlpvalidation" - "github.com/aws/amazon-cloudwatch-agent-test/test/status" - "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" -) - -func init() { - environment.RegisterEnvironmentMetaDataFlags() -} - -const ( - otlpRuntime = 3 * time.Minute - otlpEndpoint = "http://127.0.0.1:4318" -) - -type OtlpCollectTestRunner struct { - test_runner.BaseTestRunner - env *environment.MetaData -} - -var _ test_runner.ITestRunner = (*OtlpCollectTestRunner)(nil) - -func (t *OtlpCollectTestRunner) Validate() status.TestGroupResult { - return otlpvalidation.ValidateOtlpMetricsWithLabels(t.GetTestName(), t.env.Region, t.GetMeasuredMetrics(), map[string]string{ - "@resource.host.id": t.env.InstanceId, - }) -} - -func (t *OtlpCollectTestRunner) GetTestName() string { return "OtelCollectOTLP" } -func (t *OtlpCollectTestRunner) GetAgentRunDuration() time.Duration { return otlpRuntime } -func (t *OtlpCollectTestRunner) GetAgentConfigFileName() string { return "otlp_config.json" } -func (t *OtlpCollectTestRunner) GetMeasuredMetrics() []string { - return []string{"otlp_test_counter", "otlp_test_gauge"} -} - -func (t *OtlpCollectTestRunner) SetupAfterAgentRun() error { - go t.sendTestMetrics() - return nil -} - -func (t *OtlpCollectTestRunner) sendTestMetrics() { - ticker := time.NewTicker(10 * time.Second) - defer ticker.Stop() - - timeout := time.After(otlpRuntime - 30*time.Second) - for { - select { - case <-timeout: - return - case <-ticker.C: - payload := buildOtlpMetricsPayload(t.env.InstanceId) - req, _ := http.NewRequest("POST", otlpEndpoint+"/v1/metrics", bytes.NewReader(payload)) - req.Header.Set("Content-Type", "application/json") - http.DefaultClient.Do(req) //nolint:errcheck - } - } -} - -var startTime = time.Now().UnixNano() - -func buildOtlpMetricsPayload(instanceId string) []byte { - now := time.Now().UnixNano() - payload := fmt.Sprintf(`{ - "resourceMetrics": [{ - "resource": {"attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}, - "scopeMetrics": [{ - "metrics": [ - { - "name": "otlp_test_counter", - "sum": { - "dataPoints": [{"asInt": "1", "startTimeUnixNano": "%d", "timeUnixNano": "%d", "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}], - "isMonotonic": true, - "aggregationTemporality": 2 - } - }, - { - "name": "otlp_test_gauge", - "gauge": { - "dataPoints": [{"asDouble": 42.0, "timeUnixNano": "%d", "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}] - } - } - ] - }] - }] -}`, instanceId, startTime, now, instanceId, now, instanceId) - return []byte(payload) -} - -func TestOTLPCollect(t *testing.T) { - env := environment.GetEnvironmentMetaData() - - testRunner := &OtlpCollectTestRunner{ - BaseTestRunner: test_runner.BaseTestRunner{}, - env: env, - } - runner := &test_runner.TestRunner{TestRunner: testRunner} - result := runner.Run() - - for _, r := range result.TestResults { - require.Equal(t, status.SUCCESSFUL, r.Status, "%s failed: %v", r.Name, r.Reason) - } -} diff --git a/test/otel_collect/otlp/windows/parameters.yml b/test/otel_collect/otlp/windows/parameters.yml deleted file mode 100644 index f1b36542..00000000 --- a/test/otel_collect/otlp/windows/parameters.yml +++ /dev/null @@ -1,24 +0,0 @@ -# Windows OTLP integration test -# Tests that OTel host_metrics + OTLP receiver work on Windows -# and publish to the OTLP monitoring endpoint - -receivers: [] - -test_case: "win_otlp_host_metrics" -validate_type: "feature" -data_type: "metrics" - -# Number of seconds the agent should run and collect metrics (3 minutes) -agent_collection_period: 180 - -cloudwatch_agent_config: "agent_config.json" - -# Validate host metrics are published via OTLP -# Host metrics on Windows use system.cpu.time, system.memory.usage, etc. -metric_namespace: "CWAgent/System" -metric_validation: - - metric_name: "cpu_time" - metric_sample_count: 1 - - - metric_name: "memory_usage" - metric_sample_count: 1 diff --git a/test/otel_collect/otlpvalidation/validate.go b/test/otel_collect/otlpvalidation/validate.go index b4972d60..5835bcec 100644 --- a/test/otel_collect/otlpvalidation/validate.go +++ b/test/otel_collect/otlpvalidation/validate.go @@ -10,12 +10,30 @@ import ( "fmt" "log" "os" + "strings" "time" "github.com/aws/amazon-cloudwatch-agent-test/test/status" "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" ) +// onPremiseMarker identifies the on-prem agent start command (-m onPremise). +const onPremiseMarker = "onPremise" + +// ResourceHostIDLabels returns the metric label filter keyed on the entity +// host.id (the EC2 instance id) for EC2 mode. +// +// In on-prem mode (agent started with "-m onPremise") the agent does not read +// EC2 IMDS, so it does not set @resource.host.id to the instance id. Filtering +// on it there would never match, so we return nil and validate metric presence +// only. Detection is based on the agent start command from the test matrix. +func ResourceHostIDLabels(agentStartCommand, instanceID string) map[string]string { + if strings.Contains(agentStartCommand, onPremiseMarker) { + return nil + } + return map[string]string{"@resource.host.id": instanceID} +} + const ( defaultMaxRetries = 3 defaultRetryInterval = 30 * time.Second diff --git a/test/otel_collect/host_metrics/windows/agent_config.json b/test/otel_collect/windows/host_metrics/agent_config.json similarity index 100% rename from test/otel_collect/host_metrics/windows/agent_config.json rename to test/otel_collect/windows/host_metrics/agent_config.json diff --git a/test/otel_collect/host_metrics/windows/parameters.yml b/test/otel_collect/windows/host_metrics/parameters.yml similarity index 100% rename from test/otel_collect/host_metrics/windows/parameters.yml rename to test/otel_collect/windows/host_metrics/parameters.yml diff --git a/test/otel_collect/windows/otlp/otlp_unix.go b/test/otel_collect/windows/otlp/otlp_unix.go new file mode 100644 index 00000000..f9b44d7a --- /dev/null +++ b/test/otel_collect/windows/otlp/otlp_unix.go @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package otlp + +import "errors" + +// Validate is only implemented on Windows; this stub lets the validator build on other platforms. +func Validate() error { + return errors.New("otel_collect windows OTLP validation is only supported on Windows") +} diff --git a/test/otel_collect/windows/otlp/otlp_windows.go b/test/otel_collect/windows/otlp/otlp_windows.go new file mode 100644 index 00000000..6793868b --- /dev/null +++ b/test/otel_collect/windows/otlp/otlp_windows.go @@ -0,0 +1,61 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build windows + +package otlp + +import ( + "context" + _ "embed" + "fmt" + "os" + "time" + + "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" + "github.com/aws/amazon-cloudwatch-agent-test/util/common" + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +//go:embed resources/config.json +var testConfigJSON string + +const ( + tmpConfigPath = "C:\\Users\\Administrator\\AppData\\Local\\Temp\\config.json" + otlpRuntime = 3 * time.Minute + sendInterval = 10 * time.Second +) + +// Validate runs the Windows OTLP test: start the agent with the V2 opentelemetry OTLP config, +// push OTLP metrics to the agent, then validate they reach CloudWatch via the PromQL endpoint. +func Validate() error { + instanceID := awsservice.GetInstanceId() + + if err := os.WriteFile(tmpConfigPath, []byte(testConfigJSON), 0644); err != nil { + return fmt.Errorf("could not write config file: %w", err) + } + if err := common.CopyFile(tmpConfigPath, common.ConfigOutputPath); err != nil { + return fmt.Errorf("could not copy config: %w", err) + } + if err := common.StartAgent(common.ConfigOutputPath, true, false); err != nil { + return fmt.Errorf("could not start agent: %w", err) + } + time.Sleep(5 * time.Second) + + // Push OTLP metrics to the agent's OTLP/HTTP receiver for the collection period. + if err := common.SendOTLPMetrics(common.DefaultOTLPHTTPEndpoint, instanceID, sendInterval, otlpRuntime); err != nil { + return fmt.Errorf("failed to send OTLP metrics: %w", err) + } + + _ = common.StopAgent() + + // Validate the pushed metrics landed in CloudWatch, isolated by the instance's host.id. + return otelmetrics.AssertMetricsPresent( + context.Background(), + "", // region resolved from AWS_REGION env + []string{"otlp_test_counter", "otlp_test_gauge"}, + map[string]string{"@resource.host.id": instanceID}, + 3, + 30*time.Second, + ) +} diff --git a/test/otel_collect/otlp/windows/agent_config.json b/test/otel_collect/windows/otlp/resources/config.json similarity index 65% rename from test/otel_collect/otlp/windows/agent_config.json rename to test/otel_collect/windows/otlp/resources/config.json index 9eb57b0e..0cca9afb 100644 --- a/test/otel_collect/otlp/windows/agent_config.json +++ b/test/otel_collect/windows/otlp/resources/config.json @@ -3,14 +3,11 @@ "region": "us-west-2" }, "opentelemetry": { - "cluster_name": "integ-test-cluster", + "cluster_name": "cwagent-windows-integ-test", "collect": { "otlp": { "grpc_endpoint": "127.0.0.1:4317", "http_endpoint": "127.0.0.1:4318" - }, - "host_metrics": { - "collection_interval": 30 } } } diff --git a/test/otel_collect/prometheus/windows/agent_config.json b/test/otel_collect/windows/prometheus/agent_config.json similarity index 100% rename from test/otel_collect/prometheus/windows/agent_config.json rename to test/otel_collect/windows/prometheus/agent_config.json diff --git a/test/otel_collect/prometheus/windows/parameters.yml b/test/otel_collect/windows/prometheus/parameters.yml similarity index 100% rename from test/otel_collect/prometheus/windows/parameters.yml rename to test/otel_collect/windows/prometheus/parameters.yml diff --git a/test/otel_collect/prometheus/windows/prometheus_metrics b/test/otel_collect/windows/prometheus/prometheus_metrics similarity index 100% rename from test/otel_collect/prometheus/windows/prometheus_metrics rename to test/otel_collect/windows/prometheus/prometheus_metrics diff --git a/test/otel_collect/prometheus/windows/prometheus_scrape_config.yaml b/test/otel_collect/windows/prometheus/prometheus_scrape_config.yaml similarity index 100% rename from test/otel_collect/prometheus/windows/prometheus_scrape_config.yaml rename to test/otel_collect/windows/prometheus/prometheus_scrape_config.yaml diff --git a/util/common/metrics.go b/util/common/metrics.go index e553d857..38c43b33 100644 --- a/util/common/metrics.go +++ b/util/common/metrics.go @@ -75,6 +75,9 @@ func StartSendingMetrics(receiver string, duration, sendingInterval time.Duratio err = SendPrometheusMetrics(cfg, duration) case "traces": err = SendAppSignalsTraceMetrics(duration) //does app signals have dimension for metric? + case "otlp": + // metricLogGroup is the instance id (set by the feature validator's GenerateLoad). + err = SendOTLPMetrics(DefaultOTLPHTTPEndpoint, metricLogGroup, sendingInterval, duration) default: } diff --git a/util/common/otlp.go b/util/common/otlp.go new file mode 100644 index 00000000..fd9c96ab --- /dev/null +++ b/util/common/otlp.go @@ -0,0 +1,96 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package common + +import ( + "bytes" + "fmt" + "log" + "net" + "net/http" + "time" +) + +// DefaultOTLPHTTPEndpoint is the agent's default OTLP/HTTP receiver endpoint. +const DefaultOTLPHTTPEndpoint = "http://127.0.0.1:4318" + +// SendOTLPMetrics pushes OTLP metrics to the agent's OTLP/HTTP receiver until the duration elapses (cross-platform). +func SendOTLPMetrics(endpoint, instanceID string, sendingInterval, duration time.Duration) error { + if endpoint == "" { + endpoint = DefaultOTLPHTTPEndpoint + } + deadline := time.Now().Add(duration) + ticker := time.NewTicker(sendingInterval) + defer ticker.Stop() + + for { + payload := buildOTLPMetricsPayload(instanceID) + req, err := http.NewRequest(http.MethodPost, endpoint+"/v1/metrics", bytes.NewReader(payload)) + if err != nil { + return fmt.Errorf("failed to build OTLP metrics request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + if resp, err := http.DefaultClient.Do(req); err != nil { + log.Printf("SendOTLPMetrics: post failed: %v", err) + } else { + resp.Body.Close() + } + + if time.Now().After(deadline) { + return nil + } + <-ticker.C + } +} + +// buildOTLPMetricsPayload builds an OTLP/HTTP JSON metrics payload with an InstanceId attribute for isolation. +func buildOTLPMetricsPayload(instanceID string) []byte { + now := time.Now().UnixNano() + return []byte(fmt.Sprintf(`{ + "resourceMetrics": [{ + "resource": {"attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}, + "scopeMetrics": [{ + "metrics": [ + { + "name": "otlp_test_counter", + "sum": { + "dataPoints": [{"asInt": "1", "timeUnixNano": "%d", "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}], + "isMonotonic": true, + "aggregationTemporality": 2 + } + }, + { + "name": "otlp_test_gauge", + "gauge": { + "dataPoints": [{"asDouble": 42.0, "timeUnixNano": "%d", "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}] + } + } + ] + }] + }] +}`, instanceID, now, instanceID, now, instanceID)) +} + +// StartPrometheusFakeServer serves static Prometheus metrics on the port until stop() is called (cross-platform). +func StartPrometheusFakeServer(port int, exposition string) (stop func(), err error) { + mux := http.NewServeMux() + mux.HandleFunc("/metrics", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/plain; version=0.0.4") + _, _ = w.Write([]byte(exposition)) + }) + + ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + return nil, fmt.Errorf("failed to listen on port %d: %w", port, err) + } + + srv := &http.Server{Handler: mux} + go func() { + if serveErr := srv.Serve(ln); serveErr != nil && serveErr != http.ErrServerClosed { + log.Printf("StartPrometheusFakeServer: serve stopped: %v", serveErr) + } + }() + + return func() { _ = srv.Close() }, nil +} diff --git a/util/otelmetrics/validate.go b/util/otelmetrics/validate.go new file mode 100644 index 00000000..d956d3bd --- /dev/null +++ b/util/otelmetrics/validate.go @@ -0,0 +1,73 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "context" + "fmt" + "os" + "strings" + "time" +) + +// resolveRegion returns the region, falling back to AWS env vars then us-west-2. +func resolveRegion(region string) string { + if region != "" { + return region + } + if r := os.Getenv("AWS_REGION"); r != "" { + return r + } + if r := os.Getenv("AWS_DEFAULT_REGION"); r != "" { + return r + } + return "us-west-2" +} + +// AssertMetricsPresent queries the CloudWatch OTLP PromQL endpoint and returns an error if any +// metric is missing after retries. Cross-platform (returns error, no test/status dependency). +func AssertMetricsPresent(ctx context.Context, region string, metrics []string, labels map[string]string, maxRetries int, retryInterval time.Duration) error { + region = resolveRegion(region) + client, err := NewClient(ctx, TestConfig{ + Region: region, + Endpoint: fmt.Sprintf("https://monitoring.%s.amazonaws.com", region), + Timeout: 30 * time.Second, + MaxRetries: 3, + SigningService: "monitoring", + }) + if err != nil { + return fmt.Errorf("failed to create otel metrics client: %w", err) + } + + validated := make(map[string]bool, len(metrics)) + for attempt := 0; attempt < maxRetries; attempt++ { + if attempt > 0 { + time.Sleep(retryInterval) + } + for _, m := range metrics { + if validated[m] { + continue + } + query := fmt.Sprintf(`{__name__="%s"`, m) + for k, v := range labels { + query += fmt.Sprintf(`, "%s"=~"%s"`, k, v) + } + query += "}" + if results, err := client.Query(ctx, query); err == nil && len(results) > 0 { + validated[m] = true + } + } + if len(validated) == len(metrics) { + return nil + } + } + + var missing []string + for _, m := range metrics { + if !validated[m] { + missing = append(missing, m) + } + } + return fmt.Errorf("metrics not found after %d attempts: %s", maxRetries, strings.Join(missing, ", ")) +} diff --git a/validator/main.go b/validator/main.go index 9727b72d..3a20f85c 100644 --- a/validator/main.go +++ b/validator/main.go @@ -19,6 +19,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent-test/test/log_state/windows_event_log" "github.com/aws/amazon-cloudwatch-agent-test/test/nvidia_gpu" "github.com/aws/amazon-cloudwatch-agent-test/test/restart" + otelcollectotlp "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/windows/otlp" "github.com/aws/amazon-cloudwatch-agent-test/test/ssm_document" "github.com/aws/amazon-cloudwatch-agent-test/test/workload_discovery" "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" @@ -67,6 +68,8 @@ func main() { err = windows_event_log.Validate() case "ssm_document": err = ssm_document.Validate() + case "otlp": + err = otelcollectotlp.Validate() default: err = errors.New("unsupported test name") } diff --git a/validator/models/validation_config.go b/validator/models/validation_config.go index 7294cf27..067b1585 100644 --- a/validator/models/validation_config.go +++ b/validator/models/validation_config.go @@ -15,7 +15,7 @@ import ( "gopkg.in/yaml.v3" ) -var supportedReceivers = []string{"logs", "statsd", "collectd", "system", "emf", "xray", "app_signals", "prometheus", "traces"} +var supportedReceivers = []string{"logs", "statsd", "collectd", "system", "emf", "xray", "app_signals", "prometheus", "traces", "otlp"} var retryCount = 0 type ValidateConfig interface { From 9a973d2696115acc4c0e9a41e3f7552a6aeaff8d Mon Sep 17 00:00:00 2001 From: Michael Commey Date: Tue, 21 Jul 2026 20:53:31 +0000 Subject: [PATCH 3/5] fix windows, ecs and on-prem tests --- .../ecs/host_metrics/host_metrics_ecs_test.go | 13 +- .../ecs/host_metrics/resources/config.json | 3 +- test/otel_collect/linux/otlp/otlp_test.go | 111 ++++++------------ .../linux/prometheus/prometheus_test.go | 21 ++-- test/otel_collect/otlpvalidation/labels.go | 44 +++++++ test/otel_collect/otlpvalidation/validate.go | 32 ----- .../windows/host_metrics/agent_config.json | 12 +- .../windows/host_metrics/host_metrics_unix.go | 13 ++ .../host_metrics/host_metrics_windows.go | 52 ++++++++ .../windows/host_metrics/parameters.yml | 27 ++--- .../host_metrics/resources/config.json | 13 ++ .../windows/otlp/agent_config.json | 5 + .../otel_collect/windows/otlp/otlp_windows.go | 24 ++-- test/otel_collect/windows/otlp/parameters.yml | 17 +++ .../windows/prometheus/agent_config.json | 12 +- .../windows/prometheus/parameters.yml | 49 ++------ .../windows/prometheus/prometheus_metrics | 13 -- .../windows/prometheus/prometheus_unix.go | 13 ++ .../windows/prometheus/prometheus_windows.go | 69 +++++++++++ .../windows/prometheus/resources/config.json | 13 ++ .../prometheus/resources/prometheus_metrics | 7 ++ .../prometheus_scrape.yaml} | 3 +- util/common/otlp.go | 46 +++++++- util/common/traces/otlp/generator.go | 3 +- validator/main.go | 8 +- 25 files changed, 380 insertions(+), 243 deletions(-) create mode 100644 test/otel_collect/otlpvalidation/labels.go create mode 100644 test/otel_collect/windows/host_metrics/host_metrics_unix.go create mode 100644 test/otel_collect/windows/host_metrics/host_metrics_windows.go create mode 100644 test/otel_collect/windows/host_metrics/resources/config.json create mode 100644 test/otel_collect/windows/otlp/agent_config.json create mode 100644 test/otel_collect/windows/otlp/parameters.yml delete mode 100644 test/otel_collect/windows/prometheus/prometheus_metrics create mode 100644 test/otel_collect/windows/prometheus/prometheus_unix.go create mode 100644 test/otel_collect/windows/prometheus/prometheus_windows.go create mode 100644 test/otel_collect/windows/prometheus/resources/config.json create mode 100644 test/otel_collect/windows/prometheus/resources/prometheus_metrics rename test/otel_collect/windows/prometheus/{prometheus_scrape_config.yaml => resources/prometheus_scrape.yaml} (82%) diff --git a/test/otel_collect/ecs/host_metrics/host_metrics_ecs_test.go b/test/otel_collect/ecs/host_metrics/host_metrics_ecs_test.go index 7f755b7d..cfeb30e6 100644 --- a/test/otel_collect/ecs/host_metrics/host_metrics_ecs_test.go +++ b/test/otel_collect/ecs/host_metrics/host_metrics_ecs_test.go @@ -20,11 +20,8 @@ func init() { environment.RegisterEnvironmentMetaDataFlags() } -// ECSHostMetricsTestRunner validates that the OpenTelemetry (V2) host_metrics -// component, running as the CloudWatch Agent ECS daemon, publishes metrics to -// the OTLP endpoint (CloudWatch). The agent config is applied by terraform via -// the CW_CONFIG_CONTENT SSM parameter, so GetAgentConfigFileName returns "" and -// the ECS harness validates the already-running daemon without a restart. +// ECSHostMetricsTestRunner validates that the V2 host_metrics component, running +// as an ECS daemon, publishes metrics to CloudWatch via the OTLP endpoint. type ECSHostMetricsTestRunner struct { test_runner.BaseTestRunner } @@ -33,8 +30,7 @@ var _ test_runner.ITestRunner = (*ECSHostMetricsTestRunner)(nil) func (t *ECSHostMetricsTestRunner) GetTestName() string { return "ecs_otlp_host_metrics" } -// GetAgentConfigFileName returns "" so the harness uses the config terraform -// already loaded from resources/config.json (no daemon restart needed). +// GetAgentConfigFileName returns "" — config is pre-loaded by Terraform via SSM, no restart needed. func (t *ECSHostMetricsTestRunner) GetAgentConfigFileName() string { return "" } func (t *ECSHostMetricsTestRunner) GetMeasuredMetrics() []string { @@ -48,8 +44,7 @@ func (t *ECSHostMetricsTestRunner) GetMeasuredMetrics() []string { func (t *ECSHostMetricsTestRunner) Validate() status.TestGroupResult { env := environment.GetEnvironmentMetaData() - // Isolate this run's metrics by the ECS cluster ARN, which the agent's - // resourcedetection processor stamps as a resource attribute on ECS. + // Filter by ECS cluster ARN stamped by the resourcedetection processor. labels := map[string]string{ "@resource.aws.ecs.cluster.arn": env.EcsClusterArn, } diff --git a/test/otel_collect/ecs/host_metrics/resources/config.json b/test/otel_collect/ecs/host_metrics/resources/config.json index a2e4ea3d..da23ab89 100644 --- a/test/otel_collect/ecs/host_metrics/resources/config.json +++ b/test/otel_collect/ecs/host_metrics/resources/config.json @@ -1,7 +1,6 @@ { "agent": { - "run_as_user": "root", - "debug": true + "run_as_user": "root" }, "opentelemetry": { "cluster_name": "cwagent-ecs-integ-test", diff --git a/test/otel_collect/linux/otlp/otlp_test.go b/test/otel_collect/linux/otlp/otlp_test.go index 1f148b13..5e6710cd 100644 --- a/test/otel_collect/linux/otlp/otlp_test.go +++ b/test/otel_collect/linux/otlp/otlp_test.go @@ -9,6 +9,7 @@ import ( "bytes" "context" "fmt" + "log" "net/http" "testing" "time" @@ -21,6 +22,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent-test/test/status" "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" + "github.com/aws/amazon-cloudwatch-agent-test/util/common" "github.com/aws/amazon-cloudwatch-agent-test/util/common/traces/base" "github.com/aws/amazon-cloudwatch-agent-test/util/common/traces/otlp" ) @@ -32,22 +34,15 @@ func init() { const ( otlpRuntime = 3 * time.Minute otlpEndpoint = "http://127.0.0.1:4318" - - // traceTestType is used as an X-Ray annotation value so the trace validator - // can isolate this run's traces. It is set by the test (not derived from - // IMDS), so it works in both EC2 and on-prem mode. + otlpGRPCAddr = "127.0.0.1:4317" traceTestType = "otel_collect_otlp_traces" - - // otlpLogGroup is the CloudWatch Logs group where OTLP logs published via the - // V2 opentelemetry pipeline land. NOTE: confirm this matches the group the - // agent actually writes to on a real run; adjust if the V2 OTLP logs path - // uses a different group/stream. - otlpLogGroup = "/aws/cwagent" + otlpLogGroup = "/aws/cwagent" ) type OtlpCollectTestRunner struct { test_runner.BaseTestRunner - env *environment.MetaData + env *environment.MetaData + startedAt time.Time } var _ test_runner.ITestRunner = (*OtlpCollectTestRunner)(nil) @@ -57,7 +52,7 @@ func (t *OtlpCollectTestRunner) Validate() status.TestGroupResult { // Metrics metricResult := otlpvalidation.ValidateOtlpMetricsWithLabels(t.GetTestName(), t.env.Region, t.GetMeasuredMetrics(), - otlpvalidation.ResourceHostIDLabels(t.env.AgentStartCommand, t.env.InstanceId)) + otlpvalidation.OtlpMetricLabels(t.env.AgentStartCommand, t.env.InstanceId)) results = append(results, metricResult.TestResults...) // Traces @@ -69,24 +64,29 @@ func (t *OtlpCollectTestRunner) Validate() status.TestGroupResult { return status.TestGroupResult{Name: t.GetTestName(), TestResults: results} } -// validateTraces confirms the OTLP traces this run generated are queryable in -// X-Ray, filtered by the instance_id annotation for isolation. +// validateTraces queries X-Ray for traces from this run, isolated by instance_id. +// startedAt is captured at SetupAfterAgentRun so the window is stable across retries. func (t *OtlpCollectTestRunner) validateTraces() status.TestResult { annotations := map[string]interface{}{ "test_type": traceTestType, "instance_id": t.env.InstanceId, } - err := base.ValidateTraceSegments(time.Now().Add(-otlpRuntime), time.Now(), annotations, nil) - if err != nil { - return status.TestResult{Name: "OTLP_Traces", Status: status.FAILED, Reason: err} + var err error + for attempt := 0; attempt < 5; attempt++ { + if attempt > 0 { + time.Sleep(30 * time.Second) + } + err = base.ValidateTraceSegments(t.startedAt, time.Now(), annotations, nil) + if err == nil { + return status.TestResult{Name: "OTLP_Traces", Status: status.SUCCESSFUL} + } } - return status.TestResult{Name: "OTLP_Traces", Status: status.SUCCESSFUL} + return status.TestResult{Name: "OTLP_Traces", Status: status.FAILED, Reason: err} } -// validateLogs confirms the OTLP logs this run published are present in -// CloudWatch Logs for this instance. +// validateLogs checks that OTLP logs from this run are in CloudWatch Logs. func (t *OtlpCollectTestRunner) validateLogs() status.TestResult { - since := time.Now().Add(-otlpRuntime) + since := t.startedAt until := time.Now() if len(awsservice.GetLogStreams(otlpLogGroup)) == 0 { @@ -118,14 +118,22 @@ func (t *OtlpCollectTestRunner) GetMeasuredMetrics() []string { } func (t *OtlpCollectTestRunner) SetupAfterAgentRun() error { - go t.sendTestMetrics() + t.startedAt = time.Now() + go func() { + _ = common.SendOTLPMetrics(otlpEndpoint, t.env.InstanceId, 10*time.Second, otlpRuntime-30*time.Second) + }() go t.generateTraces() go t.sendTestLogs() return nil } -// generateTraces pushes OTLP traces to the agent's OTLP gRPC receiver +// generateTraces waits for gRPC port 4317, then streams traces to the agent. +// The wait is necessary because otlptracegrpc.New connects at construction time. func (t *OtlpCollectTestRunner) generateTraces() { + if err := common.WaitForTCPPort(otlpGRPCAddr, 2*time.Minute); err != nil { + log.Printf("generateTraces: gRPC port not ready, skipping: %v", err) + return + } generator := otlp.NewLoadGenerator(&base.TraceGeneratorConfig{ Interval: 10 * time.Second, Annotations: map[string]interface{}{ @@ -154,64 +162,21 @@ func (t *OtlpCollectTestRunner) sendTestLogs() { payload := buildOtlpLogsPayload(t.env.InstanceId) req, _ := http.NewRequest("POST", otlpEndpoint+"/v1/logs", bytes.NewReader(payload)) req.Header.Set("Content-Type", "application/json") - http.DefaultClient.Do(req) - } - } -} - -func (t *OtlpCollectTestRunner) sendTestMetrics() { - ticker := time.NewTicker(10 * time.Second) - defer ticker.Stop() - - timeout := time.After(otlpRuntime - 30*time.Second) - for { - select { - case <-timeout: - return - case <-ticker.C: - payload := buildOtlpMetricsPayload(t.env.InstanceId) - req, _ := http.NewRequest("POST", otlpEndpoint+"/v1/metrics", bytes.NewReader(payload)) - req.Header.Set("Content-Type", "application/json") http.DefaultClient.Do(req) //nolint:errcheck } } } -var startTime = time.Now().UnixNano() - -func buildOtlpMetricsPayload(instanceId string) []byte { - now := time.Now().UnixNano() - payload := fmt.Sprintf(`{ - "resourceMetrics": [{ - "resource": {"attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}, - "scopeMetrics": [{ - "metrics": [ - { - "name": "otlp_test_counter", - "sum": { - "dataPoints": [{"asInt": "1", "startTimeUnixNano": "%d", "timeUnixNano": "%d", "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}], - "isMonotonic": true, - "aggregationTemporality": 2 - } - }, - { - "name": "otlp_test_gauge", - "gauge": { - "dataPoints": [{"asDouble": 42.0, "timeUnixNano": "%d", "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}] - } - } - ] - }] - }] -}`, instanceId, startTime, now, instanceId, now, instanceId) - return []byte(payload) -} - func buildOtlpLogsPayload(instanceId string) []byte { now := time.Now().UnixNano() + // Routes to CW Logs via aws.log.group.name / aws.log.stream.name resource attributes. payload := fmt.Sprintf(`{ "resourceLogs": [{ - "resource": {"attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}, + "resource": {"attributes": [ + {"key": "aws.log.group.name", "value": {"stringValue": "%s"}}, + {"key": "aws.log.stream.name", "value": {"stringValue": "%s"}}, + {"key": "InstanceId", "value": {"stringValue": "%s"}} + ]}, "scopeLogs": [{ "logRecords": [{ "timeUnixNano": "%d", @@ -221,7 +186,7 @@ func buildOtlpLogsPayload(instanceId string) []byte { }] }] }] -}`, instanceId, now, instanceId) +}`, otlpLogGroup, instanceId, instanceId, now, instanceId) return []byte(payload) } diff --git a/test/otel_collect/linux/prometheus/prometheus_test.go b/test/otel_collect/linux/prometheus/prometheus_test.go index 04ecd271..093e81f8 100644 --- a/test/otel_collect/linux/prometheus/prometheus_test.go +++ b/test/otel_collect/linux/prometheus/prometheus_test.go @@ -8,7 +8,6 @@ package prometheus import ( _ "embed" "fmt" - "os" "testing" "time" @@ -62,7 +61,7 @@ func (t *PrometheusOtelTestRunner) SetupBeforeAgentRun() error { return err } - // Write prometheus scrape config + // Write the Prometheus scrape config where the agent config's config_path points. commands := []string{ fmt.Sprintf("cat <<'EOF' | sudo tee /opt/aws/prometheus.yml\n%s\nEOF", prometheusScrapeConfig), } @@ -70,19 +69,13 @@ func (t *PrometheusOtelTestRunner) SetupBeforeAgentRun() error { return err } - // Serve fake metrics on port 9100 (same pattern as test/emf_prometheus) - if err := os.WriteFile("/tmp/metrics", []byte(prometheusMetrics), os.ModePerm); err != nil { - return fmt.Errorf("unable to write /tmp/metrics: %w", err) + // Start the cross-platform fake Prometheus exporter on port 9100. + stop, err := common.StartPrometheusFakeServer(9100, prometheusMetrics) + if err != nil { + return fmt.Errorf("failed to start fake prometheus exporter: %w", err) } - commands = []string{ - "sudo python3 -m http.server 9100 --directory /tmp &> /dev/null &", - } - if err := common.RunCommands(commands); err != nil { - return err - } - t.RegisterCleanup(func() error { - return common.RunCommands([]string{"sudo pkill -f 'python3 -m http.server 9100' || true"}) - }) + t.RegisterCleanup(func() error { stop(); return nil }) + time.Sleep(2 * time.Second) return nil } diff --git a/test/otel_collect/otlpvalidation/labels.go b/test/otel_collect/otlpvalidation/labels.go new file mode 100644 index 00000000..059c7f59 --- /dev/null +++ b/test/otel_collect/otlpvalidation/labels.go @@ -0,0 +1,44 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otlpvalidation + +import ( + "os" + "strings" +) + +// onPremiseMarker identifies the on-prem agent start command (-m onPremise). +const onPremiseMarker = "onPremise" + +// ResourceHostIDLabels returns the label filter for host_metrics/prometheus tests. +// EC2 filters on host.id; on-prem returns nil (no IMDS, validate presence only). +func ResourceHostIDLabels(agentStartCommand, instanceID string) map[string]string { + if strings.Contains(agentStartCommand, onPremiseMarker) { + return nil + } + return map[string]string{"@resource.host.id": instanceID} +} + +// OtlpMetricLabels returns the label filter for the OTLP test. +// EC2 uses host.id; on-prem uses the injected InstanceId attribute. +func OtlpMetricLabels(agentStartCommand, instanceID string) map[string]string { + if strings.Contains(agentStartCommand, onPremiseMarker) { + return map[string]string{"InstanceId": instanceID} + } + return map[string]string{"@resource.host.id": instanceID} +} + +// getRegion returns the region, falling back to AWS env vars then us-west-2. +func getRegion(region string) string { + if region != "" { + return region + } + if r := os.Getenv("AWS_REGION"); r != "" { + return r + } + if r := os.Getenv("AWS_DEFAULT_REGION"); r != "" { + return r + } + return "us-west-2" +} diff --git a/test/otel_collect/otlpvalidation/validate.go b/test/otel_collect/otlpvalidation/validate.go index 5835bcec..90cfb4fa 100644 --- a/test/otel_collect/otlpvalidation/validate.go +++ b/test/otel_collect/otlpvalidation/validate.go @@ -9,49 +9,17 @@ import ( "context" "fmt" "log" - "os" - "strings" "time" "github.com/aws/amazon-cloudwatch-agent-test/test/status" "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" ) -// onPremiseMarker identifies the on-prem agent start command (-m onPremise). -const onPremiseMarker = "onPremise" - -// ResourceHostIDLabels returns the metric label filter keyed on the entity -// host.id (the EC2 instance id) for EC2 mode. -// -// In on-prem mode (agent started with "-m onPremise") the agent does not read -// EC2 IMDS, so it does not set @resource.host.id to the instance id. Filtering -// on it there would never match, so we return nil and validate metric presence -// only. Detection is based on the agent start command from the test matrix. -func ResourceHostIDLabels(agentStartCommand, instanceID string) map[string]string { - if strings.Contains(agentStartCommand, onPremiseMarker) { - return nil - } - return map[string]string{"@resource.host.id": instanceID} -} - const ( defaultMaxRetries = 3 defaultRetryInterval = 30 * time.Second ) -func getRegion(region string) string { - if region != "" { - return region - } - if r := os.Getenv("AWS_REGION"); r != "" { - return r - } - if r := os.Getenv("AWS_DEFAULT_REGION"); r != "" { - return r - } - return "us-west-2" -} - func ValidateOtlpMetrics(testName string, region string, metrics []string) status.TestGroupResult { return ValidateOtlpMetricsWithLabels(testName, region, metrics, nil) } diff --git a/test/otel_collect/windows/host_metrics/agent_config.json b/test/otel_collect/windows/host_metrics/agent_config.json index 13b50217..dbb30733 100644 --- a/test/otel_collect/windows/host_metrics/agent_config.json +++ b/test/otel_collect/windows/host_metrics/agent_config.json @@ -1,13 +1,5 @@ { - "agent": { - "region": "us-west-2" - }, - "opentelemetry": { - "cluster_name": "integ-test-cluster", - "collect": { - "host_metrics": { - "collection_interval": 30 - } + "agent": { + "debug": true } - } } diff --git a/test/otel_collect/windows/host_metrics/host_metrics_unix.go b/test/otel_collect/windows/host_metrics/host_metrics_unix.go new file mode 100644 index 00000000..657a964b --- /dev/null +++ b/test/otel_collect/windows/host_metrics/host_metrics_unix.go @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package host_metrics + +import "errors" + +// Validate is only implemented on Windows; this stub lets the validator build on other platforms. +func Validate() error { + return errors.New("otel_collect windows host_metrics validation is only supported on Windows") +} diff --git a/test/otel_collect/windows/host_metrics/host_metrics_windows.go b/test/otel_collect/windows/host_metrics/host_metrics_windows.go new file mode 100644 index 00000000..ec9abd0e --- /dev/null +++ b/test/otel_collect/windows/host_metrics/host_metrics_windows.go @@ -0,0 +1,52 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build windows + +package host_metrics + +import ( + "context" + _ "embed" + "fmt" + "os" + "time" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/otlpvalidation" + "github.com/aws/amazon-cloudwatch-agent-test/util/common" + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +//go:embed resources/config.json +var testConfigJSON string + +const ( + tmpConfigPath = "C:\\Users\\Administrator\\AppData\\Local\\Temp\\config.json" + runtime = 3 * time.Minute +) + +func Validate() error { + env := environment.GetEnvironmentMetaData() + + if err := os.WriteFile(tmpConfigPath, []byte(testConfigJSON), 0644); err != nil { + return fmt.Errorf("could not write config: %w", err) + } + if err := common.CopyFile(tmpConfigPath, common.ConfigOutputPath); err != nil { + return fmt.Errorf("could not copy config: %w", err) + } + if err := common.StartAgent(common.ConfigOutputPath, true, false); err != nil { + return fmt.Errorf("could not start agent: %w", err) + } + time.Sleep(runtime) + _ = common.StopAgent() + + return otelmetrics.AssertMetricsPresent( + context.Background(), + env.Region, + []string{"system.cpu.utilization", "system.memory.utilization", "system.network.io", "system.disk.operations"}, + otlpvalidation.ResourceHostIDLabels(env.AgentStartCommand, env.InstanceId), + 3, + 30*time.Second, + ) +} diff --git a/test/otel_collect/windows/host_metrics/parameters.yml b/test/otel_collect/windows/host_metrics/parameters.yml index 85eaae5f..f24e66cb 100644 --- a/test/otel_collect/windows/host_metrics/parameters.yml +++ b/test/otel_collect/windows/host_metrics/parameters.yml @@ -1,23 +1,14 @@ -# Windows Host Metrics OTLP integration test -# Tests that OTel host_metrics collects system metrics on Windows -# and publishes to the OTLP monitoring endpoint with cluster_name +# FILE ONLY EXISTS SO VALIDATOR WILL RUN THE GO TEST +# SEE validator/main.go receivers: [] - -test_case: "win_otlp_host_metrics" +test_case: "win_otel_host_metrics" validate_type: "feature" data_type: "metrics" - -# Number of seconds the agent should run and collect metrics (3 minutes) -agent_collection_period: 180 - -cloudwatch_agent_config: "agent_config.json" - -# Validate host metrics are published via OTLP -metric_namespace: "CWAgent/System" +number_monitored_logs: 1 +values_per_minute: "2" +agent_collection_period: 60 +cloudwatch_agent_config: "" +metric_namespace: "CloudWatchAgentWinOtel" metric_validation: - - metric_name: "cpu_time" - metric_sample_count: 1 - - - metric_name: "memory_usage" - metric_sample_count: 1 +log_validation: diff --git a/test/otel_collect/windows/host_metrics/resources/config.json b/test/otel_collect/windows/host_metrics/resources/config.json new file mode 100644 index 00000000..2f2d0212 --- /dev/null +++ b/test/otel_collect/windows/host_metrics/resources/config.json @@ -0,0 +1,13 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "cluster_name": "cwagent-windows-integ-test", + "collect": { + "host_metrics": { + "collection_interval": 30 + } + } + } +} diff --git a/test/otel_collect/windows/otlp/agent_config.json b/test/otel_collect/windows/otlp/agent_config.json new file mode 100644 index 00000000..dbb30733 --- /dev/null +++ b/test/otel_collect/windows/otlp/agent_config.json @@ -0,0 +1,5 @@ +{ + "agent": { + "debug": true + } +} diff --git a/test/otel_collect/windows/otlp/otlp_windows.go b/test/otel_collect/windows/otlp/otlp_windows.go index 6793868b..613d6012 100644 --- a/test/otel_collect/windows/otlp/otlp_windows.go +++ b/test/otel_collect/windows/otlp/otlp_windows.go @@ -12,7 +12,8 @@ import ( "os" "time" - "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/otlpvalidation" "github.com/aws/amazon-cloudwatch-agent-test/util/common" "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" ) @@ -26,13 +27,11 @@ const ( sendInterval = 10 * time.Second ) -// Validate runs the Windows OTLP test: start the agent with the V2 opentelemetry OTLP config, -// push OTLP metrics to the agent, then validate they reach CloudWatch via the PromQL endpoint. func Validate() error { - instanceID := awsservice.GetInstanceId() + env := environment.GetEnvironmentMetaData() if err := os.WriteFile(tmpConfigPath, []byte(testConfigJSON), 0644); err != nil { - return fmt.Errorf("could not write config file: %w", err) + return fmt.Errorf("could not write config: %w", err) } if err := common.CopyFile(tmpConfigPath, common.ConfigOutputPath); err != nil { return fmt.Errorf("could not copy config: %w", err) @@ -40,21 +39,20 @@ func Validate() error { if err := common.StartAgent(common.ConfigOutputPath, true, false); err != nil { return fmt.Errorf("could not start agent: %w", err) } - time.Sleep(5 * time.Second) - - // Push OTLP metrics to the agent's OTLP/HTTP receiver for the collection period. - if err := common.SendOTLPMetrics(common.DefaultOTLPHTTPEndpoint, instanceID, sendInterval, otlpRuntime); err != nil { + // Wait for the OTLP HTTP receiver before sending metrics. + if err := common.WaitForOTLPEndpoint(common.DefaultOTLPHTTPEndpoint, 2*time.Minute); err != nil { + return fmt.Errorf("OTLP endpoint not ready: %w", err) + } + if err := common.SendOTLPMetrics(common.DefaultOTLPHTTPEndpoint, env.InstanceId, sendInterval, otlpRuntime); err != nil { return fmt.Errorf("failed to send OTLP metrics: %w", err) } - _ = common.StopAgent() - // Validate the pushed metrics landed in CloudWatch, isolated by the instance's host.id. return otelmetrics.AssertMetricsPresent( context.Background(), - "", // region resolved from AWS_REGION env + env.Region, []string{"otlp_test_counter", "otlp_test_gauge"}, - map[string]string{"@resource.host.id": instanceID}, + otlpvalidation.OtlpMetricLabels(env.AgentStartCommand, env.InstanceId), 3, 30*time.Second, ) diff --git a/test/otel_collect/windows/otlp/parameters.yml b/test/otel_collect/windows/otlp/parameters.yml new file mode 100644 index 00000000..1ddb28af --- /dev/null +++ b/test/otel_collect/windows/otlp/parameters.yml @@ -0,0 +1,17 @@ +# FILE ONLY EXISTS SO VALIDATOR WILL RUN THE GO TEST +# SEE validator/main.go + +# Receivers that agent needs to test +receivers: [] + +# Test case name +test_case: "win_otlp" +validate_type: "feature" +data_type: "metrics" +number_monitored_logs: 1 +values_per_minute: "2" +agent_collection_period: 60 +cloudwatch_agent_config: "" +metric_namespace: "CloudWatchAgentWinOtel" +metric_validation: +log_validation: diff --git a/test/otel_collect/windows/prometheus/agent_config.json b/test/otel_collect/windows/prometheus/agent_config.json index 1f2bf690..dbb30733 100644 --- a/test/otel_collect/windows/prometheus/agent_config.json +++ b/test/otel_collect/windows/prometheus/agent_config.json @@ -1,13 +1,5 @@ { - "agent": { - "region": "us-west-2" - }, - "opentelemetry": { - "cluster_name": "integ-test-cluster", - "collect": { - "prometheus": { - "config_path": "C:\\prometheus.yml" - } + "agent": { + "debug": true } - } } diff --git a/test/otel_collect/windows/prometheus/parameters.yml b/test/otel_collect/windows/prometheus/parameters.yml index 42ddc1a8..77b736be 100644 --- a/test/otel_collect/windows/prometheus/parameters.yml +++ b/test/otel_collect/windows/prometheus/parameters.yml @@ -1,45 +1,14 @@ -# Windows Prometheus OTLP integration test -# Tests that OTel Prometheus receiver scrapes metrics on Windows -# and publishes to the OTLP monitoring endpoint +# FILE ONLY EXISTS SO VALIDATOR WILL RUN THE GO TEST +# SEE validator/main.go receivers: [] - -test_case: "win_otlp_prometheus" +test_case: "win_otel_prometheus" validate_type: "feature" data_type: "metrics" - -# Number of seconds the agent should run and collect metrics (3 minutes) -agent_collection_period: 180 - -cloudwatch_agent_config: "agent_config.json" - -# Preparation script: writes prometheus scrape config and starts a fake metrics server -# The validator will run this PowerShell before starting the agent -preparation_cmd: | - Set-Content -Path "C:\prometheus.yml" -Value (Get-Content "C:\prometheus_scrape_config.yaml") - $listener = [System.Net.HttpListener]::new() - $listener.Prefixes.Add("http://+:9100/") - $listener.Start() - $job = Start-Job -ScriptBlock { - param($metrics) - $l = [System.Net.HttpListener]::new() - $l.Prefixes.Add("http://+:9100/") - $l.Start() - while ($l.IsListening) { - $ctx = $l.GetContext() - $response = $ctx.Response - $buffer = [System.Text.Encoding]::UTF8.GetBytes($metrics) - $response.ContentLength64 = $buffer.Length - $response.OutputStream.Write($buffer, 0, $buffer.Length) - $response.Close() - } - } -ArgumentList (Get-Content "C:\prometheus_metrics" -Raw) - -# Validate prometheus metrics are published via OTLP -metric_namespace: "CWAgent" +number_monitored_logs: 1 +values_per_minute: "2" +agent_collection_period: 60 +cloudwatch_agent_config: "" +metric_namespace: "CloudWatchAgentWinOtel" metric_validation: - - metric_name: "node_cpu_seconds_total" - metric_sample_count: 1 - - - metric_name: "node_memory_MemAvailable_bytes" - metric_sample_count: 1 +log_validation: diff --git a/test/otel_collect/windows/prometheus/prometheus_metrics b/test/otel_collect/windows/prometheus/prometheus_metrics deleted file mode 100644 index 1e158f2a..00000000 --- a/test/otel_collect/windows/prometheus/prometheus_metrics +++ /dev/null @@ -1,13 +0,0 @@ -# HELP node_cpu_seconds_total Seconds the CPUs spent in each mode. -# TYPE node_cpu_seconds_total counter -node_cpu_seconds_total{cpu="0",mode="idle"} 12345.67 -node_cpu_seconds_total{cpu="0",mode="system"} 234.56 -# HELP node_memory_MemAvailable_bytes Memory information field MemAvailable_bytes. -# TYPE node_memory_MemAvailable_bytes gauge -node_memory_MemAvailable_bytes 4123456789 -# HELP node_filesystem_avail_bytes Filesystem space available to non-root users in bytes. -# TYPE node_filesystem_avail_bytes gauge -node_filesystem_avail_bytes{device="C:",fstype="ntfs",mountpoint="C:"} 50000000000 -# HELP node_network_receive_bytes_total Network device statistic receive_bytes. -# TYPE node_network_receive_bytes_total counter -node_network_receive_bytes_total{device="Ethernet"} 987654321 diff --git a/test/otel_collect/windows/prometheus/prometheus_unix.go b/test/otel_collect/windows/prometheus/prometheus_unix.go new file mode 100644 index 00000000..e9d8d7ae --- /dev/null +++ b/test/otel_collect/windows/prometheus/prometheus_unix.go @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package prometheus + +import "errors" + +// Validate is only implemented on Windows; this stub lets the validator build on other platforms. +func Validate() error { + return errors.New("otel_collect windows prometheus validation is only supported on Windows") +} diff --git a/test/otel_collect/windows/prometheus/prometheus_windows.go b/test/otel_collect/windows/prometheus/prometheus_windows.go new file mode 100644 index 00000000..b57b2177 --- /dev/null +++ b/test/otel_collect/windows/prometheus/prometheus_windows.go @@ -0,0 +1,69 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build windows + +package prometheus + +import ( + "context" + _ "embed" + "fmt" + "os" + "time" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/otlpvalidation" + "github.com/aws/amazon-cloudwatch-agent-test/util/common" + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +//go:embed resources/config.json +var testConfigJSON string + +//go:embed resources/prometheus_scrape.yaml +var scrapeConfig string + +//go:embed resources/prometheus_metrics +var prometheusMetrics string + +const ( + tmpConfigPath = "C:\\Users\\Administrator\\AppData\\Local\\Temp\\config.json" + scrapePath = "C:\\prometheus_scrape.yaml" + exporterPort = 8101 + runtime = 3 * time.Minute +) + +func Validate() error { + env := environment.GetEnvironmentMetaData() + + stop, err := common.StartPrometheusFakeServer(exporterPort, prometheusMetrics) + if err != nil { + return fmt.Errorf("could not start fake prometheus exporter: %w", err) + } + defer stop() + + if err := os.WriteFile(scrapePath, []byte(scrapeConfig), 0644); err != nil { + return fmt.Errorf("could not write scrape config: %w", err) + } + if err := os.WriteFile(tmpConfigPath, []byte(testConfigJSON), 0644); err != nil { + return fmt.Errorf("could not write agent config: %w", err) + } + if err := common.CopyFile(tmpConfigPath, common.ConfigOutputPath); err != nil { + return fmt.Errorf("could not copy config: %w", err) + } + if err := common.StartAgent(common.ConfigOutputPath, true, false); err != nil { + return fmt.Errorf("could not start agent: %w", err) + } + time.Sleep(runtime) + _ = common.StopAgent() + + return otelmetrics.AssertMetricsPresent( + context.Background(), + env.Region, + []string{"node_cpu_seconds_total", "node_memory_MemAvailable_bytes"}, + otlpvalidation.ResourceHostIDLabels(env.AgentStartCommand, env.InstanceId), + 3, + 30*time.Second, + ) +} diff --git a/test/otel_collect/windows/prometheus/resources/config.json b/test/otel_collect/windows/prometheus/resources/config.json new file mode 100644 index 00000000..8bc6d04b --- /dev/null +++ b/test/otel_collect/windows/prometheus/resources/config.json @@ -0,0 +1,13 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "cluster_name": "cwagent-windows-integ-test", + "collect": { + "prometheus": { + "config_path": "C:\\prometheus_scrape.yaml" + } + } + } +} diff --git a/test/otel_collect/windows/prometheus/resources/prometheus_metrics b/test/otel_collect/windows/prometheus/resources/prometheus_metrics new file mode 100644 index 00000000..9ce5127f --- /dev/null +++ b/test/otel_collect/windows/prometheus/resources/prometheus_metrics @@ -0,0 +1,7 @@ +# HELP node_cpu_seconds_total Seconds the CPUs spent in each mode. +# TYPE node_cpu_seconds_total counter +node_cpu_seconds_total{cpu="0",mode="idle"} 12345.67 +node_cpu_seconds_total{cpu="0",mode="system"} 234.56 +# HELP node_memory_MemAvailable_bytes Memory information field MemAvailable_bytes. +# TYPE node_memory_MemAvailable_bytes gauge +node_memory_MemAvailable_bytes 4123456789 diff --git a/test/otel_collect/windows/prometheus/prometheus_scrape_config.yaml b/test/otel_collect/windows/prometheus/resources/prometheus_scrape.yaml similarity index 82% rename from test/otel_collect/windows/prometheus/prometheus_scrape_config.yaml rename to test/otel_collect/windows/prometheus/resources/prometheus_scrape.yaml index 9d7a3578..efd6350e 100644 --- a/test/otel_collect/windows/prometheus/prometheus_scrape_config.yaml +++ b/test/otel_collect/windows/prometheus/resources/prometheus_scrape.yaml @@ -1,7 +1,8 @@ scrape_configs: - job_name: node + scrape_interval: 10s static_configs: - - targets: ['localhost:9100'] + - targets: ['localhost:8101'] relabel_configs: - source_labels: [job] regex: (.*) diff --git a/util/common/otlp.go b/util/common/otlp.go index fd9c96ab..402ae5c8 100644 --- a/util/common/otlp.go +++ b/util/common/otlp.go @@ -15,7 +15,40 @@ import ( // DefaultOTLPHTTPEndpoint is the agent's default OTLP/HTTP receiver endpoint. const DefaultOTLPHTTPEndpoint = "http://127.0.0.1:4318" -// SendOTLPMetrics pushes OTLP metrics to the agent's OTLP/HTTP receiver until the duration elapses (cross-platform). +// WaitForOTLPEndpoint polls the OTLP HTTP endpoint until it responds or timeout elapses. +// Use after StartAgent instead of a fixed sleep. +func WaitForOTLPEndpoint(endpoint string, timeout time.Duration) error { + if endpoint == "" { + endpoint = DefaultOTLPHTTPEndpoint + } + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + resp, err := http.Get(endpoint) //nolint:noctx + if err == nil { + resp.Body.Close() + return nil + } + time.Sleep(2 * time.Second) + } + return fmt.Errorf("OTLP endpoint %s not ready after %s", endpoint, timeout) +} + +// WaitForTCPPort dials addr until it accepts a connection or timeout elapses. +// Use before creating an otlptracegrpc exporter, which connects immediately. +func WaitForTCPPort(addr string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("tcp", addr, 2*time.Second) + if err == nil { + conn.Close() + return nil + } + time.Sleep(2 * time.Second) + } + return fmt.Errorf("TCP port %s not ready after %s", addr, timeout) +} + +// SendOTLPMetrics pushes OTLP metrics to the agent's HTTP receiver until duration elapses. func SendOTLPMetrics(endpoint, instanceID string, sendingInterval, duration time.Duration) error { if endpoint == "" { endpoint = DefaultOTLPHTTPEndpoint @@ -44,9 +77,10 @@ func SendOTLPMetrics(endpoint, instanceID string, sendingInterval, duration time } } -// buildOTLPMetricsPayload builds an OTLP/HTTP JSON metrics payload with an InstanceId attribute for isolation. +// buildOTLPMetricsPayload builds a delta counter + gauge payload tagged with instanceID. func buildOTLPMetricsPayload(instanceID string) []byte { now := time.Now().UnixNano() + start := now - int64(10*time.Second) return []byte(fmt.Sprintf(`{ "resourceMetrics": [{ "resource": {"attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}, @@ -55,9 +89,9 @@ func buildOTLPMetricsPayload(instanceID string) []byte { { "name": "otlp_test_counter", "sum": { - "dataPoints": [{"asInt": "1", "timeUnixNano": "%d", "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}], + "dataPoints": [{"asInt": "1", "startTimeUnixNano": "%d", "timeUnixNano": "%d", "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}], "isMonotonic": true, - "aggregationTemporality": 2 + "aggregationTemporality": 1 } }, { @@ -69,10 +103,10 @@ func buildOTLPMetricsPayload(instanceID string) []byte { ] }] }] -}`, instanceID, now, instanceID, now, instanceID)) +}`, instanceID, start, now, instanceID, now, instanceID)) } -// StartPrometheusFakeServer serves static Prometheus metrics on the port until stop() is called (cross-platform). +// StartPrometheusFakeServer serves static Prometheus metrics on the given port until stop() is called. func StartPrometheusFakeServer(port int, exposition string) (stop func(), err error) { mux := http.NewServeMux() mux.HandleFunc("/metrics", func(w http.ResponseWriter, _ *http.Request) { diff --git a/util/common/traces/otlp/generator.go b/util/common/traces/otlp/generator.go index 98e41eca..487f105b 100644 --- a/util/common/traces/otlp/generator.go +++ b/util/common/traces/otlp/generator.go @@ -125,7 +125,8 @@ func setupClient(ctx context.Context) (*sdktrace.TracerProvider, func(context.Co } func setupTraceProvider(ctx context.Context, res *resource.Resource) (*sdktrace.TracerProvider, error) { - exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithInsecure()) + // Use 127.0.0.1 (not "localhost", which can resolve to IPv6 [::1]) since the agent binds IPv4. + exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithInsecure(), otlptracegrpc.WithEndpoint("127.0.0.1:4317")) if err != nil { return nil, err } diff --git a/validator/main.go b/validator/main.go index 3a20f85c..339387f4 100644 --- a/validator/main.go +++ b/validator/main.go @@ -18,8 +18,10 @@ import ( "github.com/aws/amazon-cloudwatch-agent-test/test/log_state/logfile" "github.com/aws/amazon-cloudwatch-agent-test/test/log_state/windows_event_log" "github.com/aws/amazon-cloudwatch-agent-test/test/nvidia_gpu" - "github.com/aws/amazon-cloudwatch-agent-test/test/restart" + otelhostmetrics "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/windows/host_metrics" otelcollectotlp "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/windows/otlp" + otelprometheus "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/windows/prometheus" + "github.com/aws/amazon-cloudwatch-agent-test/test/restart" "github.com/aws/amazon-cloudwatch-agent-test/test/ssm_document" "github.com/aws/amazon-cloudwatch-agent-test/test/workload_discovery" "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" @@ -70,6 +72,10 @@ func main() { err = ssm_document.Validate() case "otlp": err = otelcollectotlp.Validate() + case "host_metrics": + err = otelhostmetrics.Validate() + case "prometheus": + err = otelprometheus.Validate() default: err = errors.New("unsupported test name") } From ce28a4d32733053075cbf832ee5a7c7bb1025427 Mon Sep 17 00:00:00 2001 From: Michael Commey Date: Tue, 21 Jul 2026 20:53:31 +0000 Subject: [PATCH 4/5] fix windows, ecs and on-prem tests --- generator/test_case_generator.go | 862 +++++++++--------- .../ecs/host_metrics/host_metrics_ecs_test.go | 13 +- .../ecs/host_metrics/resources/config.json | 3 +- test/otel_collect/linux/otlp/otlp_test.go | 123 +-- .../linux/prometheus/prometheus_test.go | 22 +- test/otel_collect/otlpvalidation/labels.go | 44 + test/otel_collect/otlpvalidation/validate.go | 32 - .../windows/host_metrics/agent_config.json | 12 +- .../windows/host_metrics/host_metrics_unix.go | 13 + .../host_metrics/host_metrics_windows.go | 52 ++ .../windows/host_metrics/parameters.yml | 27 +- .../host_metrics/resources/config.json | 13 + .../windows/otlp/agent_config.json | 5 + .../otel_collect/windows/otlp/otlp_windows.go | 24 +- test/otel_collect/windows/otlp/parameters.yml | 17 + .../windows/prometheus/agent_config.json | 12 +- .../windows/prometheus/parameters.yml | 49 +- .../windows/prometheus/prometheus_metrics | 13 - .../windows/prometheus/prometheus_unix.go | 13 + .../windows/prometheus/prometheus_windows.go | 69 ++ .../windows/prometheus/resources/config.json | 13 + .../prometheus/resources/prometheus_metrics | 7 + .../prometheus_scrape.yaml} | 3 +- util/common/otlp.go | 31 +- util/common/traces/otlp/generator.go | 3 +- validator/main.go | 8 +- 26 files changed, 776 insertions(+), 707 deletions(-) create mode 100644 test/otel_collect/otlpvalidation/labels.go create mode 100644 test/otel_collect/windows/host_metrics/host_metrics_unix.go create mode 100644 test/otel_collect/windows/host_metrics/host_metrics_windows.go create mode 100644 test/otel_collect/windows/host_metrics/resources/config.json create mode 100644 test/otel_collect/windows/otlp/agent_config.json create mode 100644 test/otel_collect/windows/otlp/parameters.yml delete mode 100644 test/otel_collect/windows/prometheus/prometheus_metrics create mode 100644 test/otel_collect/windows/prometheus/prometheus_unix.go create mode 100644 test/otel_collect/windows/prometheus/prometheus_windows.go create mode 100644 test/otel_collect/windows/prometheus/resources/config.json create mode 100644 test/otel_collect/windows/prometheus/resources/prometheus_metrics rename test/otel_collect/windows/prometheus/{prometheus_scrape_config.yaml => resources/prometheus_scrape.yaml} (82%) diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index 3f646854..5184824d 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -75,18 +75,18 @@ const ( // you can't have a const map in golang var testTypeToTestConfig = map[string][]testConfig{ - // "ec2_gpu": { - // {testDir: "./test/nvidia_gpu"}, - // }, - // "ec2_efa": { - // {testDir: "./test/efa_ec2", terraformDir: "terraform/ec2/efa"}, - // }, - // "ec2_linux_wd": { - // {testDir: "./test/workload_discovery"}, - // }, - // "ec2_linux_wd_nvidia": { - // {testDir: "./test/workload_discovery"}, - // }, + "ec2_gpu": { + {testDir: "./test/nvidia_gpu"}, + }, + "ec2_efa": { + {testDir: "./test/efa_ec2", terraformDir: "terraform/ec2/efa"}, + }, + "ec2_linux_wd": { + {testDir: "./test/workload_discovery"}, + }, + "ec2_linux_wd_nvidia": { + {testDir: "./test/workload_discovery"}, + }, "ec2_linux_onprem": { {testDir: "./test/cloudwatchlogs"}, {testDir: "./test/otel_collect/linux/otlp"}, @@ -94,442 +94,442 @@ var testTypeToTestConfig = map[string][]testConfig{ {testDir: "./test/otel_collect/linux/prometheus"}, }, testTypeKeyEc2Linux: { - // {testDir: "./test/ca_bundle"}, - // {testDir: "./test/cloudwatchlogs"}, - // { - // testDir: "./test/log_state/logfile", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - // }, - // { - // testDir: "./test/log_state/journald", - // targets: map[string]map[string]struct{}{"os": {"al2": {}, "al2023": {}}}, - // }, - // { - // testDir: "./test/feature/linux/journald_logs", - // targets: map[string]map[string]struct{}{"os": {"al2": {}, "al2023": {}}}, - // }, - // { - // testDir: "./test/metrics_number_dimension", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - // }, - // { - // testDir: "./test/emf_concurrent", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - // maxAttempts: 1, - // }, - // { - // testDir: "./test/emf_prometheus", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - // maxAttempts: 2, - // }, - // {testDir: "./test/entity_metrics_benchmark"}, - // { - // testDir: "./test/metric_value_benchmark", - // instanceTypeByArch: map[string]string{ - // "amd64": "i3en.large", - // "arm64": "i4g.large", - // }, - // }, - // {testDir: "./test/run_as_user"}, - // {testDir: "./test/collection_interval"}, - // {testDir: "./test/metric_dimension"}, - // {testDir: "./test/restart"}, - // {testDir: "./test/xray"}, - // {testDir: "./test/otlp"}, - // { - // testDir: "./test/otel_collect/database_insights", - // excludedOs: map[string]struct{}{"ol8": {}, "ubuntu-25": {}}, - // }, + {testDir: "./test/ca_bundle"}, + {testDir: "./test/cloudwatchlogs"}, + { + testDir: "./test/log_state/logfile", + targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + }, + { + testDir: "./test/log_state/journald", + targets: map[string]map[string]struct{}{"os": {"al2": {}, "al2023": {}}}, + }, + { + testDir: "./test/feature/linux/journald_logs", + targets: map[string]map[string]struct{}{"os": {"al2": {}, "al2023": {}}}, + }, + { + testDir: "./test/metrics_number_dimension", + targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + }, + { + testDir: "./test/emf_concurrent", + targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + maxAttempts: 1, + }, + { + testDir: "./test/emf_prometheus", + targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + maxAttempts: 2, + }, + {testDir: "./test/entity_metrics_benchmark"}, + { + testDir: "./test/metric_value_benchmark", + instanceTypeByArch: map[string]string{ + "amd64": "i3en.large", + "arm64": "i4g.large", + }, + }, + {testDir: "./test/run_as_user"}, + {testDir: "./test/collection_interval"}, + {testDir: "./test/metric_dimension"}, + {testDir: "./test/restart"}, + {testDir: "./test/xray"}, + {testDir: "./test/otlp"}, + { + testDir: "./test/otel_collect/database_insights", + excludedOs: map[string]struct{}{"ol8": {}, "ubuntu-25": {}}, + }, {testDir: "./test/otel_collect/linux/host_metrics"}, {testDir: "./test/otel_collect/linux/otlp"}, { testDir: "./test/otel_collect/linux/prometheus", excludedOs: map[string]struct{}{"rhel8": {}, "ol8": {}, "sles-15": {}}, }, - // { - // testDir: "./test/acceptance", - // targets: map[string]map[string]struct{}{"os": {"ubuntu-20.04": {}}}, - // }, - // // skipping FIPS test as the test cannot be verified - // // neither ssh nor SSM works after a reboot once FIPS is enabled - // //{ - // // testDir: "./test/fips", - // // targets: map[string]map[string]struct{}{"os": {"rhel8": {}}}, - // //}, - // { - // testDir: "./test/lvm", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - // }, - // { - // testDir: "./test/proxy", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - // }, - // { - // testDir: "./test/ssl_cert", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - // }, - // { - // testDir: "./test/userdata", - // terraformDir: "terraform/ec2/userdata", - // targets: map[string]map[string]struct{}{"os": {"ol9": {}}}, - // }, - // { - // testDir: "./test/credentials_file", - // terraformDir: "terraform/ec2/creds", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - // }, - // { - // testDir: "./test/amp", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - // }, - // { - // testDir: "./test/histograms", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - // }, - // { - // testDir: "./test/agent_otel_merging", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - // }, - // { - // testDir: "./test/assume_role", - // terraformDir: "terraform/ec2/assume_role", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - // }, - // { - // testDir: "./test/credential_chain", - // terraformDir: "terraform/ec2/assume_role", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - // }, - // { - // testDir: "./test/detailed_metrics", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - // }, - // { - // testDir: "./test/otlp", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - // }, - // { - // testDir: "./test/dualstack_endpoint", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - // }, - // {testDir: "./test/ssm_document"}, - // { - // testDir: "./test/system_metrics/enabled", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - // wip: true, - // }, - // { - // testDir: "./test/system_metrics/disabled", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - // }, - // { - // testDir: "./test/app_signals_service_events", - // targets: map[string]map[string]struct{}{"os": {"al2023": {}}, "arc": {"amd64": {}}}, - // }, + { + testDir: "./test/acceptance", + targets: map[string]map[string]struct{}{"os": {"ubuntu-20.04": {}}}, + }, + // skipping FIPS test as the test cannot be verified + // neither ssh nor SSM works after a reboot once FIPS is enabled + //{ + // testDir: "./test/fips", + // targets: map[string]map[string]struct{}{"os": {"rhel8": {}}}, + //}, + { + testDir: "./test/lvm", + targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + }, + { + testDir: "./test/proxy", + targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + }, + { + testDir: "./test/ssl_cert", + targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + }, + { + testDir: "./test/userdata", + terraformDir: "terraform/ec2/userdata", + targets: map[string]map[string]struct{}{"os": {"ol9": {}}}, + }, + { + testDir: "./test/credentials_file", + terraformDir: "terraform/ec2/creds", + targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + }, + { + testDir: "./test/amp", + targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + }, + { + testDir: "./test/histograms", + targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + }, + { + testDir: "./test/agent_otel_merging", + targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + }, + { + testDir: "./test/assume_role", + terraformDir: "terraform/ec2/assume_role", + targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + }, + { + testDir: "./test/credential_chain", + terraformDir: "terraform/ec2/assume_role", + targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + }, + { + testDir: "./test/detailed_metrics", + targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + }, + { + testDir: "./test/otlp", + targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + }, + { + testDir: "./test/dualstack_endpoint", + targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + }, + {testDir: "./test/ssm_document"}, + { + testDir: "./test/system_metrics/enabled", + targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + wip: true, + }, + { + testDir: "./test/system_metrics/disabled", + targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + }, + { + testDir: "./test/app_signals_service_events", + targets: map[string]map[string]struct{}{"os": {"al2023": {}}, "arc": {"amd64": {}}}, + }, + }, + testTypeKeyEc2SELinux: { + {testDir: "./test/ca_bundle"}, + {testDir: "./test/cloudwatchlogs"}, + { + testDir: "./test/log_state/journald", + targets: map[string]map[string]struct{}{"os": {"al2": {}, "al2023": {}}}, + }, + { + testDir: "./test/feature/linux/journald_logs", + targets: map[string]map[string]struct{}{"os": {"al2": {}, "al2023": {}}}, + }, + { + testDir: "./test/metrics_number_dimension", + targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + }, + { + testDir: "./test/emf_concurrent", + targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + maxAttempts: 1, + }, + { + testDir: "./test/emf_prometheus", + maxAttempts: 2, + }, + //{testDir: "./test/metric_value_benchmark"}, // Skipping test until it is fixed! + {testDir: "./test/run_as_user"}, + {testDir: "./test/collection_interval"}, + {testDir: "./test/metric_dimension"}, + {testDir: "./test/restart"}, + {testDir: "./test/xray"}, + {testDir: "./test/selinux_negative_test"}, + //{testDir: "./test/otlp"}, // Skipping test until it is fixed! + { + testDir: "./test/lvm", + targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + }, + { + testDir: "./test/proxy", + targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + }, + { + testDir: "./test/ssl_cert", + targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + }, + { + testDir: "./test/credentials_file", + terraformDir: "terraform/ec2/creds", + targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + }, + { + testDir: "./test/amp", + targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + }, + { + testDir: "./test/agent_otel_merging", + targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + }, + { + testDir: "./test/assume_role", + terraformDir: "terraform/ec2/assume_role", + targets: map[string]map[string]struct{}{"os": {"al2": {}}}, + }, + { + testDir: "./test/dualstack_endpoint", + targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, + }, }, - // testTypeKeyEc2SELinux: { - // {testDir: "./test/ca_bundle"}, - // {testDir: "./test/cloudwatchlogs"}, - // { - // testDir: "./test/log_state/journald", - // targets: map[string]map[string]struct{}{"os": {"al2": {}, "al2023": {}}}, - // }, - // { - // testDir: "./test/feature/linux/journald_logs", - // targets: map[string]map[string]struct{}{"os": {"al2": {}, "al2023": {}}}, - // }, - // { - // testDir: "./test/metrics_number_dimension", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - // }, - // { - // testDir: "./test/emf_concurrent", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - // maxAttempts: 1, - // }, - // { - // testDir: "./test/emf_prometheus", - // maxAttempts: 2, - // }, - // //{testDir: "./test/metric_value_benchmark"}, // Skipping test until it is fixed! - // {testDir: "./test/run_as_user"}, - // {testDir: "./test/collection_interval"}, - // {testDir: "./test/metric_dimension"}, - // {testDir: "./test/restart"}, - // {testDir: "./test/xray"}, - // {testDir: "./test/selinux_negative_test"}, - // //{testDir: "./test/otlp"}, // Skipping test until it is fixed! - // { - // testDir: "./test/lvm", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - // }, - // { - // testDir: "./test/proxy", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - // }, - // { - // testDir: "./test/ssl_cert", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - // }, - // { - // testDir: "./test/credentials_file", - // terraformDir: "terraform/ec2/creds", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - // }, - // { - // testDir: "./test/amp", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - // }, - // { - // testDir: "./test/agent_otel_merging", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - // }, - // { - // testDir: "./test/assume_role", - // terraformDir: "terraform/ec2/assume_role", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}}, - // }, - // { - // testDir: "./test/dualstack_endpoint", - // targets: map[string]map[string]struct{}{"os": {"al2": {}}, "arc": {"amd64": {}}}, - // }, - // }, /* You can only place 1 mac instance on a dedicate host a single time. Therefore, limit down the scope for testing in Mac since EC2 can be done with Linux and Mac under the hood share similar plugins with Linux */ - // "ec2_mac": { - // {testDir: "../../../test/feature/mac"}, - // }, - // "ec2_windows_wd": { - // {testDir: "../../../test/workload_discovery"}, - // }, - // "ec2_windows_wd_nvidia": { - // {testDir: "../../../test/workload_discovery"}, - // }, + "ec2_mac": { + {testDir: "../../../test/feature/mac"}, + }, + "ec2_windows_wd": { + {testDir: "../../../test/workload_discovery"}, + }, + "ec2_windows_wd_nvidia": { + {testDir: "../../../test/workload_discovery"}, + }, "ec2_windows": { - // {testDir: "../../../test/feature/windows"}, - // {testDir: "../../../test/restart"}, - // {testDir: "../../../test/acceptance"}, - // {testDir: "../../../test/feature/windows/event_logs"}, - // {testDir: "../../../test/feature/windows/eventid_logs"}, - // {testDir: "../../../test/feature/windows/event_regex_logs"}, - // {testDir: "../../../test/log_state/logfile"}, - // {testDir: "../../../test/log_state/windows_event_log"}, - // { - // testDir: "../../../test/feature/windows/custom_start/userdata", - // targets: map[string]map[string]struct{}{"os": {"win-2019": {}}}, - // }, - // { - // testDir: "../../../test/feature/windows/custom_start/ssm_start", - // targets: map[string]map[string]struct{}{"os": {"win-2019": {}}}, - // }, - // {testDir: "../../../test/ssm_document"}, + {testDir: "../../../test/feature/windows"}, + {testDir: "../../../test/restart"}, + {testDir: "../../../test/acceptance"}, + {testDir: "../../../test/feature/windows/event_logs"}, + {testDir: "../../../test/feature/windows/eventid_logs"}, + {testDir: "../../../test/feature/windows/event_regex_logs"}, + {testDir: "../../../test/log_state/logfile"}, + {testDir: "../../../test/log_state/windows_event_log"}, + { + testDir: "../../../test/feature/windows/custom_start/userdata", + targets: map[string]map[string]struct{}{"os": {"win-2019": {}}}, + }, + { + testDir: "../../../test/feature/windows/custom_start/ssm_start", + targets: map[string]map[string]struct{}{"os": {"win-2019": {}}}, + }, + {testDir: "../../../test/ssm_document"}, {testDir: "../../../test/otel_collect/windows/otlp"}, {testDir: "../../../test/otel_collect/windows/host_metrics"}, {testDir: "../../../test/otel_collect/windows/prometheus"}, // assume role test doesn't add much value, and it already being tested with linux //{testDir: "../../../test/assume_role"}, }, - // "ec2_performance": { - // {testDir: "../../test/performance/emf"}, - // {testDir: "../../test/performance/logs"}, - // {testDir: "../../test/performance/system"}, - // {testDir: "../../test/performance/statsd"}, - // {testDir: "../../test/performance/collectd"}, - // {testDir: "../../test/performance/trace/xray", runMockServer: true}, - // }, - // "ec2_windows_performance": { - // {testDir: "../../test/performance/windows/logs"}, - // {testDir: "../../test/performance/windows/system"}, - // }, - // "ec2_stress": { - // {testDir: "../../test/stress/emf"}, - // {testDir: "../../test/stress/logs"}, - // {testDir: "../../test/stress/system"}, - // {testDir: "../../test/stress/statsd"}, - // {testDir: "../../test/stress/collectd"}, - // {testDir: "../../test/stress/prometheus"}, - // }, - // "ec2_windows_stress": { - // {testDir: "../../test/stress/windows/logs"}, - // {testDir: "../../test/stress/windows/system"}, - // }, - // "ecs_fargate": { - // {testDir: "./test/ecs/service_discovery"}, - // }, + "ec2_performance": { + {testDir: "../../test/performance/emf"}, + {testDir: "../../test/performance/logs"}, + {testDir: "../../test/performance/system"}, + {testDir: "../../test/performance/statsd"}, + {testDir: "../../test/performance/collectd"}, + {testDir: "../../test/performance/trace/xray", runMockServer: true}, + }, + "ec2_windows_performance": { + {testDir: "../../test/performance/windows/logs"}, + {testDir: "../../test/performance/windows/system"}, + }, + "ec2_stress": { + {testDir: "../../test/stress/emf"}, + {testDir: "../../test/stress/logs"}, + {testDir: "../../test/stress/system"}, + {testDir: "../../test/stress/statsd"}, + {testDir: "../../test/stress/collectd"}, + {testDir: "../../test/stress/prometheus"}, + }, + "ec2_windows_stress": { + {testDir: "../../test/stress/windows/logs"}, + {testDir: "../../test/stress/windows/system"}, + }, + "ecs_fargate": { + {testDir: "./test/ecs/service_discovery"}, + }, "ecs_ec2_daemon": { - // { - // testDir: "./test/metric_value_benchmark", - // targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, - // }, - // { - // testDir: "./test/statsd", - // targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, - // }, - // { - // testDir: "./test/emf", - // targets: map[string]map[string]struct{}{"metadataEnabled": {"disabled": {}}}, - // }, - // { - // testDir: "./test/emf", - // targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, - // }, - // { - // testDir: "./test/ecs/service_discovery", - // targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, - // }, + { + testDir: "./test/metric_value_benchmark", + targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, + }, + { + testDir: "./test/statsd", + targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, + }, + { + testDir: "./test/emf", + targets: map[string]map[string]struct{}{"metadataEnabled": {"disabled": {}}}, + }, + { + testDir: "./test/emf", + targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, + }, + { + testDir: "./test/ecs/service_discovery", + targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, + }, { testDir: "./test/otel_collect/ecs/host_metrics", targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, }, }, - // "eks_addon": { - // { - // testDir: "./test/gpu", - // terraformDir: "terraform/eks/addon/gpu", - // }, - // }, - "eks_daemon": { - // { - // testDir: "./test/metric_value_benchmark", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // instanceType: "g4dn.xlarge", - // ami: "AL2_x86_64_GPU", - // }, - // { - // testDir: "./test/metric_value_benchmark", - // terraformDir: "terraform/eks/daemon/windows/2019", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // }, - // { - // testDir: "./test/metric_value_benchmark", - // terraformDir: "terraform/eks/daemon/windows/2022", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // }, - // { - // testDir: "./test/statsd", terraformDir: "terraform/eks/daemon/statsd", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // }, - // { - // testDir: "./test/emf", terraformDir: "terraform/eks/daemon/emf", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // }, - // { - // testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/d", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // }, - // {testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/bit"}, - // {testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/windows/2022"}, - // { - // testDir: "./test/gpu", terraformDir: "terraform/eks/daemon/gpu", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // instanceType: "g4dn.xlarge", - // ami: "AL2_x86_64_GPU", - // }, - // { - // testDir: "./test/gpu_high_frequency_metrics", terraformDir: "terraform/eks/daemon/gpu", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // instanceType: "g4dn.xlarge", - // ami: "AL2_x86_64_GPU", - // }, - // { - // testDir: "./test/awsneuron", terraformDir: "terraform/eks/daemon/awsneuron", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // wip: true, - // }, - // { - // testDir: "./test/entity", terraformDir: "terraform/eks/daemon/entity", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // }, - // { - // testDir: "./test/efa", terraformDir: "terraform/eks/daemon/efa", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // instanceType: "c6in.32xlarge", - // }, - // { - // testDir: "./test/metric_value_benchmark", terraformDir: "terraform/eks/daemon/credentials/pod_identity", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // }, - // { - // testDir: "./test/ebscsi", - // terraformDir: "terraform/eks/daemon/ebs", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // }, - // { - // testDir: "./test/liscsi", - // terraformDir: "terraform/eks/daemon/liscsi", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // instanceType: "i7i.xlarge", - // ami: "AL2023_x86_64_STANDARD", - // k8sVersion: "1.35", - // }, - { - testDir: "./test/otel/standard", - terraformDir: "terraform/eks/daemon/otel", - targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - ami: "AL2023_x86_64_STANDARD", - k8sVersion: "1.35", - }, - // { - // testDir: "./test/otel/attr_limit", - // terraformDir: "terraform/eks/daemon/otel-attr-limit", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // ami: "AL2023_x86_64_STANDARD", - // k8sVersion: "1.35", - // }, - // { - // testDir: "./test/otel/ebs_csi", - // terraformDir: "terraform/eks/daemon/otel-ebs-csi", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // ami: "AL2023_x86_64_STANDARD", - // k8sVersion: "1.35", - // }, - // { - // testDir: "./test/otel/efa", - // terraformDir: "terraform/eks/daemon/otel-efa", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // instanceType: "c5n.9xlarge", - // ami: "AL2023_x86_64_STANDARD", - // k8sVersion: "1.35", - // }, - // { - // testDir: "./test/otel/gpu", - // terraformDir: "terraform/eks/daemon/otel-gpu", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // instanceType: "g4dn.xlarge", - // ami: "AL2023_x86_64_NVIDIA", - // k8sVersion: "1.35", - // }, - // { - // testDir: "./test/otel/lis_csi", - // terraformDir: "terraform/eks/daemon/otel-lis-csi", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // instanceType: "i7i.xlarge", - // ami: "AL2023_x86_64_STANDARD", - // k8sVersion: "1.35", - // }, - // { - // testDir: "./test/otel/multi_efa", - // terraformDir: "terraform/eks/daemon/otel-multi-efa", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // instanceType: "c6in.32xlarge", - // ami: "AL2023_x86_64_STANDARD", - // k8sVersion: "1.35", - // }, - // { - // testDir: "./test/otel/neuron", - // terraformDir: "terraform/eks/daemon/otel-neuron", - // targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, - // instanceType: "inf2.xlarge", - // ami: "AL2023_x86_64_NEURON", - // k8sVersion: "1.35", - // }, - // }, - // "eks_deployment": { - // {testDir: "./test/metric_value_benchmark"}, - }, + "eks_addon": { + { + testDir: "./test/gpu", + terraformDir: "terraform/eks/addon/gpu", + }, + }, + "eks_daemon": { + { + testDir: "./test/metric_value_benchmark", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + instanceType: "g4dn.xlarge", + ami: "AL2_x86_64_GPU", + }, + { + testDir: "./test/metric_value_benchmark", + terraformDir: "terraform/eks/daemon/windows/2019", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/metric_value_benchmark", + terraformDir: "terraform/eks/daemon/windows/2022", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/statsd", terraformDir: "terraform/eks/daemon/statsd", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/emf", terraformDir: "terraform/eks/daemon/emf", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/d", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + {testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/bit"}, + {testDir: "./test/fluent", terraformDir: "terraform/eks/daemon/fluent/windows/2022"}, + { + testDir: "./test/gpu", terraformDir: "terraform/eks/daemon/gpu", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + instanceType: "g4dn.xlarge", + ami: "AL2_x86_64_GPU", + }, + { + testDir: "./test/gpu_high_frequency_metrics", terraformDir: "terraform/eks/daemon/gpu", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + instanceType: "g4dn.xlarge", + ami: "AL2_x86_64_GPU", + }, + { + testDir: "./test/awsneuron", terraformDir: "terraform/eks/daemon/awsneuron", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + wip: true, + }, + { + testDir: "./test/entity", terraformDir: "terraform/eks/daemon/entity", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/efa", terraformDir: "terraform/eks/daemon/efa", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + instanceType: "c6in.32xlarge", + }, + { + testDir: "./test/metric_value_benchmark", terraformDir: "terraform/eks/daemon/credentials/pod_identity", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/ebscsi", + terraformDir: "terraform/eks/daemon/ebs", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + }, + { + testDir: "./test/liscsi", + terraformDir: "terraform/eks/daemon/liscsi", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + instanceType: "i7i.xlarge", + ami: "AL2023_x86_64_STANDARD", + k8sVersion: "1.35", + }, + { + testDir: "./test/otel/standard", + terraformDir: "terraform/eks/daemon/otel", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + ami: "AL2023_x86_64_STANDARD", + k8sVersion: "1.35", + }, + { + testDir: "./test/otel/attr_limit", + terraformDir: "terraform/eks/daemon/otel-attr-limit", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + ami: "AL2023_x86_64_STANDARD", + k8sVersion: "1.35", + }, + { + testDir: "./test/otel/ebs_csi", + terraformDir: "terraform/eks/daemon/otel-ebs-csi", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + ami: "AL2023_x86_64_STANDARD", + k8sVersion: "1.35", + }, + { + testDir: "./test/otel/efa", + terraformDir: "terraform/eks/daemon/otel-efa", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + instanceType: "c5n.9xlarge", + ami: "AL2023_x86_64_STANDARD", + k8sVersion: "1.35", + }, + { + testDir: "./test/otel/gpu", + terraformDir: "terraform/eks/daemon/otel-gpu", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + instanceType: "g4dn.xlarge", + ami: "AL2023_x86_64_NVIDIA", + k8sVersion: "1.35", + }, + { + testDir: "./test/otel/lis_csi", + terraformDir: "terraform/eks/daemon/otel-lis-csi", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + instanceType: "i7i.xlarge", + ami: "AL2023_x86_64_STANDARD", + k8sVersion: "1.35", + }, + { + testDir: "./test/otel/multi_efa", + terraformDir: "terraform/eks/daemon/otel-multi-efa", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + instanceType: "c6in.32xlarge", + ami: "AL2023_x86_64_STANDARD", + k8sVersion: "1.35", + }, + { + testDir: "./test/otel/neuron", + terraformDir: "terraform/eks/daemon/otel-neuron", + targets: map[string]map[string]struct{}{"arc": {"amd64": {}}}, + instanceType: "inf2.xlarge", + ami: "AL2023_x86_64_NEURON", + k8sVersion: "1.35", + }, + }, + "eks_deployment": { + {testDir: "./test/metric_value_benchmark"}, + }, } var testTypeToTestConfigE2E = map[string][]testConfig{ @@ -563,10 +563,10 @@ var partitionTests = map[string]partition{ tests: []string{testTypeKeyEc2Linux}, ami: []string{"cloudwatch-agent-integration-test-aarch64-al2023*"}, excludedTestDirs: map[string]struct{}{ - "./test/otel_collect/database_insights": {}, - "./test/otel_collect/linux/host_metrics": {}, - "./test/otel_collect/linux/otlp": {}, - "./test/otel_collect/linux/prometheus": {}, + "./test/otel_collect/database_insights": {}, + "./test/otel_collect/linux/host_metrics": {}, + "./test/otel_collect/linux/otlp": {}, + "./test/otel_collect/linux/prometheus": {}, }, testConfigOverrides: map[string]testConfig{ "./test/metric_value_benchmark": { @@ -584,10 +584,10 @@ var partitionTests = map[string]partition{ tests: []string{testTypeKeyEc2Linux}, ami: []string{"cloudwatch-agent-integration-test-aarch64-al2023*"}, excludedTestDirs: map[string]struct{}{ - "./test/otel_collect/database_insights": {}, - "./test/otel_collect/linux/host_metrics": {}, - "./test/otel_collect/linux/otlp": {}, - "./test/otel_collect/linux/prometheus": {}, + "./test/otel_collect/database_insights": {}, + "./test/otel_collect/linux/host_metrics": {}, + "./test/otel_collect/linux/otlp": {}, + "./test/otel_collect/linux/prometheus": {}, }, testConfigOverrides: map[string]testConfig{ "./test/metric_value_benchmark": { diff --git a/test/otel_collect/ecs/host_metrics/host_metrics_ecs_test.go b/test/otel_collect/ecs/host_metrics/host_metrics_ecs_test.go index 7f755b7d..cfeb30e6 100644 --- a/test/otel_collect/ecs/host_metrics/host_metrics_ecs_test.go +++ b/test/otel_collect/ecs/host_metrics/host_metrics_ecs_test.go @@ -20,11 +20,8 @@ func init() { environment.RegisterEnvironmentMetaDataFlags() } -// ECSHostMetricsTestRunner validates that the OpenTelemetry (V2) host_metrics -// component, running as the CloudWatch Agent ECS daemon, publishes metrics to -// the OTLP endpoint (CloudWatch). The agent config is applied by terraform via -// the CW_CONFIG_CONTENT SSM parameter, so GetAgentConfigFileName returns "" and -// the ECS harness validates the already-running daemon without a restart. +// ECSHostMetricsTestRunner validates that the V2 host_metrics component, running +// as an ECS daemon, publishes metrics to CloudWatch via the OTLP endpoint. type ECSHostMetricsTestRunner struct { test_runner.BaseTestRunner } @@ -33,8 +30,7 @@ var _ test_runner.ITestRunner = (*ECSHostMetricsTestRunner)(nil) func (t *ECSHostMetricsTestRunner) GetTestName() string { return "ecs_otlp_host_metrics" } -// GetAgentConfigFileName returns "" so the harness uses the config terraform -// already loaded from resources/config.json (no daemon restart needed). +// GetAgentConfigFileName returns "" — config is pre-loaded by Terraform via SSM, no restart needed. func (t *ECSHostMetricsTestRunner) GetAgentConfigFileName() string { return "" } func (t *ECSHostMetricsTestRunner) GetMeasuredMetrics() []string { @@ -48,8 +44,7 @@ func (t *ECSHostMetricsTestRunner) GetMeasuredMetrics() []string { func (t *ECSHostMetricsTestRunner) Validate() status.TestGroupResult { env := environment.GetEnvironmentMetaData() - // Isolate this run's metrics by the ECS cluster ARN, which the agent's - // resourcedetection processor stamps as a resource attribute on ECS. + // Filter by ECS cluster ARN stamped by the resourcedetection processor. labels := map[string]string{ "@resource.aws.ecs.cluster.arn": env.EcsClusterArn, } diff --git a/test/otel_collect/ecs/host_metrics/resources/config.json b/test/otel_collect/ecs/host_metrics/resources/config.json index a2e4ea3d..da23ab89 100644 --- a/test/otel_collect/ecs/host_metrics/resources/config.json +++ b/test/otel_collect/ecs/host_metrics/resources/config.json @@ -1,7 +1,6 @@ { "agent": { - "run_as_user": "root", - "debug": true + "run_as_user": "root" }, "opentelemetry": { "cluster_name": "cwagent-ecs-integ-test", diff --git a/test/otel_collect/linux/otlp/otlp_test.go b/test/otel_collect/linux/otlp/otlp_test.go index 1f148b13..563feed7 100644 --- a/test/otel_collect/linux/otlp/otlp_test.go +++ b/test/otel_collect/linux/otlp/otlp_test.go @@ -7,22 +7,19 @@ package otlp import ( "bytes" - "context" "fmt" "net/http" "testing" "time" "github.com/stretchr/testify/require" - "go.opentelemetry.io/otel/attribute" "github.com/aws/amazon-cloudwatch-agent-test/environment" "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/otlpvalidation" "github.com/aws/amazon-cloudwatch-agent-test/test/status" "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" - "github.com/aws/amazon-cloudwatch-agent-test/util/common/traces/base" - "github.com/aws/amazon-cloudwatch-agent-test/util/common/traces/otlp" + "github.com/aws/amazon-cloudwatch-agent-test/util/common" ) func init() { @@ -32,22 +29,13 @@ func init() { const ( otlpRuntime = 3 * time.Minute otlpEndpoint = "http://127.0.0.1:4318" - - // traceTestType is used as an X-Ray annotation value so the trace validator - // can isolate this run's traces. It is set by the test (not derived from - // IMDS), so it works in both EC2 and on-prem mode. - traceTestType = "otel_collect_otlp_traces" - - // otlpLogGroup is the CloudWatch Logs group where OTLP logs published via the - // V2 opentelemetry pipeline land. NOTE: confirm this matches the group the - // agent actually writes to on a real run; adjust if the V2 OTLP logs path - // uses a different group/stream. otlpLogGroup = "/aws/cwagent" ) type OtlpCollectTestRunner struct { test_runner.BaseTestRunner - env *environment.MetaData + env *environment.MetaData + startedAt time.Time } var _ test_runner.ITestRunner = (*OtlpCollectTestRunner)(nil) @@ -57,36 +45,18 @@ func (t *OtlpCollectTestRunner) Validate() status.TestGroupResult { // Metrics metricResult := otlpvalidation.ValidateOtlpMetricsWithLabels(t.GetTestName(), t.env.Region, t.GetMeasuredMetrics(), - otlpvalidation.ResourceHostIDLabels(t.env.AgentStartCommand, t.env.InstanceId)) + otlpvalidation.OtlpMetricLabels(t.env.AgentStartCommand, t.env.InstanceId)) results = append(results, metricResult.TestResults...) - // Traces - results = append(results, t.validateTraces()) - // Logs results = append(results, t.validateLogs()) return status.TestGroupResult{Name: t.GetTestName(), TestResults: results} } -// validateTraces confirms the OTLP traces this run generated are queryable in -// X-Ray, filtered by the instance_id annotation for isolation. -func (t *OtlpCollectTestRunner) validateTraces() status.TestResult { - annotations := map[string]interface{}{ - "test_type": traceTestType, - "instance_id": t.env.InstanceId, - } - err := base.ValidateTraceSegments(time.Now().Add(-otlpRuntime), time.Now(), annotations, nil) - if err != nil { - return status.TestResult{Name: "OTLP_Traces", Status: status.FAILED, Reason: err} - } - return status.TestResult{Name: "OTLP_Traces", Status: status.SUCCESSFUL} -} - -// validateLogs confirms the OTLP logs this run published are present in -// CloudWatch Logs for this instance. +// validateLogs checks that OTLP logs from this run are in CloudWatch Logs. func (t *OtlpCollectTestRunner) validateLogs() status.TestResult { - since := time.Now().Add(-otlpRuntime) + since := t.startedAt until := time.Now() if len(awsservice.GetLogStreams(otlpLogGroup)) == 0 { @@ -118,28 +88,14 @@ func (t *OtlpCollectTestRunner) GetMeasuredMetrics() []string { } func (t *OtlpCollectTestRunner) SetupAfterAgentRun() error { - go t.sendTestMetrics() - go t.generateTraces() + t.startedAt = time.Now() + go func() { + _ = common.SendOTLPMetrics(otlpEndpoint, t.env.InstanceId, 10*time.Second, otlpRuntime-30*time.Second) + }() go t.sendTestLogs() return nil } -// generateTraces pushes OTLP traces to the agent's OTLP gRPC receiver -func (t *OtlpCollectTestRunner) generateTraces() { - generator := otlp.NewLoadGenerator(&base.TraceGeneratorConfig{ - Interval: 10 * time.Second, - Annotations: map[string]interface{}{ - "test_type": traceTestType, - "instance_id": t.env.InstanceId, - }, - Attributes: []attribute.KeyValue{ - attribute.String("test_type", traceTestType), - attribute.String("instance_id", t.env.InstanceId), - }, - }) - generator.StartSendingTraces(context.Background()) -} - // sendTestLogs pushes OTLP logs to the agent's OTLP HTTP receiver (/v1/logs). func (t *OtlpCollectTestRunner) sendTestLogs() { ticker := time.NewTicker(10 * time.Second) @@ -152,66 +108,23 @@ func (t *OtlpCollectTestRunner) sendTestLogs() { return case <-ticker.C: payload := buildOtlpLogsPayload(t.env.InstanceId) - req, _ := http.NewRequest("POST", otlpEndpoint+"/v1/logs", bytes.NewReader(payload)) - req.Header.Set("Content-Type", "application/json") - http.DefaultClient.Do(req) - } - } -} - -func (t *OtlpCollectTestRunner) sendTestMetrics() { - ticker := time.NewTicker(10 * time.Second) - defer ticker.Stop() - - timeout := time.After(otlpRuntime - 30*time.Second) - for { - select { - case <-timeout: - return - case <-ticker.C: - payload := buildOtlpMetricsPayload(t.env.InstanceId) - req, _ := http.NewRequest("POST", otlpEndpoint+"/v1/metrics", bytes.NewReader(payload)) + req, _ := http.NewRequest("POST", otlpEndpoint+"/v1/logs", bytes.NewReader(payload)) //nolint:errcheck req.Header.Set("Content-Type", "application/json") http.DefaultClient.Do(req) //nolint:errcheck } } } -var startTime = time.Now().UnixNano() - -func buildOtlpMetricsPayload(instanceId string) []byte { - now := time.Now().UnixNano() - payload := fmt.Sprintf(`{ - "resourceMetrics": [{ - "resource": {"attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}, - "scopeMetrics": [{ - "metrics": [ - { - "name": "otlp_test_counter", - "sum": { - "dataPoints": [{"asInt": "1", "startTimeUnixNano": "%d", "timeUnixNano": "%d", "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}], - "isMonotonic": true, - "aggregationTemporality": 2 - } - }, - { - "name": "otlp_test_gauge", - "gauge": { - "dataPoints": [{"asDouble": 42.0, "timeUnixNano": "%d", "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}] - } - } - ] - }] - }] -}`, instanceId, startTime, now, instanceId, now, instanceId) - return []byte(payload) -} - func buildOtlpLogsPayload(instanceId string) []byte { now := time.Now().UnixNano() + // Routes to CW Logs via aws.log.group.name / aws.log.stream.name resource attributes. payload := fmt.Sprintf(`{ "resourceLogs": [{ - "resource": {"attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}, + "resource": {"attributes": [ + {"key": "aws.log.group.name", "value": {"stringValue": "%s"}}, + {"key": "aws.log.stream.name", "value": {"stringValue": "%s"}}, + {"key": "InstanceId", "value": {"stringValue": "%s"}} + ]}, "scopeLogs": [{ "logRecords": [{ "timeUnixNano": "%d", @@ -221,7 +134,7 @@ func buildOtlpLogsPayload(instanceId string) []byte { }] }] }] -}`, instanceId, now, instanceId) +}`, otlpLogGroup, instanceId, instanceId, now, instanceId) return []byte(payload) } diff --git a/test/otel_collect/linux/prometheus/prometheus_test.go b/test/otel_collect/linux/prometheus/prometheus_test.go index 04ecd271..abfee8a0 100644 --- a/test/otel_collect/linux/prometheus/prometheus_test.go +++ b/test/otel_collect/linux/prometheus/prometheus_test.go @@ -8,7 +8,6 @@ package prometheus import ( _ "embed" "fmt" - "os" "testing" "time" @@ -62,7 +61,7 @@ func (t *PrometheusOtelTestRunner) SetupBeforeAgentRun() error { return err } - // Write prometheus scrape config + // Write the Prometheus scrape config where the agent config's config_path points. commands := []string{ fmt.Sprintf("cat <<'EOF' | sudo tee /opt/aws/prometheus.yml\n%s\nEOF", prometheusScrapeConfig), } @@ -70,20 +69,13 @@ func (t *PrometheusOtelTestRunner) SetupBeforeAgentRun() error { return err } - // Serve fake metrics on port 9100 (same pattern as test/emf_prometheus) - if err := os.WriteFile("/tmp/metrics", []byte(prometheusMetrics), os.ModePerm); err != nil { - return fmt.Errorf("unable to write /tmp/metrics: %w", err) + // Start the cross-platform fake Prometheus exporter on port 9100. + stop, err := common.StartPrometheusFakeServer(9100, prometheusMetrics) + if err != nil { + return fmt.Errorf("failed to start fake prometheus exporter: %w", err) } - commands = []string{ - "sudo python3 -m http.server 9100 --directory /tmp &> /dev/null &", - } - if err := common.RunCommands(commands); err != nil { - return err - } - t.RegisterCleanup(func() error { - return common.RunCommands([]string{"sudo pkill -f 'python3 -m http.server 9100' || true"}) - }) - time.Sleep(2 * time.Second) + t.RegisterCleanup(func() error { stop(); return nil }) + return nil } diff --git a/test/otel_collect/otlpvalidation/labels.go b/test/otel_collect/otlpvalidation/labels.go new file mode 100644 index 00000000..059c7f59 --- /dev/null +++ b/test/otel_collect/otlpvalidation/labels.go @@ -0,0 +1,44 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otlpvalidation + +import ( + "os" + "strings" +) + +// onPremiseMarker identifies the on-prem agent start command (-m onPremise). +const onPremiseMarker = "onPremise" + +// ResourceHostIDLabels returns the label filter for host_metrics/prometheus tests. +// EC2 filters on host.id; on-prem returns nil (no IMDS, validate presence only). +func ResourceHostIDLabels(agentStartCommand, instanceID string) map[string]string { + if strings.Contains(agentStartCommand, onPremiseMarker) { + return nil + } + return map[string]string{"@resource.host.id": instanceID} +} + +// OtlpMetricLabels returns the label filter for the OTLP test. +// EC2 uses host.id; on-prem uses the injected InstanceId attribute. +func OtlpMetricLabels(agentStartCommand, instanceID string) map[string]string { + if strings.Contains(agentStartCommand, onPremiseMarker) { + return map[string]string{"InstanceId": instanceID} + } + return map[string]string{"@resource.host.id": instanceID} +} + +// getRegion returns the region, falling back to AWS env vars then us-west-2. +func getRegion(region string) string { + if region != "" { + return region + } + if r := os.Getenv("AWS_REGION"); r != "" { + return r + } + if r := os.Getenv("AWS_DEFAULT_REGION"); r != "" { + return r + } + return "us-west-2" +} diff --git a/test/otel_collect/otlpvalidation/validate.go b/test/otel_collect/otlpvalidation/validate.go index 5835bcec..90cfb4fa 100644 --- a/test/otel_collect/otlpvalidation/validate.go +++ b/test/otel_collect/otlpvalidation/validate.go @@ -9,49 +9,17 @@ import ( "context" "fmt" "log" - "os" - "strings" "time" "github.com/aws/amazon-cloudwatch-agent-test/test/status" "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" ) -// onPremiseMarker identifies the on-prem agent start command (-m onPremise). -const onPremiseMarker = "onPremise" - -// ResourceHostIDLabels returns the metric label filter keyed on the entity -// host.id (the EC2 instance id) for EC2 mode. -// -// In on-prem mode (agent started with "-m onPremise") the agent does not read -// EC2 IMDS, so it does not set @resource.host.id to the instance id. Filtering -// on it there would never match, so we return nil and validate metric presence -// only. Detection is based on the agent start command from the test matrix. -func ResourceHostIDLabels(agentStartCommand, instanceID string) map[string]string { - if strings.Contains(agentStartCommand, onPremiseMarker) { - return nil - } - return map[string]string{"@resource.host.id": instanceID} -} - const ( defaultMaxRetries = 3 defaultRetryInterval = 30 * time.Second ) -func getRegion(region string) string { - if region != "" { - return region - } - if r := os.Getenv("AWS_REGION"); r != "" { - return r - } - if r := os.Getenv("AWS_DEFAULT_REGION"); r != "" { - return r - } - return "us-west-2" -} - func ValidateOtlpMetrics(testName string, region string, metrics []string) status.TestGroupResult { return ValidateOtlpMetricsWithLabels(testName, region, metrics, nil) } diff --git a/test/otel_collect/windows/host_metrics/agent_config.json b/test/otel_collect/windows/host_metrics/agent_config.json index 13b50217..dbb30733 100644 --- a/test/otel_collect/windows/host_metrics/agent_config.json +++ b/test/otel_collect/windows/host_metrics/agent_config.json @@ -1,13 +1,5 @@ { - "agent": { - "region": "us-west-2" - }, - "opentelemetry": { - "cluster_name": "integ-test-cluster", - "collect": { - "host_metrics": { - "collection_interval": 30 - } + "agent": { + "debug": true } - } } diff --git a/test/otel_collect/windows/host_metrics/host_metrics_unix.go b/test/otel_collect/windows/host_metrics/host_metrics_unix.go new file mode 100644 index 00000000..657a964b --- /dev/null +++ b/test/otel_collect/windows/host_metrics/host_metrics_unix.go @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package host_metrics + +import "errors" + +// Validate is only implemented on Windows; this stub lets the validator build on other platforms. +func Validate() error { + return errors.New("otel_collect windows host_metrics validation is only supported on Windows") +} diff --git a/test/otel_collect/windows/host_metrics/host_metrics_windows.go b/test/otel_collect/windows/host_metrics/host_metrics_windows.go new file mode 100644 index 00000000..ec9abd0e --- /dev/null +++ b/test/otel_collect/windows/host_metrics/host_metrics_windows.go @@ -0,0 +1,52 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build windows + +package host_metrics + +import ( + "context" + _ "embed" + "fmt" + "os" + "time" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/otlpvalidation" + "github.com/aws/amazon-cloudwatch-agent-test/util/common" + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +//go:embed resources/config.json +var testConfigJSON string + +const ( + tmpConfigPath = "C:\\Users\\Administrator\\AppData\\Local\\Temp\\config.json" + runtime = 3 * time.Minute +) + +func Validate() error { + env := environment.GetEnvironmentMetaData() + + if err := os.WriteFile(tmpConfigPath, []byte(testConfigJSON), 0644); err != nil { + return fmt.Errorf("could not write config: %w", err) + } + if err := common.CopyFile(tmpConfigPath, common.ConfigOutputPath); err != nil { + return fmt.Errorf("could not copy config: %w", err) + } + if err := common.StartAgent(common.ConfigOutputPath, true, false); err != nil { + return fmt.Errorf("could not start agent: %w", err) + } + time.Sleep(runtime) + _ = common.StopAgent() + + return otelmetrics.AssertMetricsPresent( + context.Background(), + env.Region, + []string{"system.cpu.utilization", "system.memory.utilization", "system.network.io", "system.disk.operations"}, + otlpvalidation.ResourceHostIDLabels(env.AgentStartCommand, env.InstanceId), + 3, + 30*time.Second, + ) +} diff --git a/test/otel_collect/windows/host_metrics/parameters.yml b/test/otel_collect/windows/host_metrics/parameters.yml index 85eaae5f..f24e66cb 100644 --- a/test/otel_collect/windows/host_metrics/parameters.yml +++ b/test/otel_collect/windows/host_metrics/parameters.yml @@ -1,23 +1,14 @@ -# Windows Host Metrics OTLP integration test -# Tests that OTel host_metrics collects system metrics on Windows -# and publishes to the OTLP monitoring endpoint with cluster_name +# FILE ONLY EXISTS SO VALIDATOR WILL RUN THE GO TEST +# SEE validator/main.go receivers: [] - -test_case: "win_otlp_host_metrics" +test_case: "win_otel_host_metrics" validate_type: "feature" data_type: "metrics" - -# Number of seconds the agent should run and collect metrics (3 minutes) -agent_collection_period: 180 - -cloudwatch_agent_config: "agent_config.json" - -# Validate host metrics are published via OTLP -metric_namespace: "CWAgent/System" +number_monitored_logs: 1 +values_per_minute: "2" +agent_collection_period: 60 +cloudwatch_agent_config: "" +metric_namespace: "CloudWatchAgentWinOtel" metric_validation: - - metric_name: "cpu_time" - metric_sample_count: 1 - - - metric_name: "memory_usage" - metric_sample_count: 1 +log_validation: diff --git a/test/otel_collect/windows/host_metrics/resources/config.json b/test/otel_collect/windows/host_metrics/resources/config.json new file mode 100644 index 00000000..2f2d0212 --- /dev/null +++ b/test/otel_collect/windows/host_metrics/resources/config.json @@ -0,0 +1,13 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "cluster_name": "cwagent-windows-integ-test", + "collect": { + "host_metrics": { + "collection_interval": 30 + } + } + } +} diff --git a/test/otel_collect/windows/otlp/agent_config.json b/test/otel_collect/windows/otlp/agent_config.json new file mode 100644 index 00000000..dbb30733 --- /dev/null +++ b/test/otel_collect/windows/otlp/agent_config.json @@ -0,0 +1,5 @@ +{ + "agent": { + "debug": true + } +} diff --git a/test/otel_collect/windows/otlp/otlp_windows.go b/test/otel_collect/windows/otlp/otlp_windows.go index 6793868b..613d6012 100644 --- a/test/otel_collect/windows/otlp/otlp_windows.go +++ b/test/otel_collect/windows/otlp/otlp_windows.go @@ -12,7 +12,8 @@ import ( "os" "time" - "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/otlpvalidation" "github.com/aws/amazon-cloudwatch-agent-test/util/common" "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" ) @@ -26,13 +27,11 @@ const ( sendInterval = 10 * time.Second ) -// Validate runs the Windows OTLP test: start the agent with the V2 opentelemetry OTLP config, -// push OTLP metrics to the agent, then validate they reach CloudWatch via the PromQL endpoint. func Validate() error { - instanceID := awsservice.GetInstanceId() + env := environment.GetEnvironmentMetaData() if err := os.WriteFile(tmpConfigPath, []byte(testConfigJSON), 0644); err != nil { - return fmt.Errorf("could not write config file: %w", err) + return fmt.Errorf("could not write config: %w", err) } if err := common.CopyFile(tmpConfigPath, common.ConfigOutputPath); err != nil { return fmt.Errorf("could not copy config: %w", err) @@ -40,21 +39,20 @@ func Validate() error { if err := common.StartAgent(common.ConfigOutputPath, true, false); err != nil { return fmt.Errorf("could not start agent: %w", err) } - time.Sleep(5 * time.Second) - - // Push OTLP metrics to the agent's OTLP/HTTP receiver for the collection period. - if err := common.SendOTLPMetrics(common.DefaultOTLPHTTPEndpoint, instanceID, sendInterval, otlpRuntime); err != nil { + // Wait for the OTLP HTTP receiver before sending metrics. + if err := common.WaitForOTLPEndpoint(common.DefaultOTLPHTTPEndpoint, 2*time.Minute); err != nil { + return fmt.Errorf("OTLP endpoint not ready: %w", err) + } + if err := common.SendOTLPMetrics(common.DefaultOTLPHTTPEndpoint, env.InstanceId, sendInterval, otlpRuntime); err != nil { return fmt.Errorf("failed to send OTLP metrics: %w", err) } - _ = common.StopAgent() - // Validate the pushed metrics landed in CloudWatch, isolated by the instance's host.id. return otelmetrics.AssertMetricsPresent( context.Background(), - "", // region resolved from AWS_REGION env + env.Region, []string{"otlp_test_counter", "otlp_test_gauge"}, - map[string]string{"@resource.host.id": instanceID}, + otlpvalidation.OtlpMetricLabels(env.AgentStartCommand, env.InstanceId), 3, 30*time.Second, ) diff --git a/test/otel_collect/windows/otlp/parameters.yml b/test/otel_collect/windows/otlp/parameters.yml new file mode 100644 index 00000000..1ddb28af --- /dev/null +++ b/test/otel_collect/windows/otlp/parameters.yml @@ -0,0 +1,17 @@ +# FILE ONLY EXISTS SO VALIDATOR WILL RUN THE GO TEST +# SEE validator/main.go + +# Receivers that agent needs to test +receivers: [] + +# Test case name +test_case: "win_otlp" +validate_type: "feature" +data_type: "metrics" +number_monitored_logs: 1 +values_per_minute: "2" +agent_collection_period: 60 +cloudwatch_agent_config: "" +metric_namespace: "CloudWatchAgentWinOtel" +metric_validation: +log_validation: diff --git a/test/otel_collect/windows/prometheus/agent_config.json b/test/otel_collect/windows/prometheus/agent_config.json index 1f2bf690..dbb30733 100644 --- a/test/otel_collect/windows/prometheus/agent_config.json +++ b/test/otel_collect/windows/prometheus/agent_config.json @@ -1,13 +1,5 @@ { - "agent": { - "region": "us-west-2" - }, - "opentelemetry": { - "cluster_name": "integ-test-cluster", - "collect": { - "prometheus": { - "config_path": "C:\\prometheus.yml" - } + "agent": { + "debug": true } - } } diff --git a/test/otel_collect/windows/prometheus/parameters.yml b/test/otel_collect/windows/prometheus/parameters.yml index 42ddc1a8..77b736be 100644 --- a/test/otel_collect/windows/prometheus/parameters.yml +++ b/test/otel_collect/windows/prometheus/parameters.yml @@ -1,45 +1,14 @@ -# Windows Prometheus OTLP integration test -# Tests that OTel Prometheus receiver scrapes metrics on Windows -# and publishes to the OTLP monitoring endpoint +# FILE ONLY EXISTS SO VALIDATOR WILL RUN THE GO TEST +# SEE validator/main.go receivers: [] - -test_case: "win_otlp_prometheus" +test_case: "win_otel_prometheus" validate_type: "feature" data_type: "metrics" - -# Number of seconds the agent should run and collect metrics (3 minutes) -agent_collection_period: 180 - -cloudwatch_agent_config: "agent_config.json" - -# Preparation script: writes prometheus scrape config and starts a fake metrics server -# The validator will run this PowerShell before starting the agent -preparation_cmd: | - Set-Content -Path "C:\prometheus.yml" -Value (Get-Content "C:\prometheus_scrape_config.yaml") - $listener = [System.Net.HttpListener]::new() - $listener.Prefixes.Add("http://+:9100/") - $listener.Start() - $job = Start-Job -ScriptBlock { - param($metrics) - $l = [System.Net.HttpListener]::new() - $l.Prefixes.Add("http://+:9100/") - $l.Start() - while ($l.IsListening) { - $ctx = $l.GetContext() - $response = $ctx.Response - $buffer = [System.Text.Encoding]::UTF8.GetBytes($metrics) - $response.ContentLength64 = $buffer.Length - $response.OutputStream.Write($buffer, 0, $buffer.Length) - $response.Close() - } - } -ArgumentList (Get-Content "C:\prometheus_metrics" -Raw) - -# Validate prometheus metrics are published via OTLP -metric_namespace: "CWAgent" +number_monitored_logs: 1 +values_per_minute: "2" +agent_collection_period: 60 +cloudwatch_agent_config: "" +metric_namespace: "CloudWatchAgentWinOtel" metric_validation: - - metric_name: "node_cpu_seconds_total" - metric_sample_count: 1 - - - metric_name: "node_memory_MemAvailable_bytes" - metric_sample_count: 1 +log_validation: diff --git a/test/otel_collect/windows/prometheus/prometheus_metrics b/test/otel_collect/windows/prometheus/prometheus_metrics deleted file mode 100644 index 1e158f2a..00000000 --- a/test/otel_collect/windows/prometheus/prometheus_metrics +++ /dev/null @@ -1,13 +0,0 @@ -# HELP node_cpu_seconds_total Seconds the CPUs spent in each mode. -# TYPE node_cpu_seconds_total counter -node_cpu_seconds_total{cpu="0",mode="idle"} 12345.67 -node_cpu_seconds_total{cpu="0",mode="system"} 234.56 -# HELP node_memory_MemAvailable_bytes Memory information field MemAvailable_bytes. -# TYPE node_memory_MemAvailable_bytes gauge -node_memory_MemAvailable_bytes 4123456789 -# HELP node_filesystem_avail_bytes Filesystem space available to non-root users in bytes. -# TYPE node_filesystem_avail_bytes gauge -node_filesystem_avail_bytes{device="C:",fstype="ntfs",mountpoint="C:"} 50000000000 -# HELP node_network_receive_bytes_total Network device statistic receive_bytes. -# TYPE node_network_receive_bytes_total counter -node_network_receive_bytes_total{device="Ethernet"} 987654321 diff --git a/test/otel_collect/windows/prometheus/prometheus_unix.go b/test/otel_collect/windows/prometheus/prometheus_unix.go new file mode 100644 index 00000000..e9d8d7ae --- /dev/null +++ b/test/otel_collect/windows/prometheus/prometheus_unix.go @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package prometheus + +import "errors" + +// Validate is only implemented on Windows; this stub lets the validator build on other platforms. +func Validate() error { + return errors.New("otel_collect windows prometheus validation is only supported on Windows") +} diff --git a/test/otel_collect/windows/prometheus/prometheus_windows.go b/test/otel_collect/windows/prometheus/prometheus_windows.go new file mode 100644 index 00000000..b57b2177 --- /dev/null +++ b/test/otel_collect/windows/prometheus/prometheus_windows.go @@ -0,0 +1,69 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build windows + +package prometheus + +import ( + "context" + _ "embed" + "fmt" + "os" + "time" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/otlpvalidation" + "github.com/aws/amazon-cloudwatch-agent-test/util/common" + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +//go:embed resources/config.json +var testConfigJSON string + +//go:embed resources/prometheus_scrape.yaml +var scrapeConfig string + +//go:embed resources/prometheus_metrics +var prometheusMetrics string + +const ( + tmpConfigPath = "C:\\Users\\Administrator\\AppData\\Local\\Temp\\config.json" + scrapePath = "C:\\prometheus_scrape.yaml" + exporterPort = 8101 + runtime = 3 * time.Minute +) + +func Validate() error { + env := environment.GetEnvironmentMetaData() + + stop, err := common.StartPrometheusFakeServer(exporterPort, prometheusMetrics) + if err != nil { + return fmt.Errorf("could not start fake prometheus exporter: %w", err) + } + defer stop() + + if err := os.WriteFile(scrapePath, []byte(scrapeConfig), 0644); err != nil { + return fmt.Errorf("could not write scrape config: %w", err) + } + if err := os.WriteFile(tmpConfigPath, []byte(testConfigJSON), 0644); err != nil { + return fmt.Errorf("could not write agent config: %w", err) + } + if err := common.CopyFile(tmpConfigPath, common.ConfigOutputPath); err != nil { + return fmt.Errorf("could not copy config: %w", err) + } + if err := common.StartAgent(common.ConfigOutputPath, true, false); err != nil { + return fmt.Errorf("could not start agent: %w", err) + } + time.Sleep(runtime) + _ = common.StopAgent() + + return otelmetrics.AssertMetricsPresent( + context.Background(), + env.Region, + []string{"node_cpu_seconds_total", "node_memory_MemAvailable_bytes"}, + otlpvalidation.ResourceHostIDLabels(env.AgentStartCommand, env.InstanceId), + 3, + 30*time.Second, + ) +} diff --git a/test/otel_collect/windows/prometheus/resources/config.json b/test/otel_collect/windows/prometheus/resources/config.json new file mode 100644 index 00000000..8bc6d04b --- /dev/null +++ b/test/otel_collect/windows/prometheus/resources/config.json @@ -0,0 +1,13 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "cluster_name": "cwagent-windows-integ-test", + "collect": { + "prometheus": { + "config_path": "C:\\prometheus_scrape.yaml" + } + } + } +} diff --git a/test/otel_collect/windows/prometheus/resources/prometheus_metrics b/test/otel_collect/windows/prometheus/resources/prometheus_metrics new file mode 100644 index 00000000..9ce5127f --- /dev/null +++ b/test/otel_collect/windows/prometheus/resources/prometheus_metrics @@ -0,0 +1,7 @@ +# HELP node_cpu_seconds_total Seconds the CPUs spent in each mode. +# TYPE node_cpu_seconds_total counter +node_cpu_seconds_total{cpu="0",mode="idle"} 12345.67 +node_cpu_seconds_total{cpu="0",mode="system"} 234.56 +# HELP node_memory_MemAvailable_bytes Memory information field MemAvailable_bytes. +# TYPE node_memory_MemAvailable_bytes gauge +node_memory_MemAvailable_bytes 4123456789 diff --git a/test/otel_collect/windows/prometheus/prometheus_scrape_config.yaml b/test/otel_collect/windows/prometheus/resources/prometheus_scrape.yaml similarity index 82% rename from test/otel_collect/windows/prometheus/prometheus_scrape_config.yaml rename to test/otel_collect/windows/prometheus/resources/prometheus_scrape.yaml index 9d7a3578..efd6350e 100644 --- a/test/otel_collect/windows/prometheus/prometheus_scrape_config.yaml +++ b/test/otel_collect/windows/prometheus/resources/prometheus_scrape.yaml @@ -1,7 +1,8 @@ scrape_configs: - job_name: node + scrape_interval: 10s static_configs: - - targets: ['localhost:9100'] + - targets: ['localhost:8101'] relabel_configs: - source_labels: [job] regex: (.*) diff --git a/util/common/otlp.go b/util/common/otlp.go index fd9c96ab..b87f00fa 100644 --- a/util/common/otlp.go +++ b/util/common/otlp.go @@ -15,7 +15,25 @@ import ( // DefaultOTLPHTTPEndpoint is the agent's default OTLP/HTTP receiver endpoint. const DefaultOTLPHTTPEndpoint = "http://127.0.0.1:4318" -// SendOTLPMetrics pushes OTLP metrics to the agent's OTLP/HTTP receiver until the duration elapses (cross-platform). +// WaitForOTLPEndpoint polls the OTLP HTTP endpoint until it responds or timeout elapses. +// Use after StartAgent instead of a fixed sleep. +func WaitForOTLPEndpoint(endpoint string, timeout time.Duration) error { + if endpoint == "" { + endpoint = DefaultOTLPHTTPEndpoint + } + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + resp, err := http.Get(endpoint) //nolint:noctx + if err == nil { + resp.Body.Close() + return nil + } + time.Sleep(2 * time.Second) + } + return fmt.Errorf("OTLP endpoint %s not ready after %s", endpoint, timeout) +} + +// SendOTLPMetrics pushes OTLP metrics to the agent's HTTP receiver until duration elapses. func SendOTLPMetrics(endpoint, instanceID string, sendingInterval, duration time.Duration) error { if endpoint == "" { endpoint = DefaultOTLPHTTPEndpoint @@ -44,9 +62,10 @@ func SendOTLPMetrics(endpoint, instanceID string, sendingInterval, duration time } } -// buildOTLPMetricsPayload builds an OTLP/HTTP JSON metrics payload with an InstanceId attribute for isolation. +// buildOTLPMetricsPayload builds a delta counter + gauge payload tagged with instanceID. func buildOTLPMetricsPayload(instanceID string) []byte { now := time.Now().UnixNano() + start := now - int64(10*time.Second) return []byte(fmt.Sprintf(`{ "resourceMetrics": [{ "resource": {"attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}, @@ -55,9 +74,9 @@ func buildOTLPMetricsPayload(instanceID string) []byte { { "name": "otlp_test_counter", "sum": { - "dataPoints": [{"asInt": "1", "timeUnixNano": "%d", "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}], + "dataPoints": [{"asInt": "1", "startTimeUnixNano": "%d", "timeUnixNano": "%d", "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}], "isMonotonic": true, - "aggregationTemporality": 2 + "aggregationTemporality": 1 } }, { @@ -69,10 +88,10 @@ func buildOTLPMetricsPayload(instanceID string) []byte { ] }] }] -}`, instanceID, now, instanceID, now, instanceID)) +}`, instanceID, start, now, instanceID, now, instanceID)) } -// StartPrometheusFakeServer serves static Prometheus metrics on the port until stop() is called (cross-platform). +// StartPrometheusFakeServer serves static Prometheus metrics on the given port until stop() is called. func StartPrometheusFakeServer(port int, exposition string) (stop func(), err error) { mux := http.NewServeMux() mux.HandleFunc("/metrics", func(w http.ResponseWriter, _ *http.Request) { diff --git a/util/common/traces/otlp/generator.go b/util/common/traces/otlp/generator.go index 98e41eca..487f105b 100644 --- a/util/common/traces/otlp/generator.go +++ b/util/common/traces/otlp/generator.go @@ -125,7 +125,8 @@ func setupClient(ctx context.Context) (*sdktrace.TracerProvider, func(context.Co } func setupTraceProvider(ctx context.Context, res *resource.Resource) (*sdktrace.TracerProvider, error) { - exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithInsecure()) + // Use 127.0.0.1 (not "localhost", which can resolve to IPv6 [::1]) since the agent binds IPv4. + exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithInsecure(), otlptracegrpc.WithEndpoint("127.0.0.1:4317")) if err != nil { return nil, err } diff --git a/validator/main.go b/validator/main.go index 3a20f85c..339387f4 100644 --- a/validator/main.go +++ b/validator/main.go @@ -18,8 +18,10 @@ import ( "github.com/aws/amazon-cloudwatch-agent-test/test/log_state/logfile" "github.com/aws/amazon-cloudwatch-agent-test/test/log_state/windows_event_log" "github.com/aws/amazon-cloudwatch-agent-test/test/nvidia_gpu" - "github.com/aws/amazon-cloudwatch-agent-test/test/restart" + otelhostmetrics "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/windows/host_metrics" otelcollectotlp "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/windows/otlp" + otelprometheus "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/windows/prometheus" + "github.com/aws/amazon-cloudwatch-agent-test/test/restart" "github.com/aws/amazon-cloudwatch-agent-test/test/ssm_document" "github.com/aws/amazon-cloudwatch-agent-test/test/workload_discovery" "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" @@ -70,6 +72,10 @@ func main() { err = ssm_document.Validate() case "otlp": err = otelcollectotlp.Validate() + case "host_metrics": + err = otelhostmetrics.Validate() + case "prometheus": + err = otelprometheus.Validate() default: err = errors.New("unsupported test name") } From 2fcc31b5deaf4a9a34667f28697d2e00c2819a58 Mon Sep 17 00:00:00 2001 From: Michael Commey Date: Fri, 24 Jul 2026 20:39:02 +0000 Subject: [PATCH 5/5] fix ecs otlp test --- generator/test_case_generator.go | 2 +- .../ecs/host_metrics/host_metrics_ecs_test.go | 78 ----------------- .../ec2_launch/daemon/ecs_taskdef.tpl | 58 +++++++++++++ test/otel_collect/ecs/otlp/otlp_ecs_test.go | 87 +++++++++++++++++++ .../resources/config.json | 5 +- 5 files changed, 149 insertions(+), 81 deletions(-) delete mode 100644 test/otel_collect/ecs/host_metrics/host_metrics_ecs_test.go create mode 100644 test/otel_collect/ecs/otlp/ecs_resources/ec2_launch/daemon/ecs_taskdef.tpl create mode 100644 test/otel_collect/ecs/otlp/otlp_ecs_test.go rename test/otel_collect/ecs/{host_metrics => otlp}/resources/config.json (60%) diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index 5184824d..07d9987b 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -381,7 +381,7 @@ var testTypeToTestConfig = map[string][]testConfig{ targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, }, { - testDir: "./test/otel_collect/ecs/host_metrics", + testDir: "./test/otel_collect/ecs/otlp", targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, }, }, diff --git a/test/otel_collect/ecs/host_metrics/host_metrics_ecs_test.go b/test/otel_collect/ecs/host_metrics/host_metrics_ecs_test.go deleted file mode 100644 index 8e91d2f3..00000000 --- a/test/otel_collect/ecs/host_metrics/host_metrics_ecs_test.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -//go:build !windows - -package ecs - -import ( - "testing" - "time" - - "github.com/stretchr/testify/suite" - - "github.com/aws/amazon-cloudwatch-agent-test/environment" - "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/otlpvalidation" - "github.com/aws/amazon-cloudwatch-agent-test/test/status" - "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" -) - -func init() { - environment.RegisterEnvironmentMetaDataFlags() -} - -// ECSHostMetricsTestRunner validates that the V2 host_metrics component, running -// as an ECS daemon, publishes metrics to CloudWatch via the OTLP endpoint. -type ECSHostMetricsTestRunner struct { - test_runner.BaseTestRunner -} - -var _ test_runner.ITestRunner = (*ECSHostMetricsTestRunner)(nil) - -func (t *ECSHostMetricsTestRunner) GetTestName() string { return "ecs_otlp_host_metrics" } - -// GetAgentConfigFileName returns "" — config is pre-loaded by Terraform via SSM, no restart needed. -func (t *ECSHostMetricsTestRunner) GetAgentConfigFileName() string { return "" } - -func (t *ECSHostMetricsTestRunner) GetMeasuredMetrics() []string { - return []string{ - "system.cpu.utilization", - "system.memory.utilization", - "system.network.io", - "system.disk.operations", - } -} - -func (t *ECSHostMetricsTestRunner) Validate() status.TestGroupResult { - env := environment.GetEnvironmentMetaData() - time.Sleep(3 * time.Minute) - // Filter by ECS cluster ARN stamped by the resourcedetection processor. - labels := map[string]string{ - "@resource.aws.ecs.cluster.arn": env.EcsClusterArn, - } - return otlpvalidation.ValidateOtlpMetricsWithLabels(t.GetTestName(), env.Region, t.GetMeasuredMetrics(), labels) -} - -func TestECSOtelHostMetricsSuite(t *testing.T) { - suite.Run(t, new(ECSHostMetricsTestSuite)) -} - -type ECSHostMetricsTestSuite struct { - suite.Suite - test_runner.TestSuite -} - -func (suite *ECSHostMetricsTestSuite) GetSuiteName() string { - return "ECSOtelHostMetrics" -} - -func (suite *ECSHostMetricsTestSuite) TestAllInSuite() { - env := environment.GetEnvironmentMetaData() - ecsTestRunner := &test_runner.ECSTestRunner{ - Runner: &ECSHostMetricsTestRunner{}, - RunStrategy: &test_runner.ECSAgentRunStrategy{}, - Env: *env, - } - ecsTestRunner.Run(suite, env) - suite.Assert().Equal(status.SUCCESSFUL, suite.Result.GetStatus(), "ECS OTLP Host Metrics Test Suite Failed") -} diff --git a/test/otel_collect/ecs/otlp/ecs_resources/ec2_launch/daemon/ecs_taskdef.tpl b/test/otel_collect/ecs/otlp/ecs_resources/ec2_launch/daemon/ecs_taskdef.tpl new file mode 100644 index 00000000..36b5fe32 --- /dev/null +++ b/test/otel_collect/ecs/otlp/ecs_resources/ec2_launch/daemon/ecs_taskdef.tpl @@ -0,0 +1,58 @@ +[ + { + "name": "cloudwatch_agent", + "image": "${cwagent_image}", + "essential": true, + "secrets": [ + { + "name": "CW_CONFIG_CONTENT", + "valueFrom": "${cwagent_ssm_parameter_arn}" + }, + { + "name": "PROMETHEUS_CONFIG_CONTENT", + "valueFrom": "${prometheus_ssm_parameter_arn}" + } + ], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-region": "${region}", + "awslogs-stream-prefix": "${testing_id}", + "awslogs-group": "${log_group}" + } + }, + "cpu": 1, + "memory": 2048 + }, + { + "name": "otlp_pusher", + "image": "curlimages/curl:8.10.1", + "essential": false, + "links": [ + "cloudwatch_agent" + ], + "environment": [ + { + "name": "TEST_ID", + "value": "${testing_id}" + } + ], + "entryPoint": [ + "sh", + "-c" + ], + "command": [ + "while true; do S=$$(date +%s); NOW=$${S}000000000; START=$$((S-10))000000000; curl -s -X POST http://cloudwatch_agent:4318/v1/metrics -H 'Content-Type: application/json' -d '{\"resourceMetrics\":[{\"resource\":{\"attributes\":[{\"key\":\"TestId\",\"value\":{\"stringValue\":\"'\"$${TEST_ID}\"'\"}}]},\"scopeMetrics\":[{\"metrics\":[{\"name\":\"otlp_test_counter\",\"sum\":{\"dataPoints\":[{\"asInt\":\"1\",\"startTimeUnixNano\":\"'\"$${START}\"'\",\"timeUnixNano\":\"'\"$${NOW}\"'\",\"attributes\":[{\"key\":\"TestId\",\"value\":{\"stringValue\":\"'\"$${TEST_ID}\"'\"}}]}],\"aggregationTemporality\":1,\"isMonotonic\":true}},{\"name\":\"otlp_test_gauge\",\"gauge\":{\"dataPoints\":[{\"asDouble\":42.0,\"timeUnixNano\":\"'\"$${NOW}\"'\",\"attributes\":[{\"key\":\"TestId\",\"value\":{\"stringValue\":\"'\"$${TEST_ID}\"'\"}}]}]}}]}]}]}'; sleep 10; done" + ], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-region": "${region}", + "awslogs-stream-prefix": "${testing_id}-pusher", + "awslogs-group": "${log_group}" + } + }, + "cpu": 1, + "memory": 256 + } +] diff --git a/test/otel_collect/ecs/otlp/otlp_ecs_test.go b/test/otel_collect/ecs/otlp/otlp_ecs_test.go new file mode 100644 index 00000000..92741ec6 --- /dev/null +++ b/test/otel_collect/ecs/otlp/otlp_ecs_test.go @@ -0,0 +1,87 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package otlp + +import ( + "strings" + "testing" + "time" + + "github.com/stretchr/testify/suite" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/otlpvalidation" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" +) + +func init() { + environment.RegisterEnvironmentMetaDataFlags() +} + +// clusterNamePrefix matches the ECS cluster naming in terraform/ecs_ec2/daemon. +const clusterNamePrefix = "cwagent-integ-test-cluster-" + +// ECSOtlpTestRunner validates that a workload (sidecar) running on the ECS daemon +// can publish OTLP metrics to the agent's OTLP receiver and have them reach CloudWatch. +type ECSOtlpTestRunner struct { + test_runner.BaseTestRunner +} + +var _ test_runner.ITestRunner = (*ECSOtlpTestRunner)(nil) + +func (t *ECSOtlpTestRunner) GetTestName() string { return "ecs_otlp" } + +// GetAgentConfigFileName returns "" — config is pre-loaded by Terraform via SSM, no restart needed. +func (t *ECSOtlpTestRunner) GetAgentConfigFileName() string { return "" } + +func (t *ECSOtlpTestRunner) GetMeasuredMetrics() []string { + return []string{"otlp_test_counter", "otlp_test_gauge"} +} + +func (t *ECSOtlpTestRunner) Validate() status.TestGroupResult { + env := environment.GetEnvironmentMetaData() + // Give the sidecar time to push and the agent to export before querying. + time.Sleep(3 * time.Minute) + // Isolate by the per-run testing_id, which the sidecar stamps as the TestId + // attribute and is embedded in the ECS cluster ARN. + labels := map[string]string{ + "TestId": testIDFromClusterArn(env.EcsClusterArn), + } + return otlpvalidation.ValidateOtlpMetricsWithLabels(t.GetTestName(), env.Region, t.GetMeasuredMetrics(), labels) +} + +// testIDFromClusterArn extracts the testing_id suffix from the ECS cluster ARN. +func testIDFromClusterArn(clusterArn string) string { + if i := strings.LastIndex(clusterArn, clusterNamePrefix); i != -1 { + return clusterArn[i+len(clusterNamePrefix):] + } + return clusterArn +} + +func TestECSOtlpSuite(t *testing.T) { + suite.Run(t, new(ECSOtlpTestSuite)) +} + +type ECSOtlpTestSuite struct { + suite.Suite + test_runner.TestSuite +} + +func (suite *ECSOtlpTestSuite) GetSuiteName() string { + return "ECSOtlp" +} + +func (suite *ECSOtlpTestSuite) TestAllInSuite() { + env := environment.GetEnvironmentMetaData() + ecsTestRunner := &test_runner.ECSTestRunner{ + Runner: &ECSOtlpTestRunner{}, + RunStrategy: &test_runner.ECSAgentRunStrategy{}, + Env: *env, + } + ecsTestRunner.Run(suite, env) + suite.Assert().Equal(status.SUCCESSFUL, suite.Result.GetStatus(), "ECS OTLP Test Suite Failed") +} diff --git a/test/otel_collect/ecs/host_metrics/resources/config.json b/test/otel_collect/ecs/otlp/resources/config.json similarity index 60% rename from test/otel_collect/ecs/host_metrics/resources/config.json rename to test/otel_collect/ecs/otlp/resources/config.json index da23ab89..c18eeb5a 100644 --- a/test/otel_collect/ecs/host_metrics/resources/config.json +++ b/test/otel_collect/ecs/otlp/resources/config.json @@ -5,8 +5,9 @@ "opentelemetry": { "cluster_name": "cwagent-ecs-integ-test", "collect": { - "host_metrics": { - "collection_interval": 30 + "otlp": { + "grpc_endpoint": "0.0.0.0:4317", + "http_endpoint": "0.0.0.0:4318" } } }