diff --git a/generator/test_case_generator.go b/generator/test_case_generator.go index 344d9690e..07d9987b7 100644 --- a/generator/test_case_generator.go +++ b/generator/test_case_generator.go @@ -89,6 +89,9 @@ var testTypeToTestConfig = map[string][]testConfig{ }, "ec2_linux_onprem": { {testDir: "./test/cloudwatchlogs"}, + {testDir: "./test/otel_collect/linux/otlp"}, + {testDir: "./test/otel_collect/linux/host_metrics"}, + {testDir: "./test/otel_collect/linux/prometheus"}, }, testTypeKeyEc2Linux: { {testDir: "./test/ca_bundle"}, @@ -137,10 +140,10 @@ var testTypeToTestConfig = map[string][]testConfig{ testDir: "./test/otel_collect/database_insights", excludedOs: map[string]struct{}{"ol8": {}, "ubuntu-25": {}}, }, - {testDir: "./test/otel_collect/host_metrics"}, - {testDir: "./test/otel_collect/otlp"}, + {testDir: "./test/otel_collect/linux/host_metrics"}, + {testDir: "./test/otel_collect/linux/otlp"}, { - testDir: "./test/otel_collect/prometheus", + testDir: "./test/otel_collect/linux/prometheus", excludedOs: map[string]struct{}{"rhel8": {}, "ol8": {}, "sles-15": {}}, }, { @@ -323,6 +326,9 @@ var testTypeToTestConfig = map[string][]testConfig{ targets: map[string]map[string]struct{}{"os": {"win-2019": {}}}, }, {testDir: "../../../test/ssm_document"}, + {testDir: "../../../test/otel_collect/windows/otlp"}, + {testDir: "../../../test/otel_collect/windows/host_metrics"}, + {testDir: "../../../test/otel_collect/windows/prometheus"}, // assume role test doesn't add much value, and it already being tested with linux //{testDir: "../../../test/assume_role"}, }, @@ -374,6 +380,10 @@ var testTypeToTestConfig = map[string][]testConfig{ testDir: "./test/ecs/service_discovery", targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, }, + { + testDir: "./test/otel_collect/ecs/otlp", + targets: map[string]map[string]struct{}{"metadataEnabled": {"enabled": {}}}, + }, }, "eks_addon": { { @@ -553,10 +563,10 @@ var partitionTests = map[string]partition{ tests: []string{testTypeKeyEc2Linux}, ami: []string{"cloudwatch-agent-integration-test-aarch64-al2023*"}, excludedTestDirs: map[string]struct{}{ - "./test/otel_collect/database_insights": {}, - "./test/otel_collect/host_metrics": {}, - "./test/otel_collect/otlp": {}, - "./test/otel_collect/prometheus": {}, + "./test/otel_collect/database_insights": {}, + "./test/otel_collect/linux/host_metrics": {}, + "./test/otel_collect/linux/otlp": {}, + "./test/otel_collect/linux/prometheus": {}, }, testConfigOverrides: map[string]testConfig{ "./test/metric_value_benchmark": { @@ -574,10 +584,10 @@ var partitionTests = map[string]partition{ tests: []string{testTypeKeyEc2Linux}, ami: []string{"cloudwatch-agent-integration-test-aarch64-al2023*"}, excludedTestDirs: map[string]struct{}{ - "./test/otel_collect/database_insights": {}, - "./test/otel_collect/host_metrics": {}, - "./test/otel_collect/otlp": {}, - "./test/otel_collect/prometheus": {}, + "./test/otel_collect/database_insights": {}, + "./test/otel_collect/linux/host_metrics": {}, + "./test/otel_collect/linux/otlp": {}, + "./test/otel_collect/linux/prometheus": {}, }, testConfigOverrides: map[string]testConfig{ "./test/metric_value_benchmark": { diff --git a/test/otel_collect/ecs/otlp/ecs_resources/ec2_launch/daemon/ecs_taskdef.tpl b/test/otel_collect/ecs/otlp/ecs_resources/ec2_launch/daemon/ecs_taskdef.tpl new file mode 100644 index 000000000..36b5fe321 --- /dev/null +++ b/test/otel_collect/ecs/otlp/ecs_resources/ec2_launch/daemon/ecs_taskdef.tpl @@ -0,0 +1,58 @@ +[ + { + "name": "cloudwatch_agent", + "image": "${cwagent_image}", + "essential": true, + "secrets": [ + { + "name": "CW_CONFIG_CONTENT", + "valueFrom": "${cwagent_ssm_parameter_arn}" + }, + { + "name": "PROMETHEUS_CONFIG_CONTENT", + "valueFrom": "${prometheus_ssm_parameter_arn}" + } + ], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-region": "${region}", + "awslogs-stream-prefix": "${testing_id}", + "awslogs-group": "${log_group}" + } + }, + "cpu": 1, + "memory": 2048 + }, + { + "name": "otlp_pusher", + "image": "curlimages/curl:8.10.1", + "essential": false, + "links": [ + "cloudwatch_agent" + ], + "environment": [ + { + "name": "TEST_ID", + "value": "${testing_id}" + } + ], + "entryPoint": [ + "sh", + "-c" + ], + "command": [ + "while true; do S=$$(date +%s); NOW=$${S}000000000; START=$$((S-10))000000000; curl -s -X POST http://cloudwatch_agent:4318/v1/metrics -H 'Content-Type: application/json' -d '{\"resourceMetrics\":[{\"resource\":{\"attributes\":[{\"key\":\"TestId\",\"value\":{\"stringValue\":\"'\"$${TEST_ID}\"'\"}}]},\"scopeMetrics\":[{\"metrics\":[{\"name\":\"otlp_test_counter\",\"sum\":{\"dataPoints\":[{\"asInt\":\"1\",\"startTimeUnixNano\":\"'\"$${START}\"'\",\"timeUnixNano\":\"'\"$${NOW}\"'\",\"attributes\":[{\"key\":\"TestId\",\"value\":{\"stringValue\":\"'\"$${TEST_ID}\"'\"}}]}],\"aggregationTemporality\":1,\"isMonotonic\":true}},{\"name\":\"otlp_test_gauge\",\"gauge\":{\"dataPoints\":[{\"asDouble\":42.0,\"timeUnixNano\":\"'\"$${NOW}\"'\",\"attributes\":[{\"key\":\"TestId\",\"value\":{\"stringValue\":\"'\"$${TEST_ID}\"'\"}}]}]}}]}]}]}'; sleep 10; done" + ], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-region": "${region}", + "awslogs-stream-prefix": "${testing_id}-pusher", + "awslogs-group": "${log_group}" + } + }, + "cpu": 1, + "memory": 256 + } +] diff --git a/test/otel_collect/ecs/otlp/otlp_ecs_test.go b/test/otel_collect/ecs/otlp/otlp_ecs_test.go new file mode 100644 index 000000000..92741ec65 --- /dev/null +++ b/test/otel_collect/ecs/otlp/otlp_ecs_test.go @@ -0,0 +1,87 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package otlp + +import ( + "strings" + "testing" + "time" + + "github.com/stretchr/testify/suite" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/otlpvalidation" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" +) + +func init() { + environment.RegisterEnvironmentMetaDataFlags() +} + +// clusterNamePrefix matches the ECS cluster naming in terraform/ecs_ec2/daemon. +const clusterNamePrefix = "cwagent-integ-test-cluster-" + +// ECSOtlpTestRunner validates that a workload (sidecar) running on the ECS daemon +// can publish OTLP metrics to the agent's OTLP receiver and have them reach CloudWatch. +type ECSOtlpTestRunner struct { + test_runner.BaseTestRunner +} + +var _ test_runner.ITestRunner = (*ECSOtlpTestRunner)(nil) + +func (t *ECSOtlpTestRunner) GetTestName() string { return "ecs_otlp" } + +// GetAgentConfigFileName returns "" — config is pre-loaded by Terraform via SSM, no restart needed. +func (t *ECSOtlpTestRunner) GetAgentConfigFileName() string { return "" } + +func (t *ECSOtlpTestRunner) GetMeasuredMetrics() []string { + return []string{"otlp_test_counter", "otlp_test_gauge"} +} + +func (t *ECSOtlpTestRunner) Validate() status.TestGroupResult { + env := environment.GetEnvironmentMetaData() + // Give the sidecar time to push and the agent to export before querying. + time.Sleep(3 * time.Minute) + // Isolate by the per-run testing_id, which the sidecar stamps as the TestId + // attribute and is embedded in the ECS cluster ARN. + labels := map[string]string{ + "TestId": testIDFromClusterArn(env.EcsClusterArn), + } + return otlpvalidation.ValidateOtlpMetricsWithLabels(t.GetTestName(), env.Region, t.GetMeasuredMetrics(), labels) +} + +// testIDFromClusterArn extracts the testing_id suffix from the ECS cluster ARN. +func testIDFromClusterArn(clusterArn string) string { + if i := strings.LastIndex(clusterArn, clusterNamePrefix); i != -1 { + return clusterArn[i+len(clusterNamePrefix):] + } + return clusterArn +} + +func TestECSOtlpSuite(t *testing.T) { + suite.Run(t, new(ECSOtlpTestSuite)) +} + +type ECSOtlpTestSuite struct { + suite.Suite + test_runner.TestSuite +} + +func (suite *ECSOtlpTestSuite) GetSuiteName() string { + return "ECSOtlp" +} + +func (suite *ECSOtlpTestSuite) TestAllInSuite() { + env := environment.GetEnvironmentMetaData() + ecsTestRunner := &test_runner.ECSTestRunner{ + Runner: &ECSOtlpTestRunner{}, + RunStrategy: &test_runner.ECSAgentRunStrategy{}, + Env: *env, + } + ecsTestRunner.Run(suite, env) + suite.Assert().Equal(status.SUCCESSFUL, suite.Result.GetStatus(), "ECS OTLP Test Suite Failed") +} diff --git a/test/otel_collect/ecs/otlp/resources/config.json b/test/otel_collect/ecs/otlp/resources/config.json new file mode 100644 index 000000000..c18eeb5a6 --- /dev/null +++ b/test/otel_collect/ecs/otlp/resources/config.json @@ -0,0 +1,14 @@ +{ + "agent": { + "run_as_user": "root" + }, + "opentelemetry": { + "cluster_name": "cwagent-ecs-integ-test", + "collect": { + "otlp": { + "grpc_endpoint": "0.0.0.0:4317", + "http_endpoint": "0.0.0.0:4318" + } + } + } +} diff --git a/test/otel_collect/host_metrics/agent_configs/host_metrics_config.json b/test/otel_collect/host_metrics/agent_configs/host_metrics_config.json deleted file mode 100644 index 7f02358ba..000000000 --- a/test/otel_collect/host_metrics/agent_configs/host_metrics_config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "agent": { - "region": "us-west-2" - }, - "opentelemetry": { - "collect": { - "host_metrics": {} - } - } -} diff --git a/test/otel_collect/linux/host_metrics/agent_configs/host_metrics_config.json b/test/otel_collect/linux/host_metrics/agent_configs/host_metrics_config.json new file mode 100644 index 000000000..13b502179 --- /dev/null +++ b/test/otel_collect/linux/host_metrics/agent_configs/host_metrics_config.json @@ -0,0 +1,13 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "cluster_name": "integ-test-cluster", + "collect": { + "host_metrics": { + "collection_interval": 30 + } + } + } +} diff --git a/test/otel_collect/host_metrics/host_metrics_test.go b/test/otel_collect/linux/host_metrics/host_metrics_test.go similarity index 93% rename from test/otel_collect/host_metrics/host_metrics_test.go rename to test/otel_collect/linux/host_metrics/host_metrics_test.go index 747c5e2b5..ed3da6ec1 100644 --- a/test/otel_collect/host_metrics/host_metrics_test.go +++ b/test/otel_collect/linux/host_metrics/host_metrics_test.go @@ -31,9 +31,8 @@ type HostMetricsTestRunner struct { var _ test_runner.ITestRunner = (*HostMetricsTestRunner)(nil) func (t *HostMetricsTestRunner) Validate() status.TestGroupResult { - return otlpvalidation.ValidateOtlpMetricsWithLabels(t.GetTestName(), t.env.Region, t.GetMeasuredMetrics(), map[string]string{ - "@resource.host.id": t.env.InstanceId, - }) + return otlpvalidation.ValidateOtlpMetricsWithLabels(t.GetTestName(), t.env.Region, t.GetMeasuredMetrics(), + otlpvalidation.ResourceHostIDLabels(t.env.AgentStartCommand, t.env.InstanceId)) } func (t *HostMetricsTestRunner) GetTestName() string { return "HostMetrics" } diff --git a/test/otel_collect/linux/otlp/agent_configs/otlp_config.json b/test/otel_collect/linux/otlp/agent_configs/otlp_config.json new file mode 100644 index 000000000..3edc5c3b0 --- /dev/null +++ b/test/otel_collect/linux/otlp/agent_configs/otlp_config.json @@ -0,0 +1,14 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "cluster_name": "integ-test-cluster", + "collect": { + "otlp": { + "grpc_endpoint": "127.0.0.1:4317", + "http_endpoint": "127.0.0.1:4318" + } + } + } +} diff --git a/test/otel_collect/linux/otlp/otlp_test.go b/test/otel_collect/linux/otlp/otlp_test.go new file mode 100644 index 000000000..48cc311c8 --- /dev/null +++ b/test/otel_collect/linux/otlp/otlp_test.go @@ -0,0 +1,211 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package otlp + +import ( + "bytes" + "context" + "fmt" + "log" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/otlpvalidation" + "github.com/aws/amazon-cloudwatch-agent-test/test/status" + "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" + "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" + "github.com/aws/amazon-cloudwatch-agent-test/util/common" + "github.com/aws/amazon-cloudwatch-agent-test/util/common/traces/base" + "github.com/aws/amazon-cloudwatch-agent-test/util/common/traces/otlp" +) + +func init() { + environment.RegisterEnvironmentMetaDataFlags() +} + +const ( + otlpRuntime = 3 * time.Minute + otlpEndpoint = "http://127.0.0.1:4318" + otlpLogGroup = "/aws/cwagent" + otlpGRPCAddr = "127.0.0.1:4317" + traceTestType = "otel_collect_otlp_traces" + // spansLogGroup is where V2 OTLP traces land (CloudWatch Logs destination). + spansLogGroup = "aws/spans" +) + +type OtlpCollectTestRunner struct { + test_runner.BaseTestRunner + env *environment.MetaData + startedAt time.Time +} + +var _ test_runner.ITestRunner = (*OtlpCollectTestRunner)(nil) + +func (t *OtlpCollectTestRunner) Validate() status.TestGroupResult { + var results []status.TestResult + + // Metrics + metricResult := otlpvalidation.ValidateOtlpMetricsWithLabels(t.GetTestName(), t.env.Region, t.GetMeasuredMetrics(), + otlpvalidation.OtlpMetricLabels(t.env.AgentStartCommand, t.env.InstanceId)) + results = append(results, metricResult.TestResults...) + + // Traces + results = append(results, t.validateTraces()) + + // Logs + results = append(results, t.validateLogs()) + + return status.TestGroupResult{Name: t.GetTestName(), TestResults: results} +} + +// validateTraces confirms this run's OTLP traces landed in aws/spans, filtered by instance_id. +func (t *OtlpCollectTestRunner) validateTraces() status.TestResult { + query := fmt.Sprintf(`fields @message | filter @message like /%s/ | limit 100`, t.env.InstanceId) + start := t.startedAt.Unix() + var err error + for attempt := 0; attempt < 5; attempt++ { + if attempt > 0 { + time.Sleep(30 * time.Second) + } + results, qErr := awsservice.GetLogQueryResults(spansLogGroup, start, time.Now().Unix(), query) + if qErr != nil { + err = qErr + continue + } + log.Printf("[OTLP_Traces] attempt %d: found %d spans in %s for instance %s", attempt+1, len(results), spansLogGroup, t.env.InstanceId) + if len(results) > 0 { + return status.TestResult{Name: "OTLP_Traces", Status: status.SUCCESSFUL} + } + err = fmt.Errorf("no spans found in %s for instance %s", spansLogGroup, t.env.InstanceId) + } + return status.TestResult{Name: "OTLP_Traces", Status: status.FAILED, Reason: err} +} + +// validateLogs checks that OTLP logs from this run are in CloudWatch Logs. +func (t *OtlpCollectTestRunner) validateLogs() status.TestResult { + since := t.startedAt + until := time.Now() + + if len(awsservice.GetLogStreams(otlpLogGroup)) == 0 { + return status.TestResult{Name: "OTLP_Logs", Status: status.FAILED, + Reason: fmt.Errorf("no log streams found in log group %s", otlpLogGroup)} + } + + err := awsservice.ValidateLogs( + otlpLogGroup, + t.env.InstanceId, + &since, + &until, + awsservice.AssertLogsNotEmpty(), + awsservice.AssertPerLog( + awsservice.AssertLogContainsSubstring(fmt.Sprintf("\"InstanceId\":\"%s\"", t.env.InstanceId)), + ), + ) + if err != nil { + return status.TestResult{Name: "OTLP_Logs", Status: status.FAILED, Reason: err} + } + return status.TestResult{Name: "OTLP_Logs", Status: status.SUCCESSFUL} +} + +func (t *OtlpCollectTestRunner) GetTestName() string { return "OtelCollectOTLP" } +func (t *OtlpCollectTestRunner) GetAgentRunDuration() time.Duration { return otlpRuntime } +func (t *OtlpCollectTestRunner) GetAgentConfigFileName() string { return "otlp_config.json" } +func (t *OtlpCollectTestRunner) GetMeasuredMetrics() []string { + return []string{"otlp_test_counter", "otlp_test_gauge"} +} + +func (t *OtlpCollectTestRunner) SetupAfterAgentRun() error { + t.startedAt = time.Now() + go func() { + _ = common.SendOTLPMetrics(otlpEndpoint, t.env.InstanceId, 10*time.Second, otlpRuntime-30*time.Second) + }() + go t.generateTraces() + go t.sendTestLogs() + return nil +} + +// generateTraces waits for gRPC port 4317, then streams traces to the agent. +// The wait is necessary because otlptracegrpc.New connects at construction time. +func (t *OtlpCollectTestRunner) generateTraces() { + if err := common.WaitForTCPPort(otlpGRPCAddr, 2*time.Minute); err != nil { + log.Printf("generateTraces: gRPC port not ready, skipping: %v", err) + return + } + generator := otlp.NewLoadGenerator(&base.TraceGeneratorConfig{ + Interval: 10 * time.Second, + Annotations: map[string]interface{}{ + "test_type": traceTestType, + "instance_id": t.env.InstanceId, + }, + Attributes: []attribute.KeyValue{ + attribute.String("test_type", traceTestType), + attribute.String("instance_id", t.env.InstanceId), + }, + }) + generator.StartSendingTraces(context.Background()) +} + +// sendTestLogs pushes OTLP logs to the agent's OTLP HTTP receiver (/v1/logs). +func (t *OtlpCollectTestRunner) sendTestLogs() { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + timeout := time.After(otlpRuntime - 30*time.Second) + for { + select { + case <-timeout: + return + case <-ticker.C: + payload := buildOtlpLogsPayload(t.env.InstanceId) + req, _ := http.NewRequest("POST", otlpEndpoint+"/v1/logs", bytes.NewReader(payload)) //nolint:errcheck + req.Header.Set("Content-Type", "application/json") + http.DefaultClient.Do(req) //nolint:errcheck + } + } +} + +func buildOtlpLogsPayload(instanceId string) []byte { + now := time.Now().UnixNano() + // Routes to CW Logs via aws.log.group.name / aws.log.stream.name resource attributes. + payload := fmt.Sprintf(`{ + "resourceLogs": [{ + "resource": {"attributes": [ + {"key": "aws.log.group.name", "value": {"stringValue": "%s"}}, + {"key": "aws.log.stream.name", "value": {"stringValue": "%s"}}, + {"key": "InstanceId", "value": {"stringValue": "%s"}} + ]}, + "scopeLogs": [{ + "logRecords": [{ + "timeUnixNano": "%d", + "severityText": "INFO", + "body": {"stringValue": "otlp integration test log"}, + "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}] + }] + }] + }] +}`, otlpLogGroup, instanceId, instanceId, now, instanceId) + return []byte(payload) +} + +func TestOTLPCollect(t *testing.T) { + env := environment.GetEnvironmentMetaData() + + testRunner := &OtlpCollectTestRunner{ + BaseTestRunner: test_runner.BaseTestRunner{}, + env: env, + } + runner := &test_runner.TestRunner{TestRunner: testRunner} + result := runner.Run() + + for _, r := range result.TestResults { + require.Equal(t, status.SUCCESSFUL, r.Status, "%s failed: %v", r.Name, r.Reason) + } +} diff --git a/test/otel_collect/linux/prometheus/agent_configs/prometheus_config.json b/test/otel_collect/linux/prometheus/agent_configs/prometheus_config.json new file mode 100644 index 000000000..725f8452b --- /dev/null +++ b/test/otel_collect/linux/prometheus/agent_configs/prometheus_config.json @@ -0,0 +1,13 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "cluster_name": "integ-test-cluster", + "collect": { + "prometheus": { + "config_path": "/opt/aws/prometheus.yml" + } + } + } +} diff --git a/test/otel_collect/prometheus/prometheus_test.go b/test/otel_collect/linux/prometheus/prometheus_test.go similarity index 77% rename from test/otel_collect/prometheus/prometheus_test.go rename to test/otel_collect/linux/prometheus/prometheus_test.go index 20ec9efb8..abfee8a09 100644 --- a/test/otel_collect/prometheus/prometheus_test.go +++ b/test/otel_collect/linux/prometheus/prometheus_test.go @@ -8,7 +8,6 @@ package prometheus import ( _ "embed" "fmt" - "os" "testing" "time" @@ -41,9 +40,8 @@ type PrometheusOtelTestRunner struct { var _ test_runner.ITestRunner = (*PrometheusOtelTestRunner)(nil) func (t *PrometheusOtelTestRunner) Validate() status.TestGroupResult { - return otlpvalidation.ValidateOtlpMetricsWithLabels(t.GetTestName(), t.env.Region, t.GetMeasuredMetrics(), map[string]string{ - "@resource.host.id": t.env.InstanceId, - }) + return otlpvalidation.ValidateOtlpMetricsWithLabels(t.GetTestName(), t.env.Region, t.GetMeasuredMetrics(), + otlpvalidation.ResourceHostIDLabels(t.env.AgentStartCommand, t.env.InstanceId)) } func (t *PrometheusOtelTestRunner) GetTestName() string { return "OtelCollectPrometheus" } @@ -63,7 +61,7 @@ func (t *PrometheusOtelTestRunner) SetupBeforeAgentRun() error { return err } - // Write prometheus scrape config + // Write the Prometheus scrape config where the agent config's config_path points. commands := []string{ fmt.Sprintf("cat <<'EOF' | sudo tee /opt/aws/prometheus.yml\n%s\nEOF", prometheusScrapeConfig), } @@ -71,20 +69,13 @@ func (t *PrometheusOtelTestRunner) SetupBeforeAgentRun() error { return err } - // Serve fake metrics on port 9100 (same pattern as test/emf_prometheus) - if err := os.WriteFile("/tmp/metrics", []byte(prometheusMetrics), os.ModePerm); err != nil { - return fmt.Errorf("unable to write /tmp/metrics: %w", err) + // Start the cross-platform fake Prometheus exporter on port 9100. + stop, err := common.StartPrometheusFakeServer(9100, prometheusMetrics) + if err != nil { + return fmt.Errorf("failed to start fake prometheus exporter: %w", err) } - commands = []string{ - "sudo python3 -m http.server 9100 --directory /tmp &> /dev/null &", - } - if err := common.RunCommands(commands); err != nil { - return err - } - t.RegisterCleanup(func() error { - return common.RunCommands([]string{"sudo pkill -f 'python3 -m http.server 9100' || true"}) - }) - time.Sleep(2 * time.Second) + t.RegisterCleanup(func() error { stop(); return nil }) + return nil } diff --git a/test/otel_collect/prometheus/resources/prometheus_metrics b/test/otel_collect/linux/prometheus/resources/prometheus_metrics similarity index 100% rename from test/otel_collect/prometheus/resources/prometheus_metrics rename to test/otel_collect/linux/prometheus/resources/prometheus_metrics diff --git a/test/otel_collect/linux/prometheus/resources/prometheus_scrape_config.yaml b/test/otel_collect/linux/prometheus/resources/prometheus_scrape_config.yaml new file mode 100644 index 000000000..9d7a35783 --- /dev/null +++ b/test/otel_collect/linux/prometheus/resources/prometheus_scrape_config.yaml @@ -0,0 +1,13 @@ +scrape_configs: + - job_name: node + static_configs: + - targets: ['localhost:9100'] + relabel_configs: + - source_labels: [job] + regex: (.*) + target_label: renamed_job + replacement: $1 + - source_labels: [instance] + regex: (.*) + target_label: host + replacement: ${1} diff --git a/test/otel_collect/otlp/agent_configs/otlp_config.json b/test/otel_collect/otlp/agent_configs/otlp_config.json deleted file mode 100644 index 85a839339..000000000 --- a/test/otel_collect/otlp/agent_configs/otlp_config.json +++ /dev/null @@ -1 +0,0 @@ -{"agent":{"region":"us-west-2"},"opentelemetry":{"collect":{"otlp":{"grpc_endpoint":"127.0.0.1:4317","http_endpoint":"127.0.0.1:4318"}}}} diff --git a/test/otel_collect/otlp/otlp_test.go b/test/otel_collect/otlp/otlp_test.go deleted file mode 100644 index bb7dd73aa..000000000 --- a/test/otel_collect/otlp/otlp_test.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -//go:build !windows - -package otlp - -import ( - "bytes" - "fmt" - "net/http" - "testing" - "time" - - "github.com/stretchr/testify/require" - - "github.com/aws/amazon-cloudwatch-agent-test/environment" - "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/otlpvalidation" - "github.com/aws/amazon-cloudwatch-agent-test/test/status" - "github.com/aws/amazon-cloudwatch-agent-test/test/test_runner" -) - -func init() { - environment.RegisterEnvironmentMetaDataFlags() -} - -const ( - otlpRuntime = 3 * time.Minute - otlpEndpoint = "http://127.0.0.1:4318" -) - -type OtlpCollectTestRunner struct { - test_runner.BaseTestRunner - env *environment.MetaData -} - -var _ test_runner.ITestRunner = (*OtlpCollectTestRunner)(nil) - -func (t *OtlpCollectTestRunner) Validate() status.TestGroupResult { - return otlpvalidation.ValidateOtlpMetricsWithLabels(t.GetTestName(), t.env.Region, t.GetMeasuredMetrics(), map[string]string{ - "@resource.host.id": t.env.InstanceId, - }) -} - -func (t *OtlpCollectTestRunner) GetTestName() string { return "OtelCollectOTLP" } -func (t *OtlpCollectTestRunner) GetAgentRunDuration() time.Duration { return otlpRuntime } -func (t *OtlpCollectTestRunner) GetAgentConfigFileName() string { return "otlp_config.json" } -func (t *OtlpCollectTestRunner) GetMeasuredMetrics() []string { - return []string{"otlp_test_counter", "otlp_test_gauge"} -} - -func (t *OtlpCollectTestRunner) SetupAfterAgentRun() error { - go t.sendTestMetrics() - return nil -} - -func (t *OtlpCollectTestRunner) sendTestMetrics() { - ticker := time.NewTicker(10 * time.Second) - defer ticker.Stop() - - timeout := time.After(otlpRuntime - 30*time.Second) - for { - select { - case <-timeout: - return - case <-ticker.C: - payload := buildOtlpMetricsPayload(t.env.InstanceId) - req, _ := http.NewRequest("POST", otlpEndpoint+"/v1/metrics", bytes.NewReader(payload)) - req.Header.Set("Content-Type", "application/json") - http.DefaultClient.Do(req) //nolint:errcheck - } - } -} - -var startTime = time.Now().UnixNano() - -func buildOtlpMetricsPayload(instanceId string) []byte { - now := time.Now().UnixNano() - payload := fmt.Sprintf(`{ - "resourceMetrics": [{ - "resource": {"attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}, - "scopeMetrics": [{ - "metrics": [ - { - "name": "otlp_test_counter", - "sum": { - "dataPoints": [{"asInt": "1", "startTimeUnixNano": "%d", "timeUnixNano": "%d", "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}], - "isMonotonic": true, - "aggregationTemporality": 2 - } - }, - { - "name": "otlp_test_gauge", - "gauge": { - "dataPoints": [{"asDouble": 42.0, "timeUnixNano": "%d", "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}] - } - } - ] - }] - }] -}`, instanceId, startTime, now, instanceId, now, instanceId) - return []byte(payload) -} - -func TestOTLPCollect(t *testing.T) { - env := environment.GetEnvironmentMetaData() - - testRunner := &OtlpCollectTestRunner{ - BaseTestRunner: test_runner.BaseTestRunner{}, - env: env, - } - runner := &test_runner.TestRunner{TestRunner: testRunner} - result := runner.Run() - - for _, r := range result.TestResults { - require.Equal(t, status.SUCCESSFUL, r.Status, "%s failed: %v", r.Name, r.Reason) - } -} diff --git a/test/otel_collect/otlpvalidation/labels.go b/test/otel_collect/otlpvalidation/labels.go new file mode 100644 index 000000000..059c7f590 --- /dev/null +++ b/test/otel_collect/otlpvalidation/labels.go @@ -0,0 +1,44 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otlpvalidation + +import ( + "os" + "strings" +) + +// onPremiseMarker identifies the on-prem agent start command (-m onPremise). +const onPremiseMarker = "onPremise" + +// ResourceHostIDLabels returns the label filter for host_metrics/prometheus tests. +// EC2 filters on host.id; on-prem returns nil (no IMDS, validate presence only). +func ResourceHostIDLabels(agentStartCommand, instanceID string) map[string]string { + if strings.Contains(agentStartCommand, onPremiseMarker) { + return nil + } + return map[string]string{"@resource.host.id": instanceID} +} + +// OtlpMetricLabels returns the label filter for the OTLP test. +// EC2 uses host.id; on-prem uses the injected InstanceId attribute. +func OtlpMetricLabels(agentStartCommand, instanceID string) map[string]string { + if strings.Contains(agentStartCommand, onPremiseMarker) { + return map[string]string{"InstanceId": instanceID} + } + return map[string]string{"@resource.host.id": instanceID} +} + +// getRegion returns the region, falling back to AWS env vars then us-west-2. +func getRegion(region string) string { + if region != "" { + return region + } + if r := os.Getenv("AWS_REGION"); r != "" { + return r + } + if r := os.Getenv("AWS_DEFAULT_REGION"); r != "" { + return r + } + return "us-west-2" +} diff --git a/test/otel_collect/otlpvalidation/validate.go b/test/otel_collect/otlpvalidation/validate.go index b4972d609..90cfb4faa 100644 --- a/test/otel_collect/otlpvalidation/validate.go +++ b/test/otel_collect/otlpvalidation/validate.go @@ -9,7 +9,6 @@ import ( "context" "fmt" "log" - "os" "time" "github.com/aws/amazon-cloudwatch-agent-test/test/status" @@ -21,19 +20,6 @@ const ( defaultRetryInterval = 30 * time.Second ) -func getRegion(region string) string { - if region != "" { - return region - } - if r := os.Getenv("AWS_REGION"); r != "" { - return r - } - if r := os.Getenv("AWS_DEFAULT_REGION"); r != "" { - return r - } - return "us-west-2" -} - func ValidateOtlpMetrics(testName string, region string, metrics []string) status.TestGroupResult { return ValidateOtlpMetricsWithLabels(testName, region, metrics, nil) } diff --git a/test/otel_collect/prometheus/agent_configs/prometheus_config.json b/test/otel_collect/prometheus/agent_configs/prometheus_config.json deleted file mode 100644 index 1de36ecf0..000000000 --- a/test/otel_collect/prometheus/agent_configs/prometheus_config.json +++ /dev/null @@ -1 +0,0 @@ -{"agent":{"region":"us-west-2"},"opentelemetry":{"collect":{"prometheus":{"config_path":"/opt/aws/prometheus.yml"}}}} diff --git a/test/otel_collect/prometheus/resources/prometheus_scrape_config.yaml b/test/otel_collect/prometheus/resources/prometheus_scrape_config.yaml deleted file mode 100644 index 088eade54..000000000 --- a/test/otel_collect/prometheus/resources/prometheus_scrape_config.yaml +++ /dev/null @@ -1,4 +0,0 @@ -scrape_configs: - - job_name: node - static_configs: - - targets: ['localhost:9100'] diff --git a/test/otel_collect/windows/host_metrics/agent_config.json b/test/otel_collect/windows/host_metrics/agent_config.json new file mode 100644 index 000000000..dbb307338 --- /dev/null +++ b/test/otel_collect/windows/host_metrics/agent_config.json @@ -0,0 +1,5 @@ +{ + "agent": { + "debug": true + } +} diff --git a/test/otel_collect/windows/host_metrics/host_metrics_unix.go b/test/otel_collect/windows/host_metrics/host_metrics_unix.go new file mode 100644 index 000000000..657a964be --- /dev/null +++ b/test/otel_collect/windows/host_metrics/host_metrics_unix.go @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package host_metrics + +import "errors" + +// Validate is only implemented on Windows; this stub lets the validator build on other platforms. +func Validate() error { + return errors.New("otel_collect windows host_metrics validation is only supported on Windows") +} diff --git a/test/otel_collect/windows/host_metrics/host_metrics_windows.go b/test/otel_collect/windows/host_metrics/host_metrics_windows.go new file mode 100644 index 000000000..ec9abd0e1 --- /dev/null +++ b/test/otel_collect/windows/host_metrics/host_metrics_windows.go @@ -0,0 +1,52 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build windows + +package host_metrics + +import ( + "context" + _ "embed" + "fmt" + "os" + "time" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/otlpvalidation" + "github.com/aws/amazon-cloudwatch-agent-test/util/common" + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +//go:embed resources/config.json +var testConfigJSON string + +const ( + tmpConfigPath = "C:\\Users\\Administrator\\AppData\\Local\\Temp\\config.json" + runtime = 3 * time.Minute +) + +func Validate() error { + env := environment.GetEnvironmentMetaData() + + if err := os.WriteFile(tmpConfigPath, []byte(testConfigJSON), 0644); err != nil { + return fmt.Errorf("could not write config: %w", err) + } + if err := common.CopyFile(tmpConfigPath, common.ConfigOutputPath); err != nil { + return fmt.Errorf("could not copy config: %w", err) + } + if err := common.StartAgent(common.ConfigOutputPath, true, false); err != nil { + return fmt.Errorf("could not start agent: %w", err) + } + time.Sleep(runtime) + _ = common.StopAgent() + + return otelmetrics.AssertMetricsPresent( + context.Background(), + env.Region, + []string{"system.cpu.utilization", "system.memory.utilization", "system.network.io", "system.disk.operations"}, + otlpvalidation.ResourceHostIDLabels(env.AgentStartCommand, env.InstanceId), + 3, + 30*time.Second, + ) +} diff --git a/test/otel_collect/windows/host_metrics/parameters.yml b/test/otel_collect/windows/host_metrics/parameters.yml new file mode 100644 index 000000000..f24e66cbe --- /dev/null +++ b/test/otel_collect/windows/host_metrics/parameters.yml @@ -0,0 +1,14 @@ +# FILE ONLY EXISTS SO VALIDATOR WILL RUN THE GO TEST +# SEE validator/main.go + +receivers: [] +test_case: "win_otel_host_metrics" +validate_type: "feature" +data_type: "metrics" +number_monitored_logs: 1 +values_per_minute: "2" +agent_collection_period: 60 +cloudwatch_agent_config: "" +metric_namespace: "CloudWatchAgentWinOtel" +metric_validation: +log_validation: diff --git a/test/otel_collect/windows/host_metrics/resources/config.json b/test/otel_collect/windows/host_metrics/resources/config.json new file mode 100644 index 000000000..2f2d0212b --- /dev/null +++ b/test/otel_collect/windows/host_metrics/resources/config.json @@ -0,0 +1,13 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "cluster_name": "cwagent-windows-integ-test", + "collect": { + "host_metrics": { + "collection_interval": 30 + } + } + } +} diff --git a/test/otel_collect/windows/otlp/agent_config.json b/test/otel_collect/windows/otlp/agent_config.json new file mode 100644 index 000000000..dbb307338 --- /dev/null +++ b/test/otel_collect/windows/otlp/agent_config.json @@ -0,0 +1,5 @@ +{ + "agent": { + "debug": true + } +} diff --git a/test/otel_collect/windows/otlp/otlp_unix.go b/test/otel_collect/windows/otlp/otlp_unix.go new file mode 100644 index 000000000..f9b44d7ac --- /dev/null +++ b/test/otel_collect/windows/otlp/otlp_unix.go @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package otlp + +import "errors" + +// Validate is only implemented on Windows; this stub lets the validator build on other platforms. +func Validate() error { + return errors.New("otel_collect windows OTLP validation is only supported on Windows") +} diff --git a/test/otel_collect/windows/otlp/otlp_windows.go b/test/otel_collect/windows/otlp/otlp_windows.go new file mode 100644 index 000000000..66b14ec33 --- /dev/null +++ b/test/otel_collect/windows/otlp/otlp_windows.go @@ -0,0 +1,192 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build windows + +package otlp + +import ( + "bytes" + "context" + _ "embed" + "fmt" + "log" + "net/http" + "os" + "time" + + "go.opentelemetry.io/otel/attribute" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/otlpvalidation" + "github.com/aws/amazon-cloudwatch-agent-test/util/awsservice" + "github.com/aws/amazon-cloudwatch-agent-test/util/common" + "github.com/aws/amazon-cloudwatch-agent-test/util/common/traces/base" + otlptraces "github.com/aws/amazon-cloudwatch-agent-test/util/common/traces/otlp" + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +//go:embed resources/config.json +var testConfigJSON string + +const ( + tmpConfigPath = "C:\\Users\\Administrator\\AppData\\Local\\Temp\\config.json" + otlpRuntime = 3 * time.Minute + sendInterval = 10 * time.Second + otlpEndpoint = "http://127.0.0.1:4318" + otlpGRPCAddr = "127.0.0.1:4317" + traceTestType = "otel_collect_windows_otlp_traces" + otlpLogGroup = "/aws/cwagent" + // spansLogGroup is where V2 OTLP traces land (CloudWatch Logs destination). + spansLogGroup = "aws/spans" +) + +func Validate() error { + env := environment.GetEnvironmentMetaData() + + if err := os.WriteFile(tmpConfigPath, []byte(testConfigJSON), 0644); err != nil { + return fmt.Errorf("could not write config: %w", err) + } + if err := common.CopyFile(tmpConfigPath, common.ConfigOutputPath); err != nil { + return fmt.Errorf("could not copy config: %w", err) + } + if err := common.StartAgent(common.ConfigOutputPath, true, false); err != nil { + return fmt.Errorf("could not start agent: %w", err) + } + // Wait for HTTP receiver before sending data. + if err := common.WaitForOTLPEndpoint(otlpEndpoint, 2*time.Minute); err != nil { + return fmt.Errorf("OTLP endpoint not ready: %w", err) + } + + // Send metrics, logs, and traces concurrently. + startedAt := time.Now() + go func() { + _ = common.SendOTLPMetrics(otlpEndpoint, env.InstanceId, sendInterval, otlpRuntime) + }() + go sendLogs(env.InstanceId) + go generateTraces(env.InstanceId) + + time.Sleep(otlpRuntime) + _ = common.StopAgent() + + // Validate metrics. + if err := otelmetrics.AssertMetricsPresent( + context.Background(), + env.Region, + []string{"otlp_test_counter", "otlp_test_gauge"}, + otlpvalidation.OtlpMetricLabels(env.AgentStartCommand, env.InstanceId), + 3, + 30*time.Second, + ); err != nil { + return fmt.Errorf("metrics validation failed: %w", err) + } + + // Validate traces. + if err := validateTraces(startedAt, env.InstanceId); err != nil { + return fmt.Errorf("traces validation failed: %w", err) + } + + // Validate logs. + if err := validateLogs(startedAt, env.InstanceId); err != nil { + return fmt.Errorf("logs validation failed: %w", err) + } + + return nil +} + +func generateTraces(instanceID string) { + if err := common.WaitForTCPPort(otlpGRPCAddr, 2*time.Minute); err != nil { + log.Printf("generateTraces: gRPC port not ready, skipping: %v", err) + return + } + generator := otlptraces.NewLoadGenerator(&base.TraceGeneratorConfig{ + Interval: 10 * time.Second, + Annotations: map[string]interface{}{ + "test_type": traceTestType, + "instance_id": instanceID, + }, + Attributes: []attribute.KeyValue{ + attribute.String("test_type", traceTestType), + attribute.String("instance_id", instanceID), + }, + }) + generator.StartSendingTraces(context.Background()) +} + +func validateTraces(startedAt time.Time, instanceID string) error { + query := fmt.Sprintf(`fields @message | filter @message like /%s/ | limit 100`, instanceID) + start := startedAt.Unix() + var err error + for attempt := 0; attempt < 5; attempt++ { + if attempt > 0 { + time.Sleep(30 * time.Second) + } + results, qErr := awsservice.GetLogQueryResults(spansLogGroup, start, time.Now().Unix(), query) + if qErr != nil { + err = qErr + continue + } + log.Printf("[OTLP_Traces] attempt %d: found %d spans in %s for instance %s", attempt+1, len(results), spansLogGroup, instanceID) + if len(results) > 0 { + return nil + } + err = fmt.Errorf("no spans found in %s for instance %s", spansLogGroup, instanceID) + } + return err +} + +func sendLogs(instanceID string) { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + timeout := time.After(otlpRuntime - 30*time.Second) + for { + select { + case <-timeout: + return + case <-ticker.C: + payload := buildLogsPayload(instanceID) + req, _ := http.NewRequest("POST", otlpEndpoint+"/v1/logs", bytes.NewReader(payload)) //nolint:errcheck + req.Header.Set("Content-Type", "application/json") + http.DefaultClient.Do(req) //nolint:errcheck + } + } +} + +func validateLogs(startedAt time.Time, instanceID string) error { + since := startedAt + until := time.Now() + if len(awsservice.GetLogStreams(otlpLogGroup)) == 0 { + return fmt.Errorf("no log streams found in %s", otlpLogGroup) + } + return awsservice.ValidateLogs( + otlpLogGroup, + instanceID, + &since, + &until, + awsservice.AssertLogsNotEmpty(), + awsservice.AssertPerLog( + awsservice.AssertLogContainsSubstring(fmt.Sprintf("\"InstanceId\":\"%s\"", instanceID)), + ), + ) +} + +func buildLogsPayload(instanceID string) []byte { + now := time.Now().UnixNano() + return []byte(fmt.Sprintf(`{ + "resourceLogs": [{ + "resource": {"attributes": [ + {"key": "aws.log.group.name", "value": {"stringValue": "%s"}}, + {"key": "aws.log.stream.name", "value": {"stringValue": "%s"}}, + {"key": "InstanceId", "value": {"stringValue": "%s"}} + ]}, + "scopeLogs": [{ + "logRecords": [{ + "timeUnixNano": "%d", + "severityText": "INFO", + "body": {"stringValue": "otlp windows integration test log"}, + "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}] + }] + }] + }] +}`, otlpLogGroup, instanceID, instanceID, now, instanceID)) +} diff --git a/test/otel_collect/windows/otlp/parameters.yml b/test/otel_collect/windows/otlp/parameters.yml new file mode 100644 index 000000000..1ddb28af2 --- /dev/null +++ b/test/otel_collect/windows/otlp/parameters.yml @@ -0,0 +1,17 @@ +# FILE ONLY EXISTS SO VALIDATOR WILL RUN THE GO TEST +# SEE validator/main.go + +# Receivers that agent needs to test +receivers: [] + +# Test case name +test_case: "win_otlp" +validate_type: "feature" +data_type: "metrics" +number_monitored_logs: 1 +values_per_minute: "2" +agent_collection_period: 60 +cloudwatch_agent_config: "" +metric_namespace: "CloudWatchAgentWinOtel" +metric_validation: +log_validation: diff --git a/test/otel_collect/windows/otlp/resources/config.json b/test/otel_collect/windows/otlp/resources/config.json new file mode 100644 index 000000000..0cca9afba --- /dev/null +++ b/test/otel_collect/windows/otlp/resources/config.json @@ -0,0 +1,14 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "cluster_name": "cwagent-windows-integ-test", + "collect": { + "otlp": { + "grpc_endpoint": "127.0.0.1:4317", + "http_endpoint": "127.0.0.1:4318" + } + } + } +} diff --git a/test/otel_collect/windows/prometheus/agent_config.json b/test/otel_collect/windows/prometheus/agent_config.json new file mode 100644 index 000000000..dbb307338 --- /dev/null +++ b/test/otel_collect/windows/prometheus/agent_config.json @@ -0,0 +1,5 @@ +{ + "agent": { + "debug": true + } +} diff --git a/test/otel_collect/windows/prometheus/parameters.yml b/test/otel_collect/windows/prometheus/parameters.yml new file mode 100644 index 000000000..77b736bed --- /dev/null +++ b/test/otel_collect/windows/prometheus/parameters.yml @@ -0,0 +1,14 @@ +# FILE ONLY EXISTS SO VALIDATOR WILL RUN THE GO TEST +# SEE validator/main.go + +receivers: [] +test_case: "win_otel_prometheus" +validate_type: "feature" +data_type: "metrics" +number_monitored_logs: 1 +values_per_minute: "2" +agent_collection_period: 60 +cloudwatch_agent_config: "" +metric_namespace: "CloudWatchAgentWinOtel" +metric_validation: +log_validation: diff --git a/test/otel_collect/windows/prometheus/prometheus_unix.go b/test/otel_collect/windows/prometheus/prometheus_unix.go new file mode 100644 index 000000000..e9d8d7ae5 --- /dev/null +++ b/test/otel_collect/windows/prometheus/prometheus_unix.go @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build !windows + +package prometheus + +import "errors" + +// Validate is only implemented on Windows; this stub lets the validator build on other platforms. +func Validate() error { + return errors.New("otel_collect windows prometheus validation is only supported on Windows") +} diff --git a/test/otel_collect/windows/prometheus/prometheus_windows.go b/test/otel_collect/windows/prometheus/prometheus_windows.go new file mode 100644 index 000000000..b57b21774 --- /dev/null +++ b/test/otel_collect/windows/prometheus/prometheus_windows.go @@ -0,0 +1,69 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build windows + +package prometheus + +import ( + "context" + _ "embed" + "fmt" + "os" + "time" + + "github.com/aws/amazon-cloudwatch-agent-test/environment" + "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/otlpvalidation" + "github.com/aws/amazon-cloudwatch-agent-test/util/common" + "github.com/aws/amazon-cloudwatch-agent-test/util/otelmetrics" +) + +//go:embed resources/config.json +var testConfigJSON string + +//go:embed resources/prometheus_scrape.yaml +var scrapeConfig string + +//go:embed resources/prometheus_metrics +var prometheusMetrics string + +const ( + tmpConfigPath = "C:\\Users\\Administrator\\AppData\\Local\\Temp\\config.json" + scrapePath = "C:\\prometheus_scrape.yaml" + exporterPort = 8101 + runtime = 3 * time.Minute +) + +func Validate() error { + env := environment.GetEnvironmentMetaData() + + stop, err := common.StartPrometheusFakeServer(exporterPort, prometheusMetrics) + if err != nil { + return fmt.Errorf("could not start fake prometheus exporter: %w", err) + } + defer stop() + + if err := os.WriteFile(scrapePath, []byte(scrapeConfig), 0644); err != nil { + return fmt.Errorf("could not write scrape config: %w", err) + } + if err := os.WriteFile(tmpConfigPath, []byte(testConfigJSON), 0644); err != nil { + return fmt.Errorf("could not write agent config: %w", err) + } + if err := common.CopyFile(tmpConfigPath, common.ConfigOutputPath); err != nil { + return fmt.Errorf("could not copy config: %w", err) + } + if err := common.StartAgent(common.ConfigOutputPath, true, false); err != nil { + return fmt.Errorf("could not start agent: %w", err) + } + time.Sleep(runtime) + _ = common.StopAgent() + + return otelmetrics.AssertMetricsPresent( + context.Background(), + env.Region, + []string{"node_cpu_seconds_total", "node_memory_MemAvailable_bytes"}, + otlpvalidation.ResourceHostIDLabels(env.AgentStartCommand, env.InstanceId), + 3, + 30*time.Second, + ) +} diff --git a/test/otel_collect/windows/prometheus/resources/config.json b/test/otel_collect/windows/prometheus/resources/config.json new file mode 100644 index 000000000..8bc6d04b4 --- /dev/null +++ b/test/otel_collect/windows/prometheus/resources/config.json @@ -0,0 +1,13 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "cluster_name": "cwagent-windows-integ-test", + "collect": { + "prometheus": { + "config_path": "C:\\prometheus_scrape.yaml" + } + } + } +} diff --git a/test/otel_collect/windows/prometheus/resources/prometheus_metrics b/test/otel_collect/windows/prometheus/resources/prometheus_metrics new file mode 100644 index 000000000..9ce5127f1 --- /dev/null +++ b/test/otel_collect/windows/prometheus/resources/prometheus_metrics @@ -0,0 +1,7 @@ +# HELP node_cpu_seconds_total Seconds the CPUs spent in each mode. +# TYPE node_cpu_seconds_total counter +node_cpu_seconds_total{cpu="0",mode="idle"} 12345.67 +node_cpu_seconds_total{cpu="0",mode="system"} 234.56 +# HELP node_memory_MemAvailable_bytes Memory information field MemAvailable_bytes. +# TYPE node_memory_MemAvailable_bytes gauge +node_memory_MemAvailable_bytes 4123456789 diff --git a/test/otel_collect/windows/prometheus/resources/prometheus_scrape.yaml b/test/otel_collect/windows/prometheus/resources/prometheus_scrape.yaml new file mode 100644 index 000000000..efd6350eb --- /dev/null +++ b/test/otel_collect/windows/prometheus/resources/prometheus_scrape.yaml @@ -0,0 +1,14 @@ +scrape_configs: + - job_name: node + scrape_interval: 10s + static_configs: + - targets: ['localhost:8101'] + relabel_configs: + - source_labels: [job] + regex: (.*) + target_label: renamed_job + replacement: $1 + - source_labels: [instance] + regex: (.*) + target_label: host + replacement: ${1} diff --git a/util/common/metrics.go b/util/common/metrics.go index e553d8574..38c43b334 100644 --- a/util/common/metrics.go +++ b/util/common/metrics.go @@ -75,6 +75,9 @@ func StartSendingMetrics(receiver string, duration, sendingInterval time.Duratio err = SendPrometheusMetrics(cfg, duration) case "traces": err = SendAppSignalsTraceMetrics(duration) //does app signals have dimension for metric? + case "otlp": + // metricLogGroup is the instance id (set by the feature validator's GenerateLoad). + err = SendOTLPMetrics(DefaultOTLPHTTPEndpoint, metricLogGroup, sendingInterval, duration) default: } diff --git a/util/common/otlp.go b/util/common/otlp.go new file mode 100644 index 000000000..402ae5c8b --- /dev/null +++ b/util/common/otlp.go @@ -0,0 +1,130 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package common + +import ( + "bytes" + "fmt" + "log" + "net" + "net/http" + "time" +) + +// DefaultOTLPHTTPEndpoint is the agent's default OTLP/HTTP receiver endpoint. +const DefaultOTLPHTTPEndpoint = "http://127.0.0.1:4318" + +// WaitForOTLPEndpoint polls the OTLP HTTP endpoint until it responds or timeout elapses. +// Use after StartAgent instead of a fixed sleep. +func WaitForOTLPEndpoint(endpoint string, timeout time.Duration) error { + if endpoint == "" { + endpoint = DefaultOTLPHTTPEndpoint + } + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + resp, err := http.Get(endpoint) //nolint:noctx + if err == nil { + resp.Body.Close() + return nil + } + time.Sleep(2 * time.Second) + } + return fmt.Errorf("OTLP endpoint %s not ready after %s", endpoint, timeout) +} + +// WaitForTCPPort dials addr until it accepts a connection or timeout elapses. +// Use before creating an otlptracegrpc exporter, which connects immediately. +func WaitForTCPPort(addr string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("tcp", addr, 2*time.Second) + if err == nil { + conn.Close() + return nil + } + time.Sleep(2 * time.Second) + } + return fmt.Errorf("TCP port %s not ready after %s", addr, timeout) +} + +// SendOTLPMetrics pushes OTLP metrics to the agent's HTTP receiver until duration elapses. +func SendOTLPMetrics(endpoint, instanceID string, sendingInterval, duration time.Duration) error { + if endpoint == "" { + endpoint = DefaultOTLPHTTPEndpoint + } + deadline := time.Now().Add(duration) + ticker := time.NewTicker(sendingInterval) + defer ticker.Stop() + + for { + payload := buildOTLPMetricsPayload(instanceID) + req, err := http.NewRequest(http.MethodPost, endpoint+"/v1/metrics", bytes.NewReader(payload)) + if err != nil { + return fmt.Errorf("failed to build OTLP metrics request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + if resp, err := http.DefaultClient.Do(req); err != nil { + log.Printf("SendOTLPMetrics: post failed: %v", err) + } else { + resp.Body.Close() + } + + if time.Now().After(deadline) { + return nil + } + <-ticker.C + } +} + +// buildOTLPMetricsPayload builds a delta counter + gauge payload tagged with instanceID. +func buildOTLPMetricsPayload(instanceID string) []byte { + now := time.Now().UnixNano() + start := now - int64(10*time.Second) + return []byte(fmt.Sprintf(`{ + "resourceMetrics": [{ + "resource": {"attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}, + "scopeMetrics": [{ + "metrics": [ + { + "name": "otlp_test_counter", + "sum": { + "dataPoints": [{"asInt": "1", "startTimeUnixNano": "%d", "timeUnixNano": "%d", "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}], + "isMonotonic": true, + "aggregationTemporality": 1 + } + }, + { + "name": "otlp_test_gauge", + "gauge": { + "dataPoints": [{"asDouble": 42.0, "timeUnixNano": "%d", "attributes": [{"key": "InstanceId", "value": {"stringValue": "%s"}}]}] + } + } + ] + }] + }] +}`, instanceID, start, now, instanceID, now, instanceID)) +} + +// StartPrometheusFakeServer serves static Prometheus metrics on the given port until stop() is called. +func StartPrometheusFakeServer(port int, exposition string) (stop func(), err error) { + mux := http.NewServeMux() + mux.HandleFunc("/metrics", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/plain; version=0.0.4") + _, _ = w.Write([]byte(exposition)) + }) + + ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + return nil, fmt.Errorf("failed to listen on port %d: %w", port, err) + } + + srv := &http.Server{Handler: mux} + go func() { + if serveErr := srv.Serve(ln); serveErr != nil && serveErr != http.ErrServerClosed { + log.Printf("StartPrometheusFakeServer: serve stopped: %v", serveErr) + } + }() + + return func() { _ = srv.Close() }, nil +} diff --git a/util/common/traces/otlp/generator.go b/util/common/traces/otlp/generator.go index 98e41eca9..487f105b6 100644 --- a/util/common/traces/otlp/generator.go +++ b/util/common/traces/otlp/generator.go @@ -125,7 +125,8 @@ func setupClient(ctx context.Context) (*sdktrace.TracerProvider, func(context.Co } func setupTraceProvider(ctx context.Context, res *resource.Resource) (*sdktrace.TracerProvider, error) { - exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithInsecure()) + // Use 127.0.0.1 (not "localhost", which can resolve to IPv6 [::1]) since the agent binds IPv4. + exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithInsecure(), otlptracegrpc.WithEndpoint("127.0.0.1:4317")) if err != nil { return nil, err } diff --git a/util/otelmetrics/validate.go b/util/otelmetrics/validate.go new file mode 100644 index 000000000..d956d3bdc --- /dev/null +++ b/util/otelmetrics/validate.go @@ -0,0 +1,73 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package otelmetrics + +import ( + "context" + "fmt" + "os" + "strings" + "time" +) + +// resolveRegion returns the region, falling back to AWS env vars then us-west-2. +func resolveRegion(region string) string { + if region != "" { + return region + } + if r := os.Getenv("AWS_REGION"); r != "" { + return r + } + if r := os.Getenv("AWS_DEFAULT_REGION"); r != "" { + return r + } + return "us-west-2" +} + +// AssertMetricsPresent queries the CloudWatch OTLP PromQL endpoint and returns an error if any +// metric is missing after retries. Cross-platform (returns error, no test/status dependency). +func AssertMetricsPresent(ctx context.Context, region string, metrics []string, labels map[string]string, maxRetries int, retryInterval time.Duration) error { + region = resolveRegion(region) + client, err := NewClient(ctx, TestConfig{ + Region: region, + Endpoint: fmt.Sprintf("https://monitoring.%s.amazonaws.com", region), + Timeout: 30 * time.Second, + MaxRetries: 3, + SigningService: "monitoring", + }) + if err != nil { + return fmt.Errorf("failed to create otel metrics client: %w", err) + } + + validated := make(map[string]bool, len(metrics)) + for attempt := 0; attempt < maxRetries; attempt++ { + if attempt > 0 { + time.Sleep(retryInterval) + } + for _, m := range metrics { + if validated[m] { + continue + } + query := fmt.Sprintf(`{__name__="%s"`, m) + for k, v := range labels { + query += fmt.Sprintf(`, "%s"=~"%s"`, k, v) + } + query += "}" + if results, err := client.Query(ctx, query); err == nil && len(results) > 0 { + validated[m] = true + } + } + if len(validated) == len(metrics) { + return nil + } + } + + var missing []string + for _, m := range metrics { + if !validated[m] { + missing = append(missing, m) + } + } + return fmt.Errorf("metrics not found after %d attempts: %s", maxRetries, strings.Join(missing, ", ")) +} diff --git a/validator/main.go b/validator/main.go index 9727b72da..339387f44 100644 --- a/validator/main.go +++ b/validator/main.go @@ -18,6 +18,9 @@ import ( "github.com/aws/amazon-cloudwatch-agent-test/test/log_state/logfile" "github.com/aws/amazon-cloudwatch-agent-test/test/log_state/windows_event_log" "github.com/aws/amazon-cloudwatch-agent-test/test/nvidia_gpu" + otelhostmetrics "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/windows/host_metrics" + otelcollectotlp "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/windows/otlp" + otelprometheus "github.com/aws/amazon-cloudwatch-agent-test/test/otel_collect/windows/prometheus" "github.com/aws/amazon-cloudwatch-agent-test/test/restart" "github.com/aws/amazon-cloudwatch-agent-test/test/ssm_document" "github.com/aws/amazon-cloudwatch-agent-test/test/workload_discovery" @@ -67,6 +70,12 @@ func main() { err = windows_event_log.Validate() case "ssm_document": err = ssm_document.Validate() + case "otlp": + err = otelcollectotlp.Validate() + case "host_metrics": + err = otelhostmetrics.Validate() + case "prometheus": + err = otelprometheus.Validate() default: err = errors.New("unsupported test name") } diff --git a/validator/models/validation_config.go b/validator/models/validation_config.go index 7294cf27c..067b15855 100644 --- a/validator/models/validation_config.go +++ b/validator/models/validation_config.go @@ -15,7 +15,7 @@ import ( "gopkg.in/yaml.v3" ) -var supportedReceivers = []string{"logs", "statsd", "collectd", "system", "emf", "xray", "app_signals", "prometheus", "traces"} +var supportedReceivers = []string{"logs", "statsd", "collectd", "system", "emf", "xray", "app_signals", "prometheus", "traces", "otlp"} var retryCount = 0 type ValidateConfig interface {