From 68b1aa36fba333120d49e4440256b499639f2fa6 Mon Sep 17 00:00:00 2001 From: consulthys Date: Thu, 3 Sep 2026 08:11:07 +0200 Subject: [PATCH 1/3] [elasticsearch] Fix audit event.outcome mapping Stop defaulting unmatched audit actions to failure so successful security_config_change events are not labeled as failures, and keep the target user on destination.user. Co-authored-by: Cursor --- packages/elasticsearch/changelog.yml | 5 + .../test-audit-logs.log-expected.json | 12 +- .../pipeline/test-security-config-change.log | 7 + ...t-security-config-change.log-expected.json | 443 ++++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 54 ++- .../ingest_pipeline/pipeline-json.yml | 86 ++-- .../data_stream/audit/fields/ecs.yml | 20 + .../data_stream/audit/fields/fields.yml | 12 + packages/elasticsearch/docs/README.md | 17 + packages/elasticsearch/manifest.yml | 2 +- 10 files changed, 611 insertions(+), 47 deletions(-) create mode 100644 packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-security-config-change.log create mode 100644 packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-security-config-change.log-expected.json diff --git a/packages/elasticsearch/changelog.yml b/packages/elasticsearch/changelog.yml index c31dd5024cf..aabde78269e 100644 --- a/packages/elasticsearch/changelog.yml +++ b/packages/elasticsearch/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.22.1" + changes: + - description: Fix audit `event.outcome` so successful security configuration changes are not labeled as failure, and map the target user to `destination.user`. + type: bugfix + link: https://github.com/elastic/integrations/issues/12163 - version: "1.22.0" changes: - description: Add security_stats data stream to collect per-node Document Level Security (DLS) bitset cache metrics from the Security Stats API (requires Elasticsearch 9.2+). diff --git a/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-audit-logs.log-expected.json b/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-audit-logs.log-expected.json index 9bbd45d4c5e..22621bd384f 100644 --- a/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-audit-logs.log-expected.json +++ b/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-audit-logs.log-expected.json @@ -37,7 +37,8 @@ "created": "2022-09-04T22:54:53.028Z", "ingested": "2023-04-26T12:37:09.665259800Z", "kind": "event", - "outcome": "success" + "outcome": "success", + "type": "access" }, "host": { "id": "rsRsMdvhREeQqLkk3twtqA" @@ -97,7 +98,8 @@ "created": "2022-09-04T22:54:53.034Z", "ingested": "2023-04-26T12:37:09.665284400Z", "kind": "event", - "outcome": "failure" + "outcome": "failure", + "type": "access" }, "host": { "id": "rsRsMdvhREeQqLkk3twtqA" @@ -166,7 +168,8 @@ "created": "2022-09-04T22:54:53.040Z", "ingested": "2023-04-26T12:37:09.665297300Z", "kind": "event", - "outcome": "success" + "outcome": "success", + "type": "access" }, "host": { "id": "rsRsMdvhREeQqLkk3twtqA" @@ -226,7 +229,8 @@ "created": "2019-01-27T20:15:10.380Z", "ingested": "2023-04-26T12:37:09.665308800Z", "kind": "event", - "outcome": "success" + "outcome": "success", + "type": "access" }, "host": { "id": "y8fa3M5zSSGo1M_KJRMUXw", diff --git a/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-security-config-change.log b/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-security-config-change.log new file mode 100644 index 00000000000..d65eea1c7a6 --- /dev/null +++ b/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-security-config-change.log @@ -0,0 +1,7 @@ +{"type":"audit", "timestamp":"2026-07-27T21:40:16,915+0000", "cluster.uuid":"TKAQJQZ4TVyphvg6roSeLA", "node.id":"TinMBx3vSfShqMncoK-jgQ", "event.type":"security_config_change", "event.action":"put_user", "request.id":"XzZOwjI5R2eau63Ol5hFTw", "put":{"user":{"name":"testuser","enabled":true,"roles":["viewer"],"full_name":"Test User","email":"testuser@example.com","has_password":true}}} +{"type":"audit", "timestamp":"2026-07-27T21:40:16,920+0000", "cluster.uuid":"TKAQJQZ4TVyphvg6roSeLA", "node.id":"TinMBx3vSfShqMncoK-jgQ", "event.type":"security_config_change", "event.action":"put_user", "authentication.type":"REALM", "user.name":"chaison", "user.realm":"default_native", "request.id":"actorPutUserReq", "put":{"user":{"name":"testuser","enabled":true,"roles":["viewer"],"full_name":"Test User","email":"testuser@example.com","has_password":true}}} +{"type":"audit", "timestamp":"2026-07-28T01:37:21,977+0000", "cluster.uuid":"TKAQJQZ4TVyphvg6roSeLA", "node.id":"TinMBx3vSfShqMncoK-jgQ", "event.type":"security_config_change", "event.action":"change_password", "request.id":"hvwql0rXQEerGN0dIx6_Cw", "change":{"password":{"user":{"name":"testuser"}}}} +{"type":"audit", "timestamp":"2026-07-28T01:37:22,010+0000", "cluster.uuid":"TKAQJQZ4TVyphvg6roSeLA", "node.id":"TinMBx3vSfShqMncoK-jgQ", "event.type":"security_config_change", "event.action":"put_role", "request.id":"putRoleReq", "put":{"role":{"name":"audit_test_role","role_descriptor":{"cluster":["monitor"]}}}} +{"type":"audit", "timestamp":"2026-07-28T01:37:22,050+0000", "cluster.uuid":"TKAQJQZ4TVyphvg6roSeLA", "node.id":"TinMBx3vSfShqMncoK-jgQ", "event.type":"security_config_change", "event.action":"delete_user", "request.id":"deleteUserReq", "delete":{"user":{"name":"testuser"}}} +{"type":"audit", "timestamp":"2026-07-28T01:37:22,100+0000", "cluster.uuid":"TKAQJQZ4TVyphvg6roSeLA", "node.id":"TinMBx3vSfShqMncoK-jgQ", "event.type":"transport", "event.action":"access_denied", "authentication.type":"REALM", "user.name":"viewer", "user.realm":"default_native", "origin.type":"rest", "origin.address":"172.31.18.32:38722", "request.id":"accessDeniedReq", "action":"cluster:admin/xpack/security/user/put", "request.name":"PutUserRequest"} +{"type":"audit", "timestamp":"2026-07-28T01:37:22,150+0000", "cluster.uuid":"TKAQJQZ4TVyphvg6roSeLA", "node.id":"TinMBx3vSfShqMncoK-jgQ", "event.type":"rest", "event.action":"authentication_failed", "origin.type":"rest", "origin.address":"172.31.18.32:38723", "url.path":"/_security/user/testuser", "request.method":"PUT", "request.id":"authFailedReq"} diff --git a/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-security-config-change.log-expected.json b/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-security-config-change.log-expected.json new file mode 100644 index 00000000000..cd75f8f285c --- /dev/null +++ b/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-security-config-change.log-expected.json @@ -0,0 +1,443 @@ +{ + "expected": [ + { + "@timestamp": "2026-07-27T21:40:16.915Z", + "destination": { + "user": { + "email": "testuser@example.com", + "full_name": "Test User", + "name": "testuser" + } + }, + "elasticsearch": { + "audit": { + "cluster": {}, + "event": {}, + "layer": "security_config_change", + "put": { + "user": { + "email": "testuser@example.com", + "enabled": true, + "full_name": "Test User", + "has_password": true, + "name": "testuser", + "roles": [ + "viewer" + ] + } + }, + "request": { + "id": "XzZOwjI5R2eau63Ol5hFTw" + } + }, + "cluster": { + "uuid": "TKAQJQZ4TVyphvg6roSeLA" + }, + "node": { + "id": "TinMBx3vSfShqMncoK-jgQ" + } + }, + "event": { + "action": "put_user", + "category": "iam", + "created": "2026-07-27T21:40:16.915Z", + "ingested": "2026-07-27T21:40:16.915Z", + "kind": "event", + "outcome": "success", + "type": "change" + }, + "host": { + "id": "TinMBx3vSfShqMncoK-jgQ" + }, + "http": { + "request": { + "id": "XzZOwjI5R2eau63Ol5hFTw" + } + }, + "log": { + "level": "info" + }, + "message": "{\"type\":\"audit\", \"timestamp\":\"2026-07-27T21:40:16,915+0000\", \"cluster.uuid\":\"TKAQJQZ4TVyphvg6roSeLA\", \"node.id\":\"TinMBx3vSfShqMncoK-jgQ\", \"event.type\":\"security_config_change\", \"event.action\":\"put_user\", \"request.id\":\"XzZOwjI5R2eau63Ol5hFTw\", \"put\":{\"user\":{\"name\":\"testuser\",\"enabled\":true,\"roles\":[\"viewer\"],\"full_name\":\"Test User\",\"email\":\"testuser@example.com\",\"has_password\":true}}}", + "related": { + "user": [ + "testuser" + ] + }, + "service": { + "type": "elasticsearch" + }, + "user": { + "name": "testuser" + } + }, + { + "@timestamp": "2026-07-27T21:40:16.920Z", + "destination": { + "user": { + "email": "testuser@example.com", + "full_name": "Test User", + "name": "testuser" + } + }, + "elasticsearch": { + "audit": { + "authentication.type": "REALM", + "cluster": {}, + "event": {}, + "layer": "security_config_change", + "put": { + "user": { + "email": "testuser@example.com", + "enabled": true, + "full_name": "Test User", + "has_password": true, + "name": "testuser", + "roles": [ + "viewer" + ] + } + }, + "request": { + "id": "actorPutUserReq" + }, + "user": {}, + "user.realm": "default_native" + }, + "cluster": { + "uuid": "TKAQJQZ4TVyphvg6roSeLA" + }, + "node": { + "id": "TinMBx3vSfShqMncoK-jgQ" + } + }, + "event": { + "action": "put_user", + "category": "iam", + "created": "2026-07-27T21:40:16.920Z", + "ingested": "2026-07-27T21:40:16.920Z", + "kind": "event", + "outcome": "success", + "type": "change" + }, + "host": { + "id": "TinMBx3vSfShqMncoK-jgQ" + }, + "http": { + "request": { + "id": "actorPutUserReq" + } + }, + "log": { + "level": "info" + }, + "message": "{\"type\":\"audit\", \"timestamp\":\"2026-07-27T21:40:16,920+0000\", \"cluster.uuid\":\"TKAQJQZ4TVyphvg6roSeLA\", \"node.id\":\"TinMBx3vSfShqMncoK-jgQ\", \"event.type\":\"security_config_change\", \"event.action\":\"put_user\", \"authentication.type\":\"REALM\", \"user.name\":\"chaison\", \"user.realm\":\"default_native\", \"request.id\":\"actorPutUserReq\", \"put\":{\"user\":{\"name\":\"testuser\",\"enabled\":true,\"roles\":[\"viewer\"],\"full_name\":\"Test User\",\"email\":\"testuser@example.com\",\"has_password\":true}}}", + "related": { + "user": [ + "chaison", + "testuser" + ] + }, + "service": { + "type": "elasticsearch" + }, + "user": { + "name": "chaison" + } + }, + { + "@timestamp": "2026-07-28T01:37:21.977Z", + "destination": { + "user": { + "name": "testuser" + } + }, + "elasticsearch": { + "audit": { + "change": { + "password": { + "user": { + "name": "testuser" + } + } + }, + "cluster": {}, + "event": {}, + "layer": "security_config_change", + "request": { + "id": "hvwql0rXQEerGN0dIx6_Cw" + } + }, + "cluster": { + "uuid": "TKAQJQZ4TVyphvg6roSeLA" + }, + "node": { + "id": "TinMBx3vSfShqMncoK-jgQ" + } + }, + "event": { + "action": "change_password", + "category": "iam", + "created": "2026-07-28T01:37:21.977Z", + "ingested": "2026-07-28T01:37:21.977Z", + "kind": "event", + "outcome": "success", + "type": "change" + }, + "host": { + "id": "TinMBx3vSfShqMncoK-jgQ" + }, + "http": { + "request": { + "id": "hvwql0rXQEerGN0dIx6_Cw" + } + }, + "log": { + "level": "info" + }, + "message": "{\"type\":\"audit\", \"timestamp\":\"2026-07-28T01:37:21,977+0000\", \"cluster.uuid\":\"TKAQJQZ4TVyphvg6roSeLA\", \"node.id\":\"TinMBx3vSfShqMncoK-jgQ\", \"event.type\":\"security_config_change\", \"event.action\":\"change_password\", \"request.id\":\"hvwql0rXQEerGN0dIx6_Cw\", \"change\":{\"password\":{\"user\":{\"name\":\"testuser\"}}}}", + "related": { + "user": [ + "testuser" + ] + }, + "service": { + "type": "elasticsearch" + }, + "user": { + "name": "testuser" + } + }, + { + "@timestamp": "2026-07-28T01:37:22.010Z", + "elasticsearch": { + "audit": { + "cluster": {}, + "event": {}, + "layer": "security_config_change", + "put": { + "role": { + "name": "audit_test_role", + "role_descriptor": { + "cluster": [ + "monitor" + ] + } + } + }, + "request": { + "id": "putRoleReq" + } + }, + "cluster": { + "uuid": "TKAQJQZ4TVyphvg6roSeLA" + }, + "node": { + "id": "TinMBx3vSfShqMncoK-jgQ" + } + }, + "event": { + "action": "put_role", + "category": "iam", + "created": "2026-07-28T01:37:22.010Z", + "ingested": "2026-07-28T01:37:22.010Z", + "kind": "event", + "outcome": "success", + "type": "change" + }, + "host": { + "id": "TinMBx3vSfShqMncoK-jgQ" + }, + "http": { + "request": { + "id": "putRoleReq" + } + }, + "log": { + "level": "info" + }, + "message": "{\"type\":\"audit\", \"timestamp\":\"2026-07-28T01:37:22,010+0000\", \"cluster.uuid\":\"TKAQJQZ4TVyphvg6roSeLA\", \"node.id\":\"TinMBx3vSfShqMncoK-jgQ\", \"event.type\":\"security_config_change\", \"event.action\":\"put_role\", \"request.id\":\"putRoleReq\", \"put\":{\"role\":{\"name\":\"audit_test_role\",\"role_descriptor\":{\"cluster\":[\"monitor\"]}}}}", + "service": { + "type": "elasticsearch" + } + }, + { + "@timestamp": "2026-07-28T01:37:22.050Z", + "destination": { + "user": { + "name": "testuser" + } + }, + "elasticsearch": { + "audit": { + "cluster": {}, + "delete": { + "user": { + "name": "testuser" + } + }, + "event": {}, + "layer": "security_config_change", + "request": { + "id": "deleteUserReq" + } + }, + "cluster": { + "uuid": "TKAQJQZ4TVyphvg6roSeLA" + }, + "node": { + "id": "TinMBx3vSfShqMncoK-jgQ" + } + }, + "event": { + "action": "delete_user", + "category": "iam", + "created": "2026-07-28T01:37:22.050Z", + "ingested": "2026-07-28T01:37:22.050Z", + "kind": "event", + "outcome": "success", + "type": "change" + }, + "host": { + "id": "TinMBx3vSfShqMncoK-jgQ" + }, + "http": { + "request": { + "id": "deleteUserReq" + } + }, + "log": { + "level": "info" + }, + "message": "{\"type\":\"audit\", \"timestamp\":\"2026-07-28T01:37:22,050+0000\", \"cluster.uuid\":\"TKAQJQZ4TVyphvg6roSeLA\", \"node.id\":\"TinMBx3vSfShqMncoK-jgQ\", \"event.type\":\"security_config_change\", \"event.action\":\"delete_user\", \"request.id\":\"deleteUserReq\", \"delete\":{\"user\":{\"name\":\"testuser\"}}}", + "related": { + "user": [ + "testuser" + ] + }, + "service": { + "type": "elasticsearch" + }, + "user": { + "name": "testuser" + } + }, + { + "@timestamp": "2026-07-28T01:37:22.100Z", + "elasticsearch": { + "audit": { + "action": "cluster:admin/xpack/security/user/put", + "authentication.type": "REALM", + "cluster": {}, + "event": {}, + "layer": "transport", + "origin": {}, + "origin.type": "rest", + "request": { + "id": "accessDeniedReq" + }, + "request.name": "PutUserRequest", + "user": {}, + "user.realm": "default_native" + }, + "cluster": { + "uuid": "TKAQJQZ4TVyphvg6roSeLA" + }, + "node": { + "id": "TinMBx3vSfShqMncoK-jgQ" + } + }, + "event": { + "action": "access_denied", + "category": "database", + "created": "2026-07-28T01:37:22.100Z", + "ingested": "2026-07-28T01:37:22.100Z", + "kind": "event", + "outcome": "failure", + "type": "access" + }, + "host": { + "id": "TinMBx3vSfShqMncoK-jgQ" + }, + "http": { + "request": { + "id": "accessDeniedReq" + } + }, + "log": { + "level": "info" + }, + "message": "{\"type\":\"audit\", \"timestamp\":\"2026-07-28T01:37:22,100+0000\", \"cluster.uuid\":\"TKAQJQZ4TVyphvg6roSeLA\", \"node.id\":\"TinMBx3vSfShqMncoK-jgQ\", \"event.type\":\"transport\", \"event.action\":\"access_denied\", \"authentication.type\":\"REALM\", \"user.name\":\"viewer\", \"user.realm\":\"default_native\", \"origin.type\":\"rest\", \"origin.address\":\"172.31.18.32:38722\", \"request.id\":\"accessDeniedReq\", \"action\":\"cluster:admin/xpack/security/user/put\", \"request.name\":\"PutUserRequest\"}", + "related": { + "user": [ + "viewer" + ] + }, + "service": { + "type": "elasticsearch" + }, + "source": { + "address": "172.31.18.32:38722", + "ip": "172.31.18.32", + "port": 38722 + }, + "user": { + "name": "viewer" + } + }, + { + "@timestamp": "2026-07-28T01:37:22.150Z", + "elasticsearch": { + "audit": { + "cluster": {}, + "event": {}, + "layer": "rest", + "origin": {}, + "origin.type": "rest", + "request": { + "id": "authFailedReq" + }, + "url": {} + }, + "cluster": { + "uuid": "TKAQJQZ4TVyphvg6roSeLA" + }, + "node": { + "id": "TinMBx3vSfShqMncoK-jgQ" + } + }, + "event": { + "action": "authentication_failed", + "category": "database", + "created": "2026-07-28T01:37:22.150Z", + "ingested": "2026-07-28T01:37:22.150Z", + "kind": "event", + "outcome": "failure", + "type": "access" + }, + "host": { + "id": "TinMBx3vSfShqMncoK-jgQ" + }, + "http": { + "request": { + "id": "authFailedReq", + "method": "PUT" + } + }, + "log": { + "level": "info" + }, + "message": "{\"type\":\"audit\", \"timestamp\":\"2026-07-28T01:37:22,150+0000\", \"cluster.uuid\":\"TKAQJQZ4TVyphvg6roSeLA\", \"node.id\":\"TinMBx3vSfShqMncoK-jgQ\", \"event.type\":\"rest\", \"event.action\":\"authentication_failed\", \"origin.type\":\"rest\", \"origin.address\":\"172.31.18.32:38723\", \"url.path\":\"/_security/user/testuser\", \"request.method\":\"PUT\", \"request.id\":\"authFailedReq\"}", + "service": { + "type": "elasticsearch" + }, + "source": { + "address": "172.31.18.32:38723", + "ip": "172.31.18.32", + "port": 38723 + }, + "url": { + "original": "/_security/user/testuser" + } + } + ] +} diff --git a/packages/elasticsearch/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/elasticsearch/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index ba87b176b12..acecf1117f3 100644 --- a/packages/elasticsearch/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/elasticsearch/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -25,22 +25,47 @@ processors: field: event.category value: database - set: - if: "ctx?.elasticsearch?.audit?.event_type != null" + if: ctx.elasticsearch?.audit?.layer == 'security_config_change' + field: event.category + value: iam + - set: + if: ctx.elasticsearch?.audit?.layer != null && ctx.elasticsearch.audit.layer != 'security_config_change' field: event.type value: access + - set: + if: ctx.elasticsearch?.audit?.layer == 'security_config_change' + field: event.type + value: change + # Elasticsearch does not emit event.outcome. Map it from event.action / audit + # layer. Do not default to failure: unknown actions must stay unknown. + # security_config_change is authorization-shaped success (the change was + # recorded), not a confirmed HTTP 200. - script: lang: painless - source: >- - def successEvents = ['authentication_success', 'access_granted', 'run_as_granted', 'connection_granted']; - if (ctx?.elasticsearch?.audit?.event_type != null && successEvents.contains(ctx.elasticsearch.audit.event_type)) { - ctx.event.outcome = 'success'; - } else { + source: | + def failureActions = [ + 'authentication_failed', + 'access_denied', + 'anonymous_access_denied', + 'realm_authentication_failed', + 'run_as_denied', + 'connection_denied', + 'tampered_request' + ]; + def successActions = [ + 'authentication_success', + 'access_granted', + 'run_as_granted', + 'connection_granted' + ]; + def action = ctx.event?.action; + def layer = ctx.elasticsearch?.audit?.layer; + if (action != null && failureActions.contains(action)) { ctx.event.outcome = 'failure'; - } - if (ctx?.event.action != null && successEvents.contains(ctx.event.action)) { + } else if ((action != null && successActions.contains(action)) || layer == 'security_config_change') { ctx.event.outcome = 'success'; } else { - ctx.event.outcome = 'failure'; + ctx.event.outcome = 'unknown'; } - set: field: host.id @@ -53,11 +78,18 @@ processors: - append: field: related.user value: "{{user.name}}" - if: "ctx?.user?.name != null" + if: ctx?.user?.name != null + allow_duplicates: false - append: field: related.user value: "{{user.effective.name}}" - if: "ctx?.user?.effective?.name != null" + if: ctx?.user?.effective?.name != null + allow_duplicates: false + - append: + field: related.user + value: "{{destination.user.name}}" + if: ctx?.destination?.user?.name != null + allow_duplicates: false - remove: field: elasticsearch.audit.@timestamp - remove: diff --git a/packages/elasticsearch/data_stream/audit/elasticsearch/ingest_pipeline/pipeline-json.yml b/packages/elasticsearch/data_stream/audit/elasticsearch/ingest_pipeline/pipeline-json.yml index 7d1c6eef276..a291c34d03f 100644 --- a/packages/elasticsearch/data_stream/audit/elasticsearch/ingest_pipeline/pipeline-json.yml +++ b/packages/elasticsearch/data_stream/audit/elasticsearch/ingest_pipeline/pipeline-json.yml @@ -93,37 +93,61 @@ processors: - rename: field: elasticsearch.audit.node target_field: elasticsearch.node - - rename: - field: elasticsearch.audit.change.disable.user.name - target_field: user.name - ignore_missing: true - - rename: - field: elasticsearch.audit.change.enable.user.name - target_field: user.name - ignore_missing: true - - rename: - field: elasticsearch.audit.delete.user.name - target_field: user.name - ignore_missing: true - - rename: - field: elasticsearch.audit.put.user.name - target_field: user.name - ignore_missing: true - - rename: - field: elasticsearch.audit.put.user.full_name - target_field: user.full_name - ignore_missing: true - - rename: - field: elasticsearch.audit.put.user.email - target_field: user.email - ignore_missing: true - - remove: - field: elasticsearch.audit.put - ignore_missing: true - - rename: - field: elasticsearch.audit.invalidate.apikeys.user.name - target_field: user.name - ignore_missing: true + # Target of a security_config_change (the user being created/updated/deleted). + # Actor attribution, when the producer emits it, is mapped to user.name below. + - set: + field: destination.user.name + copy_from: elasticsearch.audit.put.user.name + ignore_empty_value: true + if: ctx.elasticsearch?.audit?.put?.user?.name != null + - set: + field: destination.user.full_name + copy_from: elasticsearch.audit.put.user.full_name + ignore_empty_value: true + if: ctx.elasticsearch?.audit?.put?.user?.full_name != null + - set: + field: destination.user.email + copy_from: elasticsearch.audit.put.user.email + ignore_empty_value: true + if: ctx.elasticsearch?.audit?.put?.user?.email != null + - set: + field: destination.user.name + copy_from: elasticsearch.audit.delete.user.name + ignore_empty_value: true + override: false + if: ctx.elasticsearch?.audit?.delete?.user?.name != null + - set: + field: destination.user.name + copy_from: elasticsearch.audit.change.enable.user.name + ignore_empty_value: true + override: false + if: ctx.elasticsearch?.audit?.change?.enable?.user?.name != null + - set: + field: destination.user.name + copy_from: elasticsearch.audit.change.disable.user.name + ignore_empty_value: true + override: false + if: ctx.elasticsearch?.audit?.change?.disable?.user?.name != null + - set: + field: destination.user.name + copy_from: elasticsearch.audit.change.password.user.name + ignore_empty_value: true + override: false + if: ctx.elasticsearch?.audit?.change?.password?.user?.name != null + - set: + field: destination.user.name + copy_from: elasticsearch.audit.invalidate.apikeys.user.name + ignore_empty_value: true + override: false + if: ctx.elasticsearch?.audit?.invalidate?.apikeys?.user?.name != null + # Backward compatible fallback: if the producer did not emit an actor, keep + # populating user.name from the target so existing dashboards keep working. + - set: + field: user.name + copy_from: destination.user.name + ignore_empty_value: true + override: false + if: ctx.destination?.user?.name != null - rename: field: elasticsearch.audit.invalidate.apikeys.user.realm target_field: elasticsearch.audit.user.realm diff --git a/packages/elasticsearch/data_stream/audit/fields/ecs.yml b/packages/elasticsearch/data_stream/audit/fields/ecs.yml index 8c6a701efb9..154280b57ac 100644 --- a/packages/elasticsearch/data_stream/audit/fields/ecs.yml +++ b/packages/elasticsearch/data_stream/audit/fields/ecs.yml @@ -16,6 +16,16 @@ name: user - external: ecs name: user.name +- external: ecs + name: user.full_name +- external: ecs + name: user.email +- external: ecs + name: destination.user.name +- external: ecs + name: destination.user.full_name +- external: ecs + name: destination.user.email - external: ecs name: http.request.id - external: ecs @@ -34,6 +44,16 @@ name: trace.id - external: ecs name: message +- external: ecs + name: event.action +- external: ecs + name: event.category +- external: ecs + name: event.kind +- external: ecs + name: event.outcome +- external: ecs + name: event.type - external: ecs name: event.created - external: ecs diff --git a/packages/elasticsearch/data_stream/audit/fields/fields.yml b/packages/elasticsearch/data_stream/audit/fields/fields.yml index 7d2bbef22fb..302904df25c 100644 --- a/packages/elasticsearch/data_stream/audit/fields/fields.yml +++ b/packages/elasticsearch/data_stream/audit/fields/fields.yml @@ -46,5 +46,17 @@ description: The type of request that was executed - name: message type: text + - name: put + type: flattened + description: Security configuration object that was created or updated + - name: delete + type: flattened + description: Security configuration object that was deleted + - name: change + type: flattened + description: Security configuration object that was changed + - name: create + type: flattened + description: Security configuration object that was created - name: invalidate.apikeys.owned_by_authenticated_user type: boolean diff --git a/packages/elasticsearch/docs/README.md b/packages/elasticsearch/docs/README.md index 3901f27dd66..8ba4f5049bf 100644 --- a/packages/elasticsearch/docs/README.md +++ b/packages/elasticsearch/docs/README.md @@ -20,10 +20,18 @@ NOTE: Configure the `var.paths` setting to point to JSON logs. | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | +| destination.user.email | User email address. | keyword | +| destination.user.full_name | User's full name, if available. | keyword | +| destination.user.full_name.text | Multi-field of `destination.user.full_name`. | match_only_text | +| destination.user.name | Short name or login of the user. | keyword | +| destination.user.name.text | Multi-field of `destination.user.name`. | match_only_text | | ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | elasticsearch.audit.action | The name of the action that was executed | keyword | | elasticsearch.audit.authentication.type | | keyword | +| elasticsearch.audit.change | Security configuration object that was changed | flattened | | elasticsearch.audit.component | | keyword | +| elasticsearch.audit.create | Security configuration object that was created | flattened | +| elasticsearch.audit.delete | Security configuration object that was deleted | flattened | | elasticsearch.audit.event_type | The type of event that occurred: anonymous_access_denied, authentication_failed, access_denied, access_granted, connection_granted, connection_denied, tampered_request, run_as_granted, run_as_denied | keyword | | elasticsearch.audit.indices | Indices accessed by action | keyword | | elasticsearch.audit.invalidate.apikeys.owned_by_authenticated_user | | boolean | @@ -31,6 +39,7 @@ NOTE: Configure the `var.paths` setting to point to JSON logs. | elasticsearch.audit.message | | text | | elasticsearch.audit.opaque_id | | keyword | | elasticsearch.audit.origin.type | Where the request originated: rest (request originated from a REST API request), transport (request was received on the transport channel), local_node (the local node issued the request) | keyword | +| elasticsearch.audit.put | Security configuration object that was created or updated | flattened | | elasticsearch.audit.realm | The authentication realm the authentication was validated against | keyword | | elasticsearch.audit.request.id | Unique ID of request | keyword | | elasticsearch.audit.request.name | The type of request that was executed | keyword | @@ -47,9 +56,14 @@ NOTE: Configure the `var.paths` setting to point to JSON logs. | elasticsearch.node.id | ID of the node | keyword | | elasticsearch.node.name | Name of the node | keyword | | elasticsearch.shard.id | Id of the shard | keyword | +| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | | event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | | event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | +| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | http | Fields related to HTTP activity. Use the `url` field set to store the url of the request. | group | | http.request.body.content | The full HTTP request body. | wildcard | | http.request.body.content.text | Multi-field of `http.request.body.content`. | match_only_text | @@ -71,6 +85,9 @@ NOTE: Configure the `var.paths` setting to point to JSON logs. | url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | | url.original.text | Multi-field of `url.original`. | match_only_text | | user | The user fields describe information about the user that is relevant to the event. Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. | group | +| user.email | User email address. | keyword | +| user.full_name | User's full name, if available. | keyword | +| user.full_name.text | Multi-field of `user.full_name`. | match_only_text | | user.name | Short name or login of the user. | keyword | | user.name.text | Multi-field of `user.name`. | match_only_text | diff --git a/packages/elasticsearch/manifest.yml b/packages/elasticsearch/manifest.yml index 0d97fb3c569..8a53cc3b73f 100644 --- a/packages/elasticsearch/manifest.yml +++ b/packages/elasticsearch/manifest.yml @@ -1,6 +1,6 @@ name: elasticsearch title: Elasticsearch -version: 1.22.0 +version: 1.22.1 description: Elasticsearch Integration type: integration icons: From 121f49a2e1a807c1e77ad55f79f25a5f56c952d7 Mon Sep 17 00:00:00 2001 From: consulthys Date: Thu, 3 Sep 2026 08:38:54 +0200 Subject: [PATCH 2/3] [elasticsearch] Fix audit pipeline test failures Set event.category and event.type as ECS arrays, and apply the target-user fallback only after actor user.name is copied. Co-authored-by: Cursor --- .../test-audit-logs.log-expected.json | 34 ++++++++--- ...t-security-config-change.log-expected.json | 56 ++++++++++++++----- .../elasticsearch/ingest_pipeline/default.yml | 8 +-- .../ingest_pipeline/pipeline-json.yml | 16 +++--- 4 files changed, 79 insertions(+), 35 deletions(-) diff --git a/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-audit-logs.log-expected.json b/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-audit-logs.log-expected.json index 22621bd384f..0cfc6e7fe40 100644 --- a/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-audit-logs.log-expected.json +++ b/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-audit-logs.log-expected.json @@ -33,12 +33,16 @@ }, "event": { "action": "access_granted", - "category": "database", + "category": [ + "database" + ], "created": "2022-09-04T22:54:53.028Z", "ingested": "2023-04-26T12:37:09.665259800Z", "kind": "event", "outcome": "success", - "type": "access" + "type": [ + "access" + ] }, "host": { "id": "rsRsMdvhREeQqLkk3twtqA" @@ -94,12 +98,16 @@ }, "event": { "action": "anonymous_access_denied", - "category": "database", + "category": [ + "database" + ], "created": "2022-09-04T22:54:53.034Z", "ingested": "2023-04-26T12:37:09.665284400Z", "kind": "event", "outcome": "failure", - "type": "access" + "type": [ + "access" + ] }, "host": { "id": "rsRsMdvhREeQqLkk3twtqA" @@ -164,12 +172,16 @@ }, "event": { "action": "access_granted", - "category": "database", + "category": [ + "database" + ], "created": "2022-09-04T22:54:53.040Z", "ingested": "2023-04-26T12:37:09.665297300Z", "kind": "event", "outcome": "success", - "type": "access" + "type": [ + "access" + ] }, "host": { "id": "rsRsMdvhREeQqLkk3twtqA" @@ -225,12 +237,16 @@ }, "event": { "action": "authentication_success", - "category": "database", + "category": [ + "database" + ], "created": "2019-01-27T20:15:10.380Z", "ingested": "2023-04-26T12:37:09.665308800Z", "kind": "event", "outcome": "success", - "type": "access" + "type": [ + "access" + ] }, "host": { "id": "y8fa3M5zSSGo1M_KJRMUXw", @@ -270,4 +286,4 @@ } } ] -} \ No newline at end of file +} diff --git a/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-security-config-change.log-expected.json b/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-security-config-change.log-expected.json index cd75f8f285c..cbf7e7f4649 100644 --- a/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-security-config-change.log-expected.json +++ b/packages/elasticsearch/data_stream/audit/_dev/test/pipeline/test-security-config-change.log-expected.json @@ -39,12 +39,16 @@ }, "event": { "action": "put_user", - "category": "iam", + "category": [ + "iam" + ], "created": "2026-07-27T21:40:16.915Z", "ingested": "2026-07-27T21:40:16.915Z", "kind": "event", "outcome": "success", - "type": "change" + "type": [ + "change" + ] }, "host": { "id": "TinMBx3vSfShqMncoK-jgQ" @@ -112,12 +116,16 @@ }, "event": { "action": "put_user", - "category": "iam", + "category": [ + "iam" + ], "created": "2026-07-27T21:40:16.920Z", "ingested": "2026-07-27T21:40:16.920Z", "kind": "event", "outcome": "success", - "type": "change" + "type": [ + "change" + ] }, "host": { "id": "TinMBx3vSfShqMncoK-jgQ" @@ -176,12 +184,16 @@ }, "event": { "action": "change_password", - "category": "iam", + "category": [ + "iam" + ], "created": "2026-07-28T01:37:21.977Z", "ingested": "2026-07-28T01:37:21.977Z", "kind": "event", "outcome": "success", - "type": "change" + "type": [ + "change" + ] }, "host": { "id": "TinMBx3vSfShqMncoK-jgQ" @@ -237,12 +249,16 @@ }, "event": { "action": "put_role", - "category": "iam", + "category": [ + "iam" + ], "created": "2026-07-28T01:37:22.010Z", "ingested": "2026-07-28T01:37:22.010Z", "kind": "event", "outcome": "success", - "type": "change" + "type": [ + "change" + ] }, "host": { "id": "TinMBx3vSfShqMncoK-jgQ" @@ -290,12 +306,16 @@ }, "event": { "action": "delete_user", - "category": "iam", + "category": [ + "iam" + ], "created": "2026-07-28T01:37:22.050Z", "ingested": "2026-07-28T01:37:22.050Z", "kind": "event", "outcome": "success", - "type": "change" + "type": [ + "change" + ] }, "host": { "id": "TinMBx3vSfShqMncoK-jgQ" @@ -348,12 +368,16 @@ }, "event": { "action": "access_denied", - "category": "database", + "category": [ + "database" + ], "created": "2026-07-28T01:37:22.100Z", "ingested": "2026-07-28T01:37:22.100Z", "kind": "event", "outcome": "failure", - "type": "access" + "type": [ + "access" + ] }, "host": { "id": "TinMBx3vSfShqMncoK-jgQ" @@ -407,12 +431,16 @@ }, "event": { "action": "authentication_failed", - "category": "database", + "category": [ + "database" + ], "created": "2026-07-28T01:37:22.150Z", "ingested": "2026-07-28T01:37:22.150Z", "kind": "event", "outcome": "failure", - "type": "access" + "type": [ + "access" + ] }, "host": { "id": "TinMBx3vSfShqMncoK-jgQ" diff --git a/packages/elasticsearch/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/elasticsearch/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index acecf1117f3..0715d1b3f5d 100644 --- a/packages/elasticsearch/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/elasticsearch/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -23,19 +23,19 @@ processors: value: event - set: field: event.category - value: database + value: ["database"] - set: if: ctx.elasticsearch?.audit?.layer == 'security_config_change' field: event.category - value: iam + value: ["iam"] - set: if: ctx.elasticsearch?.audit?.layer != null && ctx.elasticsearch.audit.layer != 'security_config_change' field: event.type - value: access + value: ["access"] - set: if: ctx.elasticsearch?.audit?.layer == 'security_config_change' field: event.type - value: change + value: ["change"] # Elasticsearch does not emit event.outcome. Map it from event.action / audit # layer. Do not default to failure: unknown actions must stay unknown. # security_config_change is authorization-shaped success (the change was diff --git a/packages/elasticsearch/data_stream/audit/elasticsearch/ingest_pipeline/pipeline-json.yml b/packages/elasticsearch/data_stream/audit/elasticsearch/ingest_pipeline/pipeline-json.yml index a291c34d03f..e8d287e33f9 100644 --- a/packages/elasticsearch/data_stream/audit/elasticsearch/ingest_pipeline/pipeline-json.yml +++ b/packages/elasticsearch/data_stream/audit/elasticsearch/ingest_pipeline/pipeline-json.yml @@ -140,14 +140,6 @@ processors: ignore_empty_value: true override: false if: ctx.elasticsearch?.audit?.invalidate?.apikeys?.user?.name != null - # Backward compatible fallback: if the producer did not emit an actor, keep - # populating user.name from the target so existing dashboards keep working. - - set: - field: user.name - copy_from: destination.user.name - ignore_empty_value: true - override: false - if: ctx.destination?.user?.name != null - rename: field: elasticsearch.audit.invalidate.apikeys.user.realm target_field: elasticsearch.audit.user.realm @@ -173,6 +165,14 @@ processors: field: elasticsearch.audit.user.name target_field: user.name ignore_missing: true + # If the producer did not emit an actor, keep populating user.name from the + # target so existing dashboards keep working. override:false preserves actor. + - set: + field: user.name + copy_from: destination.user.name + ignore_empty_value: true + override: false + if: ctx.destination?.user?.name != null - dot_expander: field: user.email path: elasticsearch.audit From dca95b243c272bf568b5bea58dfc87db0978b489 Mon Sep 17 00:00:00 2001 From: consulthys Date: Thu, 3 Sep 2026 09:22:48 +0200 Subject: [PATCH 3/3] [elasticsearch] Align audit sample event with ECS arrays Static tests require event.category and event.type to be arrays after those fields were declared as ECS in the audit data stream. Co-authored-by: Cursor --- .../elasticsearch/data_stream/audit/sample_event.json | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/elasticsearch/data_stream/audit/sample_event.json b/packages/elasticsearch/data_stream/audit/sample_event.json index 1986a2e4e25..868533363b8 100644 --- a/packages/elasticsearch/data_stream/audit/sample_event.json +++ b/packages/elasticsearch/data_stream/audit/sample_event.json @@ -49,12 +49,17 @@ "event": { "action": "access_granted", "agent_id_status": "verified", - "category": "database", + "category": [ + "database" + ], "created": "2023-04-26T12:26:24.580Z", "dataset": "elasticsearch.audit", "ingested": "2023-04-26T12:26:40Z", "kind": "event", - "outcome": "success" + "outcome": "success", + "type": [ + "access" + ] }, "host": { "architecture": "x86_64",