feat(ocsf): OCSF-to-proto generator (ocsf-protogen) [AI-1408]

.github/CODEOWNERS

@@ -10,6 +10,7 @@
 /api/ @sago2k8 @weeco
 /kube/ @redpanda-data/k8s
 /authz/ @redpanda-data/team-ai
+/ocsf/ @redpanda-data/team-ai
 /licenseupdater/ @redpanda-data/k8s
 /alias/ @redpanda-data/k8s
 /goldenfile/ @redpanda-data/k8s

.github/workflows/golangci-lint.yaml

@@ -30,7 +30,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        dir: [api, rpadmin, secrets, net, rpsr, redpanda-otel-exporter, authz, kvstore, license, licenseupdater, kube, telemetry]
+        dir: [api, rpadmin, secrets, net, rpsr, redpanda-otel-exporter, authz, kvstore, license, licenseupdater, kube, telemetry, ocsf]
     steps:
       - uses: actions/checkout@v4
 

.github/workflows/test.yaml

@@ -6,12 +6,14 @@ on:
       - main
     paths:
       - '**.go'
+      - '**.proto'
       - '**/go.mod'
       - '**/go.sum'
       - '.github/workflows/test.yaml'
   pull_request:
     paths:
       - '**.go'
+      - '**.proto'
       - '**/go.mod'
       - '**/go.sum'
       - '.github/workflows/test.yaml'
@@ -32,3 +34,98 @@ jobs:
       - name: Run tests (${{ matrix.dir }})
         run: go test ./...
         working-directory: ${{ matrix.dir }}
+
+  ocsf-checks:
+    name: ocsf-checks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Fetch full history so we can resolve the base branch ref for the
+          # tagmap compat check below.
+          fetch-depth: 0
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: 'stable'
+
+      # buf is used to lint and build the generated .proto files.
+      - name: Install buf
+        uses: bufbuild/buf-setup-action@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      # Wire-stability gate: compare the committed field-numbers.json on this
+      # branch against the base-branch version.  A changed tag number, a
+      # dropped field without a reserved entry, or a reused reserved tag all
+      # fail here before the baseline-drift check runs.
+      #
+      # On push events github.base_ref is empty; we skip gracefully.
+      # On the first PR that introduces the tagmap (old file absent on base),
+      # CompatCheck prints a bootstrap notice and exits 0.
+      - name: ocsf-protogen --compat-check (wire-stability gate)
+        if: ${{ github.base_ref != '' }}
+        working-directory: ocsf
+        run: |
+          git fetch origin "${{ github.base_ref }}" --depth=1
+          git show "origin/${{ github.base_ref }}:ocsf/cmd/ocsf-protogen/testdata/field-numbers.json" \
+            > /tmp/old-field-numbers.json 2>/dev/null || true
+          go run ./cmd/ocsf-protogen \
+            --compat-check \
+            --old /tmp/old-field-numbers.json \
+            --new cmd/ocsf-protogen/testdata/field-numbers.json
+
+      # Verify the committed proto baseline tree (ocsf/v1/*.proto +
+      # field-numbers.json) exactly matches what the generator would produce today
+      # from the committed schema fixture.  --out is a module root directory; the
+      # generator writes one file per class plus a shared objects.proto under
+      # ocsf/v1/.  Drift here means someone edited the baseline by hand or forgot
+      # to regenerate after a schema/generator change.
+      - name: ocsf-protogen --check (baseline drift detection)
+        working-directory: ocsf
+        run: |
+          go run ./cmd/ocsf-protogen \
+            --check \
+            --schema  internal/ocsf/schema/testdata/ocsf-1.8.0.json \
+            --classes api_activity,entity_management \
+            --version 1.8.0 \
+            --out     cmd/ocsf-protogen/testdata \
+            --tagmap  cmd/ocsf-protogen/testdata/field-numbers.json \
+            --merged-message AuditEvent \
+            --merged-sr-subject redpanda.ocsf.audit-events-value
+
+      # buf lint + build on the committed proto baseline.
+      - name: buf lint (ocsf testdata)
+        working-directory: ocsf/cmd/ocsf-protogen/testdata
+        run: buf lint
+
+      - name: buf build (ocsf testdata)
+        working-directory: ocsf/cmd/ocsf-protogen/testdata
+        run: buf build
+
+      # buf lint + build on the golden proto used by gen tests.
+      - name: buf lint (gen golden)
+        working-directory: ocsf/internal/ocsf/gen/testdata/golden
+        run: buf lint
+
+      - name: buf build (gen golden)
+        working-directory: ocsf/internal/ocsf/gen/testdata/golden
+        run: buf build
+
+      # buf lint + build on the merged single-event golden (AuditEvent layout).
+      - name: buf lint (gen golden-merged)
+        working-directory: ocsf/internal/ocsf/gen/testdata/golden-merged
+        run: buf lint
+
+      - name: buf build (gen golden-merged)
+        working-directory: ocsf/internal/ocsf/gen/testdata/golden-merged
+        run: buf build
+
+      # Run the full ocsf test suite including TestServerConformance which
+      # POSTs exported OCSF JSON to schema.ocsf.io/api/v2/validate and gates
+      # on error_count == 0.  This test skips automatically when
+      # schema.ocsf.io is unreachable (e.g. local dev without internet) but
+      # must pass here in CI.
+      - name: ocsf go test (with server conformance)
+        working-directory: ocsf
+        run: go test ./... -count=1

.licenseupdater.yaml

@@ -43,6 +43,10 @@ matches:
     type: go
     directory: net
     license: Apache
+  - extension: .go
+    type: go
+    directory: ocsf
+    license: BSL
   - extension: .go
     type: go
     directory: proto

README.md

@@ -21,6 +21,7 @@ All packages contain their own licensing information and README about how they a
 | [authz](./authz) | BSL |
 | [kvstore](./kvstore) | Apache |
 | [net](./net) | Apache |
+| [ocsf](./ocsf) | BSL |
 | [proto](./proto) | BSL |
 | [redpanda-otel-exporter](./redpanda-otel-exporter) | BSL |
 | [rpadmin](./rpadmin) | BSL |

ocsf/LICENSE

@@ -0,0 +1,72 @@
+Redpanda Business Source License 1.1
+
+<u>License</u>: BSL 1.1
+
+<u>Licensor</u>: Redpanda Data, Inc.
+
+<u>Licensed Work</u>: Redpanda The Licensed Work is © 2026
+Redpanda Data, Inc.
+
+<u>Additional Use Grant</u>: You may make use of the Licensed Work,
+provided that you may not use the Licensed Work for a Streaming or
+Queuing Service. A "Streaming or Queueing Service" is a commercial
+offering that allows third parties (other than your employees and
+individual contractors) to access the functionality of the Licensed Work
+by performing an action directly or indirectly that causes the creation
+of a topic in the Licensed Work. For clarity, a Streaming or Queuing
+Service would include providers of infrastructure services, such as
+cloud services, hosting services, data center services and similarly
+situated third parties (including affiliates of such entities) that
+would offer the Licensed Work in connection with a broader service
+offering to customers or subscribers of such of such third party’s core
+services.
+
+<u>Change Date</u>: Change date is four years from
+release date. Please see [GitHub releases](https://github.com/redpanda-data/redpanda/releases/)
+for exact dates
+
+<u>Change License</u>: [Apache License, Version
+2.0](https://www.apache.org/licenses/LICENSE-2.0), as published by the
+Apache Foundation.
+
+Text of BSL 1.1
+
+The Licensor hereby grants you the right to copy, modify, create
+derivative works, redistribute, and make non-production use of the
+Licensed Work. The Licensor may make an Additional Use Grant, above,
+permitting limited production use.
+
+Effective on the Change Date, or the fifth anniversary of the first
+publicly available distribution of a specific version of the Licensed
+Work under this License, whichever comes first, the Licensor hereby
+grants you rights under the terms of the Change License, and the rights
+granted in the paragraph above terminate.
+
+If your use of the Licensed Work does not comply with the requirements
+currently in effect as described in this License, you must purchase a
+commercial license from the Licensor, its affiliated entities, or
+authorized resellers, or you must refrain from using the Licensed Work.
+
+All copies of the original and modified Licensed Work, and derivative
+works of the Licensed Work, are subject to this License. This License
+applies separately for each version of the Licensed Work and the Change
+Date may vary for each version of the Licensed Work released by
+Licensor.
+
+You must conspicuously display this License on each original or modified
+copy of the Licensed Work. If you receive the Licensed Work in original
+or modified form from a third party, the terms and conditions set forth
+in this License apply to your use of that work.
+
+Any use of the Licensed Work in violation of this License will
+automatically terminate your rights under this License for the current
+and all other versions of the Licensed Work.
+
+This License does not grant you any right in any trademark or logo of
+Licensor or its affiliates (provided that you may use a trademark or
+logo of Licensor as expressly required by this License).TO THE EXTENT
+PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON AN “AS IS”
+BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, EXPRESS
+OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
+TITLE.

ocsf/README.md

@@ -0,0 +1,103 @@
+## OCSF
+
+`ocsf` turns the compiled [OCSF](https://schema.ocsf.io) (Open Cybersecurity
+Schema Framework) security-event schema into wire-stable proto3, and marshals
+generated events to schema-valid OCSF JSON.
+
+### Why
+
+OCSF ships no protobuf binding, and the schema is versioned JSON. This module
+generates proto3 for a chosen set of OCSF classes with **stable field numbers**
+across schema versions, so events can be stored and streamed as proto and
+exported as OCSF JSON at the boundary.
+
+### What a run produces
+
+Two artifacts are the point of this module; two more keep them safe to evolve.
+
+1. **Proto source files** under `ocsf/v<N>/`: one message per selected class
+   plus the merged single-event message, all sharing `objects.proto`. Fields
+   carry wire-stable numbers from the tagmap and `buf.validate` annotations
+   (required levels, plus class-aware CEL rules on the merged message), so
+   protovalidate enforces OCSF semantics on every event before it is
+   published. With `--merged-sr-subject`, the merged message also carries the
+   `(redpanda.api.common.v1.schema_registry)` annotation that drives
+   `protoc-gen-go-sr-normalize`.
+2. **Generated Go code**, produced from those protos with `buf generate` and
+   `protoc-gen-go`. The bindings committed under
+   `internal/ocsf/conformance/genpb/` are the reference: `buf.gen.yaml` and
+   `buf.gen.merged.yaml` in that directory hold the exact recipe. This module
+   does not export event types; consumers run the same recipe in their own
+   repo, pinned to their OCSF version and class selection, and get identical
+   wire-compatible types.
+3. **Schema Registry schemas** (`<name>.sr.proto`): self-contained copies with
+   no imports and no custom options, event message at Confluent index 0.
+   Register as-is.
+4. **The tagmap** (`field-numbers.json`): the append-only field-number map
+   that keeps every regeneration wire-compatible with the previous one,
+   enforced in CI by `--compat-check`.
+
+### Layouts
+
+The generator emits two layouts from one tagmap:
+
+- **Per-class** (always): one message per class (`ApiActivity`,
+  `EntityManagement`, ...), one file each, plus a shared `objects.proto`.
+- **Merged single-event** (`--merged-message AuditEvent`): ONE flat message
+  holding the union of all selected classes' attributes, for a single
+  audit-log topic carrying every class. Shared base_event attributes appear
+  once; class-scoped enums (`activity_id`) are demoted to plain scalars
+  because their values mean different things per class (semantics live in the
+  `(class_uid, activity_id)` pair and the unioned `TypeUid` enum); class-aware
+  validation is generated as protovalidate CEL: `type_uid == class_uid * 100 +
+  activity_id`, per-class field ownership, and per-class requiredness. A
+  merged event exports OCSF JSON identical to the per-class message.
+
+### Packages
+
+- `internal/ocsf/schema` — loads the OCSF `export/v2` compiled schema.
+- `internal/ocsf/gen` — maps OCSF types/enums to proto3 and emits deterministic
+  `.proto` with protovalidate; `MergeClasses`/`EmitMerged` produce the
+  single-event layout.
+- `internal/ocsf/tagmap` — the append-only `(message, attribute) -> tag` map and
+  its compatibility check (the wire-stability guarantee).
+- `internal/ocsf/exporter` — `ToOCSFJSON` marshals a generated proto message to
+  OCSF JSON (integer enums, numeric int64, snake_case keys); `FromOCSFJSON` is
+  its exact inverse (unknown keys land in `unmapped`), so events round-trip
+  proto → JSON → proto losslessly and JSON → proto → JSON byte-identically.
+- `cmd/ocsf-protogen` — the generator CLI.
+
+### CLI
+
+```
+go run ./cmd/ocsf-protogen \
+  --schema  internal/ocsf/schema/testdata/ocsf-1.8.0.json \
+  --classes api_activity,entity_management \
+  --version 1.8.0 \
+  --out     out-dir \
+  --tagmap  field-numbers.json \
+  --merged-message AuditEvent \
+  --sr-schema-out sr-dir
+```
+
+- `--merged-message <Name>` additionally emits the merged single-event layout
+  (`ocsf/v<N>/<snake_name>.proto`, and `<snake_name>.sr.proto` under
+  `--sr-schema-out`).
+- `--merged-sr-subject <subject>` annotates the merged message with
+  `(redpanda.api.common.v1.schema_registry) = { subject: "..." }` (from
+  `buf.build/redpandadata/common`). Consumers that run
+  [`protoc-gen-go-sr-normalize`](../protoc-gen-go-sr-normalize) in their buf
+  pass then get the self-contained SR schema and subject as Go constants
+  (`<Name>SRSchema`, `<Name>SRSubject`) with no hand-copied proto. Requires
+  `--merged-message`.
+- `--sr-schema-out <dir>` writes self-contained Schema-Registry schemas (no
+  imports to resolve; register as-is, event message at Confluent index 0).
+  This is the non-buf path to the same schema text; Go consumers should
+  prefer the annotation plus `protoc-gen-go-sr-normalize`.
+- `--check` regenerates and fails on output drift or newly-stubbed objects.
+- `--compat-check --old <a> --new <b>` fails if field numbers changed
+  incompatibly between two tag maps (used in CI against the base branch).
+
+Pinned to OCSF 1.8.0. The schema is fetched from
+`https://schema.ocsf.io/{version}/export/v2/schema`; generated events are
+conformance-validated against `https://schema.ocsf.io/{version}/api/v2/validate`.