From b996af9e0119b08caa486db549c364ada6eae8ed Mon Sep 17 00:00:00 2001 From: Yevhenii Basarab Date: Fri, 2 Oct 2026 16:10:27 +0200 Subject: [PATCH 1/2] fix: CU-17tkuw5tebe use release-bundled osquery schema only --- .github/workflows/test.yml | 4 +- charts/fleet/values.yaml | 5 - cmd/fleet/osquery_schema_refresh_openframe.go | 55 ------- .../osquery_schema_refresh_openframe_test.go | 77 --------- cmd/fleet/serve.go | 3 - openframe/docs/README.md | 2 +- openframe/docs/api-osquery-schema-search.md | 42 ++--- openframe/docs/fork-file-manifest.md | 8 +- schema/osquery_refresh.go | 43 ----- schema/osquery_refresh_test.go | 153 ------------------ schema/osquery_search.go | 28 +--- schema/osquery_search_test.go | 24 +++ server/config/config.go | 39 +---- server/config/config_openframe_test.go | 89 ++-------- 14 files changed, 59 insertions(+), 513 deletions(-) delete mode 100644 cmd/fleet/osquery_schema_refresh_openframe.go delete mode 100644 cmd/fleet/osquery_schema_refresh_openframe_test.go delete mode 100644 schema/osquery_refresh.go delete mode 100644 schema/osquery_refresh_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aa8f65f9c95..f63d56ca6f1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -166,8 +166,8 @@ jobs: with: install-only: true - - name: Test OpenFrame osquery schema search and refresh - run: go test -race ./schema ./server/config ./server/service ./cmd/fleet -run 'Test(SearchOsquery|RefreshOsquery|RankOsquery|RunOsquery|OpenframeConfig|OsquerySchema)' -count=1 + - name: Test OpenFrame release-bundled osquery schema search + run: go test -race ./schema ./server/config ./server/service ./cmd/fleet -run 'Test(SearchOsquery|ParseOsquery|NormalizeOsquery|SearchTerms|RankOsquery|OpenframeConfig)' -count=1 - name: Build with GoReleaser env: diff --git a/charts/fleet/values.yaml b/charts/fleet/values.yaml index bec6e35186c..b77be391163 100644 --- a/charts/fleet/values.yaml +++ b/charts/fleet/values.yaml @@ -448,11 +448,6 @@ environments: # you store this private key in a secret and use envsFrom to reference the secret below. # For more information, check out the docs: https://fleetdm.com/docs/configuration/fleet-server-configuration#server-private-key FLEET_SERVER_PRIVATE_KEY: "" - # >>> OPENFRAME(osquery-schema-search): keep the query-generation schema current in OpenFrame deployments - FLEET_OSQUERY_SCHEMA_REFRESH_ENABLED: "false" - FLEET_OSQUERY_SCHEMA_REFRESH_INTERVAL: "24h" - FLEET_OSQUERY_SCHEMA_REFRESH_URL: "https://raw.githubusercontent.com/flamingo-stack/fleetmdm/main/schema/osquery_fleet_schema.json" - # <<< OPENFRAME(osquery-schema-search) ## Section: Environment Variables from Secrets/CMs # envsFrom: diff --git a/cmd/fleet/osquery_schema_refresh_openframe.go b/cmd/fleet/osquery_schema_refresh_openframe.go deleted file mode 100644 index 0502ed8fba5..00000000000 --- a/cmd/fleet/osquery_schema_refresh_openframe.go +++ /dev/null @@ -1,55 +0,0 @@ -// OPENFRAME(osquery-schema-search): runs configurable per-replica osquery schema refreshes. -package main - -import ( - "context" - "log/slog" - "net/http" - "time" - - "github.com/fleetdm/fleet/v4/schema" - configpkg "github.com/fleetdm/fleet/v4/server/config" -) - -const osquerySchemaRefreshTimeout = 15 * time.Second - -type osquerySchemaRefreshFunc func(context.Context) (int, error) - -func startOsquerySchemaRefresh(ctx context.Context, config configpkg.OsqueryConfig, logger *slog.Logger) { - if !config.SchemaRefreshEnabled { - return - } - - client := &http.Client{Timeout: osquerySchemaRefreshTimeout} - go runOsquerySchemaRefresh(ctx, config.SchemaRefreshInterval, logger, func(ctx context.Context) (int, error) { - return schema.RefreshOsquerySchema(ctx, client, config.SchemaRefreshURL) - }) -} - -func runOsquerySchemaRefresh( - ctx context.Context, - interval time.Duration, - logger *slog.Logger, - refresh osquerySchemaRefreshFunc, -) { - refreshAndLog := func() { - count, err := refresh(ctx) - if err != nil { - logger.WarnContext(ctx, "osquery schema refresh failed; keeping previous schema", "err", err) - return - } - logger.InfoContext(ctx, "osquery schema refreshed", "tables", count) - } - - refreshAndLog() - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - refreshAndLog() - } - } -} diff --git a/cmd/fleet/osquery_schema_refresh_openframe_test.go b/cmd/fleet/osquery_schema_refresh_openframe_test.go deleted file mode 100644 index 4f876721671..00000000000 --- a/cmd/fleet/osquery_schema_refresh_openframe_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package main - -import ( - "context" - "errors" - "io" - "log/slog" - "sync/atomic" - "testing" - "testing/synctest" - "time" - - "github.com/stretchr/testify/require" -) - -func TestRunOsquerySchemaRefreshRunsImmediatelyAndOnInterval(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - var calls atomic.Int32 - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - - go runOsquerySchemaRefresh(ctx, time.Hour, discardOsquerySchemaRefreshLogger(), func(context.Context) (int, error) { - return int(calls.Add(1)), nil - }) - - synctest.Wait() - require.Equal(t, int32(1), calls.Load()) - - time.Sleep(time.Hour + time.Nanosecond) - synctest.Wait() - require.Equal(t, int32(2), calls.Load()) - }) -} - -func TestRunOsquerySchemaRefreshRetriesAfterFailure(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - var calls atomic.Int32 - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - - go runOsquerySchemaRefresh(ctx, time.Hour, discardOsquerySchemaRefreshLogger(), func(context.Context) (int, error) { - if calls.Add(1) == 1 { - return 0, errors.New("schema source unavailable") - } - return 1, nil - }) - - synctest.Wait() - require.Equal(t, int32(1), calls.Load()) - - time.Sleep(time.Hour + time.Nanosecond) - synctest.Wait() - require.Equal(t, int32(2), calls.Load()) - }) -} - -func TestRunOsquerySchemaRefreshStopsWhenContextIsCancelled(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - var calls atomic.Int32 - ctx, cancel := context.WithCancel(t.Context()) - - go runOsquerySchemaRefresh(ctx, time.Hour, discardOsquerySchemaRefreshLogger(), func(context.Context) (int, error) { - return int(calls.Add(1)), nil - }) - - synctest.Wait() - cancel() - synctest.Wait() - time.Sleep(time.Hour + time.Nanosecond) - synctest.Wait() - require.Equal(t, int32(1), calls.Load()) - }) -} - -func discardOsquerySchemaRefreshLogger() *slog.Logger { - return slog.New(slog.NewTextHandler(io.Discard, nil)) -} diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index 24621390966..9724c100dd9 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -430,9 +430,6 @@ func runServeCmd(cmd *cobra.Command, configManager configpkg.Manager, debug, dev baseCtx := licensectx.NewContext(context.Background(), license) ctx, cancelFunc := context.WithCancel(baseCtx) defer cancelFunc() - // >>> OPENFRAME(osquery-schema-search): start per-replica schema auto-refresh — openframe/docs/api-osquery-schema-search.md - startOsquerySchemaRefresh(ctx, config.Osquery, logger) - // <<< OPENFRAME(osquery-schema-search) // Channel used to trigger graceful shutdown on fatal DB errors (e.g. Aurora failover). dbFatalCh := make(chan error, 1) diff --git a/openframe/docs/README.md b/openframe/docs/README.md index a2e87428a3b..ef56de2a9f6 100644 --- a/openframe/docs/README.md +++ b/openframe/docs/README.md @@ -41,7 +41,7 @@ The agent has its own switch, `--openframe-mode` / `ORBIT_OPENFRAME_MODE`. | [api-host-assignments.md](api-host-assignments.md) | REST API for the above (add/remove/replace/list hosts). | | [managed-policies.md](managed-policies.md) | `policies.openframe_managed` — platform-owned policies omitted from the policy list/count endpoints (and from GitOps deletion) while still running on hosts and reporting results. | | [managed-queries.md](managed-queries.md) | `queries.openframe_managed` — the queries twin: platform-owned queries omitted from the query listing and its count. | -| [api-osquery-schema-search.md](api-osquery-schema-search.md) | Authenticated read-only search over Fleet's vendored canonical osquery schema for AI query generation. | +| [api-osquery-schema-search.md](api-osquery-schema-search.md) | Authenticated read-only search over the osquery schema bundled with the Fleet server release; no runtime downloads. | | [api-expose-osquery-host-id.md](api-expose-osquery-host-id.md) | Exposes `osquery_host_id` in the host JSON so the OpenFrame control plane can match agents. | | [query-results-ttl-cleanup.md](query-results-ttl-cleanup.md) | Time-based cleanup of `query_results` (keeps the Debezium CDC pipeline alive without unbounded growth). Gated by OpenFrame mode **and** a positive TTL. | | [redis-key-prefix.md](redis-key-prefix.md) | Per-tenant Redis key/channel prefix (`FLEET_REDIS_KEY_PREFIX`) so tenants can share one Redis. | diff --git a/openframe/docs/api-osquery-schema-search.md b/openframe/docs/api-osquery-schema-search.md index b0a9256da20..564ed47809a 100644 --- a/openframe/docs/api-osquery-schema-search.md +++ b/openframe/docs/api-osquery-schema-search.md @@ -1,31 +1,16 @@ # osquery Schema Search API The schema search endpoint gives OpenFrame services the canonical Fleet/osquery table and column -documentation they need before generating SQL. Search is local and deterministic. Fleet loads the -vendored `schema/osquery_fleet_schema.json` as an immediate fallback, then can refresh the in-memory -snapshot from a configured HTTP source without restarting. +documentation they need before generating SQL. Search is local and deterministic. Fleet loads only +the `schema/osquery_fleet_schema.json` embedded in its server binary. It does not download a schema +at startup, during search, or on a timer. The catalog changes only when a newly built Fleet server +containing an updated JSON file is deployed. -## Automatic refresh +## Release ownership -```yaml -osquery: - schema_refresh_enabled: true - schema_refresh_interval: 24h - schema_refresh_url: https://raw.githubusercontent.com/fleetdm/fleet//schema/osquery_fleet_schema.json -``` - -The equivalent environment variables are `FLEET_OSQUERY_SCHEMA_REFRESH_ENABLED`, -`FLEET_OSQUERY_SCHEMA_REFRESH_INTERVAL`, and `FLEET_OSQUERY_SCHEMA_REFRESH_URL`. Generic Fleet -configuration and the Fleet Helm chart keep refresh disabled and default to the upstream commit that -produced the embedded schema. OpenFrame environment overlays can opt in to a 24-hour refresh from -the reviewed `flamingo-stack/fleetmdm/main` schema. Do not point production deployments directly at -mutable upstream `fleetdm/fleet/main`; pin the source if a deployment must remain on an older schema. - -Each Fleet replica performs one best-effort refresh after startup and then retries at the configured -interval. A snapshot is replaced atomically only after a `200` response has been fully read and -validated as a non-empty schema. Network, HTTP, size, and JSON failures keep the previous snapshot, -so the embedded schema remains available during startup and outages. Search requests never perform -network calls. +The Fleet fork release owns the bundled schema. The agent release and rollout must keep device +osquery versions compatible with that schema; updating the Fleet server alone does not upgrade +device osquery. This endpoint does not select a historical schema by device version. ## Endpoint @@ -99,17 +84,12 @@ Darwin and Windows). ## Files changed -- `schema/osquery_search.go` — embedded fallback, atomic snapshot, and deterministic ranking. -- `schema/osquery_refresh.go` — bounded HTTP refresh and validation. -- `cmd/fleet/osquery_schema_refresh_openframe.go` — cancellable per-replica refresh loop. -- `server/config/config.go` — YAML, environment, and CLI configuration. -- `charts/fleet/values.yaml` — OpenFrame deployment defaults. +- `schema/osquery_search.go` — release-bundled schema, validation, and deterministic ranking. - `server/service/osquery_schema_openframe.go` — HTTP request/response and endpoint. - `server/service/handler.go` — route registration. - `server/api_endpoints/api_endpoints.yml` — API-only user allowlist catalog entry. ## Upstream sync notes -The schema search and refresh Go files are fork-added. Preserve the -`OPENFRAME(osquery-schema-search)` blocks in `server/config/config.go`, `cmd/fleet/serve.go`, -`server/service/handler.go`, and the Helm values when syncing upstream. +The schema search Go files are fork-added. Preserve the +`OPENFRAME(osquery-schema-search)` route block in `server/service/handler.go` when syncing upstream. diff --git a/openframe/docs/fork-file-manifest.md b/openframe/docs/fork-file-manifest.md index 2327fdb3cf1..3c6222fb581 100644 --- a/openframe/docs/fork-file-manifest.md +++ b/openframe/docs/fork-file-manifest.md @@ -131,7 +131,7 @@ and the heaviest standing rebase cost. | Area | Files | |------|-------| | Host assignments | `server/fleet/{policies,queries,hosts,datastore,service}.go`, `server/datastore/mysql/{policies,queries,hosts}.go`, `server/service/{global_policies,queries,handler,labels_util}.go`, `server/mock/{datastore,datastore_mock}.go`, `server/mock/service/service_mock.go`, `server/datastore/mysql/mysql.go`, `cmd/fleet/prepare.go` | -| osquery schema search and refresh | `schema/osquery_{search,refresh}.go`, `cmd/fleet/osquery_schema_refresh_openframe.go`, `cmd/fleet/serve.go`, `server/config/config.go`, `server/service/{osquery_schema_openframe,handler}.go`, `server/api_endpoints/api_endpoints.yml`, `charts/fleet/values.yaml` | +| osquery schema search | `schema/osquery_search.go`, `server/service/{osquery_schema_openframe,handler}.go`, `server/api_endpoints/api_endpoints.yml` | | Managed queries / policies | flag: `server/fleet/{queries,policies}.go`, `server/service/{queries,global_policies,team_policies}.go`, `server/datastore/mysql/{queries,policies}.go`, `schema.sql`; queries listing opt-in (`include_openframe_managed`): `server/fleet/{app,api_queries,service}.go`, `server/service/{global_schedule,team_schedule,queries_test}.go`, `server/mock/service/service_mock.go`; tests: `server/datastore/mysql/{queries,policies}_openframe_managed_test.go` — see [managed-queries.md](managed-queries.md), [managed-policies.md](managed-policies.md) | | osquery host id | `server/fleet/hosts.go` | | Query-results TTL cleanup | `server/config/config.go`, `server/fleet/{cron_schedules,datastore}.go`, `server/datastore/mysql/query_results.go`, `cmd/fleet/{cron,serve}.go` | @@ -188,7 +188,7 @@ Computed from the fork working tree vs the upstream baseline `server/datastore/mysql/migrations/data/` (the ~473 idempotent upstream migrations — see [migrations.md](migrations.md)). Paths are repo-root-relative. -### Added (49) +### Added (45) ``` .github/steps/sign-macos-package/action.yml @@ -202,8 +202,6 @@ charts/fleet/templates/configmap.yaml charts/fleet/templates/secret.yaml charts/fleet/templates/vulnprocessing/bind-job.yaml charts/fleet/templates/vulnprocessing/pvc.yaml -cmd/fleet/osquery_schema_refresh_openframe.go -cmd/fleet/osquery_schema_refresh_openframe_test.go openframe/docs/README.md openframe/docs/agent-openframe-mode.md openframe/docs/api-expose-osquery-host-id.md @@ -232,8 +230,6 @@ server/datastore/redis/keyprefix_test.go server/fleet/openframe.go schema/osquery_search.go schema/osquery_search_test.go -schema/osquery_refresh.go -schema/osquery_refresh_test.go server/service/osquery_schema_openframe.go server/service/osquery_schema_openframe_test.go server/service/openframe/openframe-encryption-service.go diff --git a/schema/osquery_refresh.go b/schema/osquery_refresh.go deleted file mode 100644 index 48989afac53..00000000000 --- a/schema/osquery_refresh.go +++ /dev/null @@ -1,43 +0,0 @@ -// OPENFRAME(osquery-schema-search): refreshes the searchable osquery schema while preserving the embedded fallback. -package schema - -import ( - "context" - "fmt" - "io" - "net/http" -) - -const maxOsquerySchemaSize = 16 * 1024 * 1024 - -func RefreshOsquerySchema(ctx context.Context, client *http.Client, sourceURL string) (int, error) { - request, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil) - if err != nil { - return 0, fmt.Errorf("build osquery schema request: %w", err) - } - request.Header.Set("User-Agent", "fleet-osquery-schema-refresh") - - response, err := client.Do(request) - if err != nil { - return 0, fmt.Errorf("fetch osquery schema: %w", err) - } - defer response.Body.Close() - if response.StatusCode != http.StatusOK { - return 0, fmt.Errorf("fetch osquery schema: HTTP %d", response.StatusCode) - } - - body, err := io.ReadAll(io.LimitReader(response.Body, maxOsquerySchemaSize+1)) - if err != nil { - return 0, fmt.Errorf("read osquery schema: %w", err) - } - if len(body) > maxOsquerySchemaSize { - return 0, fmt.Errorf("osquery schema exceeds %d bytes", maxOsquerySchemaSize) - } - - tables, err := parseOsquerySchemaJSON(body) - if err != nil { - return 0, err - } - storeOsqueryTables(tables) - return len(tables), nil -} diff --git a/schema/osquery_refresh_test.go b/schema/osquery_refresh_test.go deleted file mode 100644 index 018d072ecf1..00000000000 --- a/schema/osquery_refresh_test.go +++ /dev/null @@ -1,153 +0,0 @@ -package schema - -import ( - "context" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -const refreshedSchemaFixture = `[ - { - "name": "openframe_refresh_probe", - "platforms": ["all"], - "description": "OpenFrame schema refresh probe table", - "columns": [ - { - "name": "probe_value", - "type": "text", - "description": "Probe value", - "notes": null, - "required": false, - "hidden": null - } - ], - "examples": null, - "notes": null, - "evented": false, - "cacheable": false - } -]` - -func preserveOsquerySchema(t *testing.T) { - t.Helper() - tables, err := currentOsqueryTables() - require.NoError(t, err) - t.Cleanup(func() { - storeOsqueryTables(tables) - }) -} - -func TestRefreshOsquerySchemaReplacesSearchableSnapshot(t *testing.T) { - preserveOsquerySchema(t) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(refreshedSchemaFixture)) - })) - t.Cleanup(server.Close) - - count, err := RefreshOsquerySchema(t.Context(), server.Client(), server.URL) - require.NoError(t, err) - require.Equal(t, 1, count) - - tables, err := SearchOsqueryTables("openframe refresh probe", "all", 5) - require.NoError(t, err) - require.Len(t, tables, 1) - require.Equal(t, "openframe_refresh_probe", tables[0].Name) - require.Equal(t, "probe_value", tables[0].Columns[0].Name) -} - -func TestRefreshOsquerySchemaFailureKeepsPreviousSnapshot(t *testing.T) { - preserveOsquerySchema(t) - validServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(refreshedSchemaFixture)) - })) - t.Cleanup(validServer.Close) - _, err := RefreshOsquerySchema(t.Context(), validServer.Client(), validServer.URL) - require.NoError(t, err) - - tests := []struct { - name string - status int - body string - }{ - {name: "http error", status: http.StatusBadGateway, body: `{"message":"unavailable"}`}, - {name: "malformed json", status: http.StatusOK, body: `{`}, - {name: "empty schema", status: http.StatusOK, body: `[]`}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(tt.status) - _, _ = w.Write([]byte(tt.body)) - })) - t.Cleanup(server.Close) - - _, err := RefreshOsquerySchema(t.Context(), server.Client(), server.URL) - require.Error(t, err) - - tables, searchErr := SearchOsqueryTables("openframe refresh probe", "all", 5) - require.NoError(t, searchErr) - require.Len(t, tables, 1) - require.Equal(t, "openframe_refresh_probe", tables[0].Name) - }) - } -} - -func TestRefreshOsquerySchemaRejectsInvalidStructureAndKeepsPreviousSnapshot(t *testing.T) { - preserveOsquerySchema(t) - baseline, err := parseOsquerySchemaJSON([]byte(refreshedSchemaFixture)) - require.NoError(t, err) - - tests := []struct { - name string - body string - }{ - {name: "missing table name", body: `[{"name":"","columns":[{"name":"value","type":"text"}]}]`}, - {name: "missing columns", body: `[{"name":"broken_table","columns":[]}]`}, - {name: "duplicate table name", body: `[{"name":"duplicate","columns":[{"name":"one","type":"text"}]},{"name":"duplicate","columns":[{"name":"two","type":"text"}]}]`}, - {name: "missing column name", body: `[{"name":"broken_table","columns":[{"name":"","type":"text"}]}]`}, - {name: "missing column type", body: `[{"name":"broken_table","columns":[{"name":"value","type":""}]}]`}, - {name: "duplicate column name", body: `[{"name":"broken_table","columns":[{"name":"value","type":"text"},{"name":"value","type":"integer"}]}]`}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - storeOsqueryTables(baseline) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(tt.body)) - })) - t.Cleanup(server.Close) - - _, err := RefreshOsquerySchema(t.Context(), server.Client(), server.URL) - require.Error(t, err) - - tables, searchErr := SearchOsqueryTables("openframe refresh probe", "all", 5) - require.NoError(t, searchErr) - require.Len(t, tables, 1) - require.Equal(t, "openframe_refresh_probe", tables[0].Name) - }) - } -} - -func TestRefreshOsquerySchemaRejectsOversizedResponse(t *testing.T) { - preserveOsquerySchema(t) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _, _ = w.Write([]byte(strings.Repeat("x", maxOsquerySchemaSize+1))) - })) - t.Cleanup(server.Close) - - _, err := RefreshOsquerySchema(t.Context(), server.Client(), server.URL) - require.ErrorContains(t, err, "exceeds") -} - -func TestRefreshOsquerySchemaUsesRequestContext(t *testing.T) { - ctx, cancel := context.WithCancel(t.Context()) - cancel() - - _, err := RefreshOsquerySchema(ctx, http.DefaultClient, "https://schema.example.test/osquery.json") - require.ErrorIs(t, err, context.Canceled) -} diff --git a/schema/osquery_search.go b/schema/osquery_search.go index 109eadb3a7f..e4cab7cbfe3 100644 --- a/schema/osquery_search.go +++ b/schema/osquery_search.go @@ -10,7 +10,6 @@ import ( "slices" "sort" "strings" - "sync/atomic" ) const ( @@ -50,15 +49,11 @@ type rankedOsqueryTable struct { score int } -type osquerySchemaSnapshot struct { - tables []OsqueryTable -} - var ( //go:embed osquery_fleet_schema.json osquerySchemaJSON []byte - osquerySchemaState atomic.Pointer[osquerySchemaSnapshot] + osqueryTables []OsqueryTable nonSearchCharacter = regexp.MustCompile(`[^a-z0-9]+`) searchStopWords = map[string]struct{}{ @@ -73,7 +68,7 @@ func init() { if err != nil { panic(fmt.Sprintf("parse embedded osquery schema: %v", err)) } - storeOsqueryTables(tables) + osqueryTables = tables } // SearchOsqueryTables returns the canonical table definitions most relevant to a phrase. @@ -94,11 +89,6 @@ func SearchOsqueryTables(query, platform string, limit int) ([]OsqueryTable, err return nil, fmt.Errorf("limit must be between 1 and %d", MaxOsquerySearchLimit) } - tables, err := currentOsqueryTables() - if err != nil { - return nil, err - } - normalizedQuery := normalizeSearchText(query) terms := searchTerms(normalizedQuery, platform) if len(terms) == 0 { @@ -108,7 +98,7 @@ func SearchOsqueryTables(query, platform string, limit int) ([]OsqueryTable, err return nil, fmt.Errorf("query must not contain more than %d searchable terms", maxOsquerySearchTerms) } - platformTables := slices.DeleteFunc(slices.Clone(tables), func(table OsqueryTable) bool { + platformTables := slices.DeleteFunc(slices.Clone(osqueryTables), func(table OsqueryTable) bool { return !osqueryTableSupportsPlatform(table, platform) }) return rankOsqueryTables(platformTables, terms, normalizedQuery, limit), nil @@ -157,18 +147,6 @@ func parseOsquerySchemaJSON(data []byte) ([]OsqueryTable, error) { return tables, nil } -func currentOsqueryTables() ([]OsqueryTable, error) { - snapshot := osquerySchemaState.Load() - if snapshot == nil || len(snapshot.tables) == 0 { - return nil, errors.New("osquery schema is not loaded") - } - return snapshot.tables, nil -} - -func storeOsqueryTables(tables []OsqueryTable) { - osquerySchemaState.Store(&osquerySchemaSnapshot{tables: tables}) -} - // NormalizeOsqueryPlatform validates Fleet's public platform aliases. func NormalizeOsqueryPlatform(platform string) (string, error) { switch strings.ToLower(strings.TrimSpace(platform)) { diff --git a/schema/osquery_search_test.go b/schema/osquery_search_test.go index 1706fed86df..fa725f035fe 100644 --- a/schema/osquery_search_test.go +++ b/schema/osquery_search_test.go @@ -117,3 +117,27 @@ func TestNormalizeOsqueryPlatformAliases(t *testing.T) { func TestSearchTermsDeduplicatesTerms(t *testing.T) { require.Equal(t, []string{"processes", "ports"}, searchTerms("processes processes ports processes", "all")) } + +func TestParseOsquerySchemaJSONRejectsInvalidCatalog(t *testing.T) { + tests := []struct { + name string + body string + }{ + {name: "malformed json", body: `{`}, + {name: "empty schema", body: `[]`}, + {name: "missing table name", body: `[{"name":"","columns":[{"name":"value","type":"text"}]}]`}, + {name: "missing columns", body: `[{"name":"broken_table","columns":[]}]`}, + {name: "duplicate table name", body: `[{"name":"duplicate","columns":[{"name":"one","type":"text"}]},{"name":"duplicate","columns":[{"name":"two","type":"text"}]}]`}, + {name: "missing column name", body: `[{"name":"broken_table","columns":[{"name":"","type":"text"}]}]`}, + {name: "missing column type", body: `[{"name":"broken_table","columns":[{"name":"value","type":""}]}]`}, + {name: "duplicate column name", body: `[{"name":"broken_table","columns":[{"name":"value","type":"text"},{"name":"value","type":"integer"}]}]`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := parseOsquerySchemaJSON([]byte(tt.body)) + + require.Error(t, err) + }) + } +} diff --git a/server/config/config.go b/server/config/config.go index 61e0f3c552f..13bef061a55 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -267,11 +267,6 @@ type OsqueryConfig struct { LabelUpdateInterval time.Duration `yaml:"label_update_interval"` PolicyUpdateInterval time.Duration `yaml:"policy_update_interval"` DetailUpdateInterval time.Duration `yaml:"detail_update_interval"` - // >>> OPENFRAME(osquery-schema-search): configurable schema auto-refresh — openframe/docs/api-osquery-schema-search.md - SchemaRefreshEnabled bool `yaml:"schema_refresh_enabled"` - SchemaRefreshInterval time.Duration `yaml:"schema_refresh_interval"` - SchemaRefreshURL string `yaml:"schema_refresh_url"` - // <<< OPENFRAME(osquery-schema-search) // StatusLogFile is deprecated. It was replaced by FilesystemConfig.StatusLogFile. // @@ -332,19 +327,6 @@ func (o OsqueryConfig) Validate(initFatal func(err error, msg string)) { initFatal(fmt.Errorf("%s is not a valid value for osquery_host_identifier", o.HostIdentifier), "set host identifier") } - - // >>> OPENFRAME(osquery-schema-search): reject unsafe auto-refresh configuration — openframe/docs/api-osquery-schema-search.md - if o.SchemaRefreshEnabled { - if o.SchemaRefreshInterval <= 0 { - initFatal(errors.New("osquery schema refresh interval must be greater than zero"), "validate osquery schema refresh") - } else { - sourceURL, err := url.Parse(o.SchemaRefreshURL) - if err != nil || sourceURL.Host == "" || sourceURL.Scheme != "http" && sourceURL.Scheme != "https" { - initFatal(errors.New("osquery schema refresh URL must use http or https"), "validate osquery schema refresh") - } - } - } - // <<< OPENFRAME(osquery-schema-search) } // AsyncTaskName is the type of names that identify tasks supporting @@ -1471,14 +1453,6 @@ func (man Manager) addConfigs() { "Interval to update host policy membership (i.e. 1h)") man.addConfigDuration("osquery.detail_update_interval", 1*time.Hour, "Interval to update host details (i.e. 1h)") - // >>> OPENFRAME(osquery-schema-search): register schema auto-refresh flags — openframe/docs/api-osquery-schema-search.md - man.addConfigBool("osquery.schema_refresh_enabled", false, - "Refresh the in-memory osquery schema from a remote JSON source") - man.addConfigDuration("osquery.schema_refresh_interval", 24*time.Hour, - "Interval between osquery schema refresh attempts") - man.addConfigString("osquery.schema_refresh_url", "https://raw.githubusercontent.com/fleetdm/fleet/2cb8509c210a7443f715bc9c2b64fc7bc85fd5f7/schema/osquery_fleet_schema.json", - "URL of the canonical osquery schema JSON") - // <<< OPENFRAME(osquery-schema-search) man.addConfigString("osquery.status_log_file", "", "(DEPRECATED: Use filesystem.status_log_file) Path for osqueryd status logs") man.addConfigString("osquery.result_log_file", "", @@ -1961,15 +1935,10 @@ func (man Manager) LoadConfig() FleetConfig { // StatusLogFile is deprecated. FilesystemConfig.StatusLogFile is used instead. StatusLogFile: man.getConfigString("osquery.status_log_file"), // ResultLogFile is deprecated. FilesystemConfig.ResultLogFile is used instead. - ResultLogFile: man.getConfigString("osquery.result_log_file"), - LabelUpdateInterval: man.getConfigDuration("osquery.label_update_interval"), - PolicyUpdateInterval: man.getConfigDuration("osquery.policy_update_interval"), - DetailUpdateInterval: man.getConfigDuration("osquery.detail_update_interval"), - // >>> OPENFRAME(osquery-schema-search): read schema auto-refresh config — openframe/docs/api-osquery-schema-search.md - SchemaRefreshEnabled: man.getConfigBool("osquery.schema_refresh_enabled"), - SchemaRefreshInterval: man.getConfigDuration("osquery.schema_refresh_interval"), - SchemaRefreshURL: man.getConfigString("osquery.schema_refresh_url"), - // <<< OPENFRAME(osquery-schema-search) + ResultLogFile: man.getConfigString("osquery.result_log_file"), + LabelUpdateInterval: man.getConfigDuration("osquery.label_update_interval"), + PolicyUpdateInterval: man.getConfigDuration("osquery.policy_update_interval"), + DetailUpdateInterval: man.getConfigDuration("osquery.detail_update_interval"), EnableLogRotation: man.getConfigBool("osquery.enable_log_rotation"), MaxJitterPercent: man.getConfigInt("osquery.max_jitter_percent"), EnableAsyncHostProcessing: man.getConfigString("osquery.enable_async_host_processing"), diff --git a/server/config/config_openframe_test.go b/server/config/config_openframe_test.go index 9f796ce2f39..9764eaefca2 100644 --- a/server/config/config_openframe_test.go +++ b/server/config/config_openframe_test.go @@ -10,7 +10,6 @@ package config import ( "os" - "strings" "testing" "time" @@ -41,9 +40,6 @@ func TestOpenframeConfigDefaults(t *testing.T) { require.Empty(t, cfg.Redis.KeyPrefix, "key prefix must default to off (single-tenant Redis)") require.Equal(t, 60*24*time.Hour, cfg.Server.QueryResultsTTL) require.Equal(t, 1*time.Hour, cfg.Server.QueryResultsCleanupInterval) - require.False(t, cfg.Osquery.SchemaRefreshEnabled) - require.Equal(t, 24*time.Hour, cfg.Osquery.SchemaRefreshInterval) - require.Equal(t, "https://raw.githubusercontent.com/fleetdm/fleet/2cb8509c210a7443f715bc9c2b64fc7bc85fd5f7/schema/osquery_fleet_schema.json", cfg.Osquery.SchemaRefreshURL) } func TestOpenframeConfigEnvOverrides(t *testing.T) { @@ -51,88 +47,27 @@ func TestOpenframeConfigEnvOverrides(t *testing.T) { "FLEET_REDIS_KEY_PREFIX": "tenant-a", "FLEET_SERVER_QUERY_RESULTS_TTL": "24h", "FLEET_SERVER_QUERY_RESULTS_CLEANUP_INTERVAL": "30m", - "FLEET_OSQUERY_SCHEMA_REFRESH_ENABLED": "true", - "FLEET_OSQUERY_SCHEMA_REFRESH_INTERVAL": "6h", - "FLEET_OSQUERY_SCHEMA_REFRESH_URL": "https://schema.example.test/osquery.json", }) require.Equal(t, "tenant-a", cfg.Redis.KeyPrefix) require.Equal(t, 24*time.Hour, cfg.Server.QueryResultsTTL) require.Equal(t, 30*time.Minute, cfg.Server.QueryResultsCleanupInterval) - require.True(t, cfg.Osquery.SchemaRefreshEnabled) - require.Equal(t, 6*time.Hour, cfg.Osquery.SchemaRefreshInterval) - require.Equal(t, "https://schema.example.test/osquery.json", cfg.Osquery.SchemaRefreshURL) } -func TestOpenframeConfigYamlOverridesOsquerySchemaRefresh(t *testing.T) { - testutils.SaveEnv(t) - os.Clearenv() +func TestOpenframeConfigRejectsRemovedOsquerySchemaRefreshFlags(t *testing.T) { + for _, flag := range []string{ + "--osquery_schema_refresh_enabled=true", + "--osquery_schema_refresh_interval=24h", + "--osquery_schema_refresh_url=https://schema.example.test/osquery.json", + } { + t.Run(flag, func(t *testing.T) { + cmd := &cobra.Command{} + cmd.PersistentFlags().StringP("config", "c", "", "Path to a configuration file") + NewManager(cmd) - cmd := &cobra.Command{} - cmd.PersistentFlags().StringP("config", "c", "", "Path to a configuration file") - man := NewManager(cmd) - man.viper.SetConfigType("yaml") - require.NoError(t, man.viper.ReadConfig(strings.NewReader(` -osquery: - schema_refresh_enabled: true - schema_refresh_interval: 12h - schema_refresh_url: https://schema.example.test/fleet.json -`))) - - cfg := man.LoadConfig() - require.True(t, cfg.Osquery.SchemaRefreshEnabled) - require.Equal(t, 12*time.Hour, cfg.Osquery.SchemaRefreshInterval) - require.Equal(t, "https://schema.example.test/fleet.json", cfg.Osquery.SchemaRefreshURL) -} - -func TestOsquerySchemaRefreshConfigValidation(t *testing.T) { - tests := []struct { - name string - config OsqueryConfig - errorReason string - }{ - { - name: "enabled with zero interval", - config: OsqueryConfig{ - HostIdentifier: "provided", - SchemaRefreshEnabled: true, - SchemaRefreshInterval: 0, - SchemaRefreshURL: "https://schema.example.test/osquery.json", - }, - errorReason: "interval must be greater than zero", - }, - { - name: "enabled with unsupported URL scheme", - config: OsqueryConfig{ - HostIdentifier: "provided", - SchemaRefreshEnabled: true, - SchemaRefreshInterval: time.Hour, - SchemaRefreshURL: "file:///tmp/osquery.json", - }, - errorReason: "URL must use http or https", - }, - { - name: "disabled ignores refresh settings", - config: OsqueryConfig{ - HostIdentifier: "provided", - SchemaRefreshEnabled: false, - SchemaRefreshInterval: 0, - SchemaRefreshURL: "", - }, - }, - } + err := cmd.ParseFlags([]string{flag}) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var validationErr error - tt.config.Validate(func(err error, _ string) { - validationErr = err - }) - if tt.errorReason == "" { - require.NoError(t, validationErr) - return - } - require.ErrorContains(t, validationErr, tt.errorReason) + require.ErrorContains(t, err, "unknown flag") }) } } From ed2aa81ce96cec3e13197cc41ed52f435176e760 Mon Sep 17 00:00:00 2001 From: Yevhenii Basarab Date: Fri, 2 Oct 2026 20:36:10 +0200 Subject: [PATCH 2/2] refactor: CU-17tkuw5tebe retire Fleet osquery schema API --- .github/workflows/changes.yaml | 2 +- .github/workflows/test.yml | 4 +- openframe/docs/README.md | 1 - openframe/docs/api-osquery-schema-search.md | 95 ------ openframe/docs/fork-file-manifest.md | 12 +- openframe/scripts/verify.sh | 2 +- schema/osquery_search.go | 279 ------------------ schema/osquery_search_test.go | 143 --------- server/api_endpoints/api_endpoints.yml | 3 - server/service/handler.go | 3 - server/service/handler_test.go | 32 ++ server/service/osquery_schema_openframe.go | 55 ---- .../service/osquery_schema_openframe_test.go | 114 ------- 13 files changed, 39 insertions(+), 706 deletions(-) delete mode 100644 openframe/docs/api-osquery-schema-search.md delete mode 100644 schema/osquery_search.go delete mode 100644 schema/osquery_search_test.go delete mode 100644 server/service/osquery_schema_openframe.go delete mode 100644 server/service/osquery_schema_openframe_test.go diff --git a/.github/workflows/changes.yaml b/.github/workflows/changes.yaml index d70a9b5eaba..6343c51dbb5 100644 --- a/.github/workflows/changes.yaml +++ b/.github/workflows/changes.yaml @@ -37,7 +37,7 @@ jobs: { "id": "server", "name": "fleet", - "paths": ["./server/**", "./cmd/**", "./pkg/**", "./schema/**", "./ee/**", "./tools/**", "./Dockerfile", "./.goreleaser.yml"] + "paths": ["./server/**", "./cmd/**", "./pkg/**", "./ee/**", "./tools/**", "./Dockerfile", "./.goreleaser.yml"] } ]' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f63d56ca6f1..7ada687ecee 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -166,8 +166,8 @@ jobs: with: install-only: true - - name: Test OpenFrame release-bundled osquery schema search - run: go test -race ./schema ./server/config ./server/service ./cmd/fleet -run 'Test(SearchOsquery|ParseOsquery|NormalizeOsquery|SearchTerms|RankOsquery|OpenframeConfig)' -count=1 + - name: Test OpenFrame configuration and schema endpoint retirement + run: go test -race ./server/config ./server/service ./cmd/fleet -run 'Test(OpenframeConfig|OpenframeOsquerySchemaRoute)' -count=1 - name: Build with GoReleaser env: diff --git a/openframe/docs/README.md b/openframe/docs/README.md index ef56de2a9f6..c36f6aef53c 100644 --- a/openframe/docs/README.md +++ b/openframe/docs/README.md @@ -41,7 +41,6 @@ The agent has its own switch, `--openframe-mode` / `ORBIT_OPENFRAME_MODE`. | [api-host-assignments.md](api-host-assignments.md) | REST API for the above (add/remove/replace/list hosts). | | [managed-policies.md](managed-policies.md) | `policies.openframe_managed` — platform-owned policies omitted from the policy list/count endpoints (and from GitOps deletion) while still running on hosts and reporting results. | | [managed-queries.md](managed-queries.md) | `queries.openframe_managed` — the queries twin: platform-owned queries omitted from the query listing and its count. | -| [api-osquery-schema-search.md](api-osquery-schema-search.md) | Authenticated read-only search over the osquery schema bundled with the Fleet server release; no runtime downloads. | | [api-expose-osquery-host-id.md](api-expose-osquery-host-id.md) | Exposes `osquery_host_id` in the host JSON so the OpenFrame control plane can match agents. | | [query-results-ttl-cleanup.md](query-results-ttl-cleanup.md) | Time-based cleanup of `query_results` (keeps the Debezium CDC pipeline alive without unbounded growth). Gated by OpenFrame mode **and** a positive TTL. | | [redis-key-prefix.md](redis-key-prefix.md) | Per-tenant Redis key/channel prefix (`FLEET_REDIS_KEY_PREFIX`) so tenants can share one Redis. | diff --git a/openframe/docs/api-osquery-schema-search.md b/openframe/docs/api-osquery-schema-search.md deleted file mode 100644 index 564ed47809a..00000000000 --- a/openframe/docs/api-osquery-schema-search.md +++ /dev/null @@ -1,95 +0,0 @@ -# osquery Schema Search API - -The schema search endpoint gives OpenFrame services the canonical Fleet/osquery table and column -documentation they need before generating SQL. Search is local and deterministic. Fleet loads only -the `schema/osquery_fleet_schema.json` embedded in its server binary. It does not download a schema -at startup, during search, or on a timer. The catalog changes only when a newly built Fleet server -containing an updated JSON file is deployed. - -## Release ownership - -The Fleet fork release owns the bundled schema. The agent release and rollout must keep device -osquery versions compatible with that schema; updating the Fleet server alone does not upgrade -device osquery. This endpoint does not select a historical schema by device version. - -## Endpoint - -```http -GET /api/v1/fleet/osquery/schema/search?query=windows%20updates&platform=windows&limit=20 -Authorization: Bearer -X-Tenant-Id: -``` - -The endpoint uses Fleet's existing user-authenticated route middleware and is included in the API -endpoint catalog for restricted API-only users. In shared OpenFrame mode, the existing tenant -middleware also requires `X-Tenant-Id`, even though the schema itself is global. - -## Query parameters - -| Parameter | Required | Default | Description | -|-----------|----------|---------|-------------| -| `query` | yes | — | Non-empty English search phrase, at most 512 bytes and 32 unique searchable terms. Table names, descriptions, examples, notes, and column documentation are searched. | -| `platform` | no | `all` | `all`, `darwin`/`macos`, `windows`, `linux`, or `chrome`/`chromeos`. | -| `limit` | no | `20` | Maximum returned tables, from `1` through `50`. | - -Results are ordered by deterministic relevance score and then table name. The platform filter is -applied before ranking. A successful search with no matches returns `tables: []` and `count: 0`. - -## Response - -```json -{ - "query": "windows updates", - "platform": "windows", - "count": 1, - "tables": [ - { - "name": "windows_update_history", - "platforms": ["windows"], - "description": "Provides the history of the windows update events.", - "columns": [ - { - "name": "result_code", - "type": "text", - "description": "Result of an operation on an update", - "notes": null, - "required": false, - "hidden": false, - "index": false - } - ], - "examples": "select * from windows_update_history", - "notes": null, - "url": "https://fleetdm.com/tables/windows_update_history", - "evented": false, - "cacheable": false - } - ] -} -``` - -The API preserves nullable canonical fields: table `examples` and `notes`, plus column `notes` and -`hidden`, may be `null`. Column `required` is always a boolean. When the canonical schema restricts -a column more narrowly than its table, `columns[].platforms` preserves that platform list and its -original case (for example, `battery.health` is `macOS`-only even though `battery` supports both -Darwin and Windows). - -## Errors - -| Status | Meaning | -|--------|---------| -| `400` | Empty/unsearchable or oversized query, unsupported platform, or limit outside `1..50` | -| `401` | Missing/invalid Fleet token or, in shared mode, missing/invalid tenant header | -| `403` | Authenticated principal is not allowed to call user API endpoints | - -## Files changed - -- `schema/osquery_search.go` — release-bundled schema, validation, and deterministic ranking. -- `server/service/osquery_schema_openframe.go` — HTTP request/response and endpoint. -- `server/service/handler.go` — route registration. -- `server/api_endpoints/api_endpoints.yml` — API-only user allowlist catalog entry. - -## Upstream sync notes - -The schema search Go files are fork-added. Preserve the -`OPENFRAME(osquery-schema-search)` route block in `server/service/handler.go` when syncing upstream. diff --git a/openframe/docs/fork-file-manifest.md b/openframe/docs/fork-file-manifest.md index 3c6222fb581..04f777a7c5d 100644 --- a/openframe/docs/fork-file-manifest.md +++ b/openframe/docs/fork-file-manifest.md @@ -108,7 +108,6 @@ never had was relocated into this directory): README.md agent-openframe-mode.md architecture-host-assignments.md api-host-assignments.md api-expose-osquery-host-id.md query-results-ttl-cleanup.md -api-osquery-schema-search.md redis-key-prefix.md migrations.md helm-chart.md ci-cd-release-pipeline.md node-key-management.md local-setup.md @@ -131,7 +130,6 @@ and the heaviest standing rebase cost. | Area | Files | |------|-------| | Host assignments | `server/fleet/{policies,queries,hosts,datastore,service}.go`, `server/datastore/mysql/{policies,queries,hosts}.go`, `server/service/{global_policies,queries,handler,labels_util}.go`, `server/mock/{datastore,datastore_mock}.go`, `server/mock/service/service_mock.go`, `server/datastore/mysql/mysql.go`, `cmd/fleet/prepare.go` | -| osquery schema search | `schema/osquery_search.go`, `server/service/{osquery_schema_openframe,handler}.go`, `server/api_endpoints/api_endpoints.yml` | | Managed queries / policies | flag: `server/fleet/{queries,policies}.go`, `server/service/{queries,global_policies,team_policies}.go`, `server/datastore/mysql/{queries,policies}.go`, `schema.sql`; queries listing opt-in (`include_openframe_managed`): `server/fleet/{app,api_queries,service}.go`, `server/service/{global_schedule,team_schedule,queries_test}.go`, `server/mock/service/service_mock.go`; tests: `server/datastore/mysql/{queries,policies}_openframe_managed_test.go` — see [managed-queries.md](managed-queries.md), [managed-policies.md](managed-policies.md) | | osquery host id | `server/fleet/hosts.go` | | Query-results TTL cleanup | `server/config/config.go`, `server/fleet/{cron_schedules,datastore}.go`, `server/datastore/mysql/query_results.go`, `cmd/fleet/{cron,serve}.go` | @@ -188,7 +186,7 @@ Computed from the fork working tree vs the upstream baseline `server/datastore/mysql/migrations/data/` (the ~473 idempotent upstream migrations — see [migrations.md](migrations.md)). Paths are repo-root-relative. -### Added (45) +### Added (40) ``` .github/steps/sign-macos-package/action.yml @@ -206,7 +204,6 @@ openframe/docs/README.md openframe/docs/agent-openframe-mode.md openframe/docs/api-expose-osquery-host-id.md openframe/docs/api-host-assignments.md -openframe/docs/api-osquery-schema-search.md openframe/docs/architecture-host-assignments.md openframe/docs/ci-cd-release-pipeline.md openframe/docs/fork-file-manifest.md @@ -228,17 +225,13 @@ server/datastore/mysql/migrations_openframe_test.go server/datastore/redis/keyprefix.go server/datastore/redis/keyprefix_test.go server/fleet/openframe.go -schema/osquery_search.go -schema/osquery_search_test.go -server/service/osquery_schema_openframe.go -server/service/osquery_schema_openframe_test.go server/service/openframe/openframe-encryption-service.go server/service/openframe/openframe-token-extractor.go server/service/openframe/openframe_authorization_manager.go server/service/openframe/openframe_token_refresher.go ``` -### Modified (52) +### Modified (53) ``` .github/pull_request_template.md @@ -287,6 +280,7 @@ server/service/base_client.go server/service/global_policies.go server/service/global_schedule.go server/service/handler.go +server/service/handler_test.go server/service/labels_util.go server/service/orbit_client.go server/service/osquery_utils/queries.go diff --git a/openframe/scripts/verify.sh b/openframe/scripts/verify.sh index 6cd4bf62f24..aede2fdd65c 100755 --- a/openframe/scripts/verify.sh +++ b/openframe/scripts/verify.sh @@ -46,7 +46,7 @@ rm -f vet.err # 3. Marker presence: if a merge silently dropped fork code, its OPENFRAME markers # vanish too. A slug dropping to zero is a red flag worth a human look. step "OPENFRAME marker presence (dropped-fork-code detector)" -for slug in host-assignments managed-policies managed-queries redis-key-prefix redis-seed-nodes query-results-ttl osquery-host-id osquery-schema-search agent-openframe-mode agent-json-content-type agent-skip-setup-experience migration-race; do +for slug in host-assignments managed-policies managed-queries redis-key-prefix redis-seed-nodes query-results-ttl osquery-host-id agent-openframe-mode agent-json-content-type agent-skip-setup-experience migration-race; do n=$(grep -rIl "OPENFRAME($slug" --include='*.go' --include='*.yaml' --include='*.tpl' . 2>/dev/null | wc -l | tr -d ' ') if [ "$n" -gt 0 ]; then ok "$slug — present in $n file(s)"; else bad "$slug — NO markers found (fork code may have been dropped in the merge)"; fi done diff --git a/schema/osquery_search.go b/schema/osquery_search.go deleted file mode 100644 index e4cab7cbfe3..00000000000 --- a/schema/osquery_search.go +++ /dev/null @@ -1,279 +0,0 @@ -// Package schema exposes Fleet's vendored osquery schema to server-side consumers. -package schema - -import ( - _ "embed" - "encoding/json" - "errors" - "fmt" - "regexp" - "slices" - "sort" - "strings" -) - -const ( - DefaultOsquerySearchLimit = 20 - MaxOsquerySearchLimit = 50 - maxOsquerySearchQueryBytes = 512 - maxOsquerySearchTerms = 32 -) - -// OsqueryTable is the searchable subset of a table in Fleet's canonical schema. -type OsqueryTable struct { - Name string `json:"name"` - Platforms []string `json:"platforms"` - Description string `json:"description"` - Columns []OsqueryColumn `json:"columns"` - Examples *string `json:"examples"` - Notes *string `json:"notes"` - URL string `json:"url,omitempty"` - Evented bool `json:"evented"` - Cacheable bool `json:"cacheable"` -} - -// OsqueryColumn describes a column, including constraints required to query it. -type OsqueryColumn struct { - Name string `json:"name"` - Type string `json:"type"` - Description string `json:"description"` - Platforms []string `json:"platforms,omitempty"` - Notes *string `json:"notes"` - Required bool `json:"required"` - Hidden *bool `json:"hidden"` - Index *bool `json:"index,omitempty"` -} - -type rankedOsqueryTable struct { - table OsqueryTable - score int -} - -var ( - //go:embed osquery_fleet_schema.json - osquerySchemaJSON []byte - - osqueryTables []OsqueryTable - - nonSearchCharacter = regexp.MustCompile(`[^a-z0-9]+`) - searchStopWords = map[string]struct{}{ - "a": {}, "an": {}, "and": {}, "are": {}, "for": {}, "from": {}, - "get": {}, "how": {}, "in": {}, "is": {}, "of": {}, "on": {}, - "show": {}, "the": {}, "to": {}, "with": {}, - } -) - -func init() { - tables, err := parseOsquerySchemaJSON(osquerySchemaJSON) - if err != nil { - panic(fmt.Sprintf("parse embedded osquery schema: %v", err)) - } - osqueryTables = tables -} - -// SearchOsqueryTables returns the canonical table definitions most relevant to a phrase. -func SearchOsqueryTables(query, platform string, limit int) ([]OsqueryTable, error) { - query = strings.TrimSpace(query) - if query == "" { - return nil, errors.New("query must not be empty") - } - if len(query) > maxOsquerySearchQueryBytes { - return nil, fmt.Errorf("query must not exceed %d bytes", maxOsquerySearchQueryBytes) - } - - platform, err := NormalizeOsqueryPlatform(platform) - if err != nil { - return nil, err - } - if limit < 1 || limit > MaxOsquerySearchLimit { - return nil, fmt.Errorf("limit must be between 1 and %d", MaxOsquerySearchLimit) - } - - normalizedQuery := normalizeSearchText(query) - terms := searchTerms(normalizedQuery, platform) - if len(terms) == 0 { - return nil, errors.New("query must contain a searchable term") - } - if len(terms) > maxOsquerySearchTerms { - return nil, fmt.Errorf("query must not contain more than %d searchable terms", maxOsquerySearchTerms) - } - - platformTables := slices.DeleteFunc(slices.Clone(osqueryTables), func(table OsqueryTable) bool { - return !osqueryTableSupportsPlatform(table, platform) - }) - return rankOsqueryTables(platformTables, terms, normalizedQuery, limit), nil -} - -func parseOsquerySchemaJSON(data []byte) ([]OsqueryTable, error) { - var tables []OsqueryTable - if err := json.Unmarshal(data, &tables); err != nil { - return nil, fmt.Errorf("parse osquery schema: %w", err) - } - if len(tables) == 0 { - return nil, errors.New("osquery schema contains no tables") - } - - tableNames := make(map[string]struct{}, len(tables)) - for tableIndex, table := range tables { - tableName := strings.TrimSpace(table.Name) - if tableName == "" { - return nil, fmt.Errorf("osquery schema table %d has no name", tableIndex) - } - normalizedTableName := strings.ToLower(tableName) - if _, ok := tableNames[normalizedTableName]; ok { - return nil, fmt.Errorf("osquery schema contains duplicate table %q", tableName) - } - tableNames[normalizedTableName] = struct{}{} - if len(table.Columns) == 0 { - return nil, fmt.Errorf("osquery schema table %q has no columns", tableName) - } - - columnNames := make(map[string]struct{}, len(table.Columns)) - for columnIndex, column := range table.Columns { - columnName := strings.TrimSpace(column.Name) - if columnName == "" { - return nil, fmt.Errorf("osquery schema table %q column %d has no name", tableName, columnIndex) - } - if strings.TrimSpace(column.Type) == "" { - return nil, fmt.Errorf("osquery schema table %q column %q has no type", tableName, columnName) - } - normalizedColumnName := strings.ToLower(columnName) - if _, ok := columnNames[normalizedColumnName]; ok { - return nil, fmt.Errorf("osquery schema table %q contains duplicate column %q", tableName, columnName) - } - columnNames[normalizedColumnName] = struct{}{} - } - } - return tables, nil -} - -// NormalizeOsqueryPlatform validates Fleet's public platform aliases. -func NormalizeOsqueryPlatform(platform string) (string, error) { - switch strings.ToLower(strings.TrimSpace(platform)) { - case "", "all": - return "all", nil - case "darwin", "macos": - return "darwin", nil - case "windows", "linux": - return strings.ToLower(strings.TrimSpace(platform)), nil - case "chrome", "chromeos": - return "chrome", nil - default: - return "", fmt.Errorf("unsupported platform %q", platform) - } -} - -func searchTerms(normalizedQuery, platform string) []string { - terms := strings.Fields(normalizedQuery) - terms = slices.DeleteFunc(terms, func(term string) bool { - _, stopWord := searchStopWords[term] - return len(term) < 2 || stopWord || term == platform || (platform == "darwin" && term == "macos") || (platform == "chrome" && term == "chromeos") - }) - - seen := make(map[string]struct{}, len(terms)) - return slices.DeleteFunc(terms, func(term string) bool { - if _, ok := seen[term]; ok { - return true - } - seen[term] = struct{}{} - return false - }) -} - -func osqueryTableSupportsPlatform(table OsqueryTable, platform string) bool { - if platform == "all" || len(table.Platforms) == 0 { - return true - } - return slices.ContainsFunc(table.Platforms, func(tablePlatform string) bool { - normalized, err := NormalizeOsqueryPlatform(tablePlatform) - return err == nil && normalized == platform - }) -} - -func rankOsqueryTables(tables []OsqueryTable, terms []string, normalizedQuery string, limit int) []OsqueryTable { - ranked := make([]rankedOsqueryTable, 0, len(tables)) - for _, table := range tables { - score := osqueryTableSearchScore(table, terms, normalizedQuery) - if score > 0 { - ranked = append(ranked, rankedOsqueryTable{table: table, score: score}) - } - } - - sort.Slice(ranked, func(i, j int) bool { - if ranked[i].score == ranked[j].score { - return ranked[i].table.Name < ranked[j].table.Name - } - return ranked[i].score > ranked[j].score - }) - if len(ranked) > limit { - ranked = ranked[:limit] - } - - results := make([]OsqueryTable, 0, len(ranked)) - for _, result := range ranked { - results = append(results, result.table) - } - return results -} - -func osqueryTableSearchScore(table OsqueryTable, terms []string, normalizedQuery string) int { - tableName := normalizeSearchText(table.Name) - score := phraseScore(tableName, normalizedQuery, 1_000, 500) - score += phraseScore(normalizeSearchText(table.Description), normalizedQuery, 0, 100) - - for _, term := range terms { - score += termScore(tableName, term, 120, 70) - score += containsScore(normalizeSearchText(table.Description), term, 20) - score += containsScore(normalizeNullableSearchText(table.Examples), term, 8) - score += containsScore(normalizeNullableSearchText(table.Notes), term, 5) - for _, column := range table.Columns { - score += termScore(normalizeSearchText(column.Name), term, 100, 50) - score += containsScore(normalizeSearchText(column.Description), term, 15) - score += containsScore(normalizeNullableSearchText(column.Notes), term, 5) - } - } - return score -} - -func phraseScore(text, phrase string, exactScore, containsScore int) int { - if phrase == "" { - return 0 - } - if text == phrase { - return exactScore - } - if strings.Contains(text, phrase) { - return containsScore - } - return 0 -} - -func termScore(text, term string, exactScore, containsWeight int) int { - if text == term { - return exactScore - } - for _, word := range strings.Fields(text) { - if word == term { - return exactScore - } - } - return containsScore(text, term, containsWeight) -} - -func containsScore(text, term string, score int) int { - if strings.Contains(text, term) { - return score - } - return 0 -} - -func normalizeNullableSearchText(value *string) string { - if value == nil { - return "" - } - return normalizeSearchText(*value) -} - -func normalizeSearchText(value string) string { - return strings.TrimSpace(nonSearchCharacter.ReplaceAllString(strings.ToLower(value), " ")) -} diff --git a/schema/osquery_search_test.go b/schema/osquery_search_test.go deleted file mode 100644 index fa725f035fe..00000000000 --- a/schema/osquery_search_test.go +++ /dev/null @@ -1,143 +0,0 @@ -package schema - -import ( - "fmt" - "strings" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestSearchOsqueryTablesRanksExactTableAndReturnsColumnTypes(t *testing.T) { - tables, err := SearchOsqueryTables("windows update result code", "windows", 5) - require.NoError(t, err) - require.NotEmpty(t, tables) - require.Equal(t, "windows_update_history", tables[0].Name) - - var resultCode *OsqueryColumn - for i := range tables[0].Columns { - if tables[0].Columns[i].Name == "result_code" { - resultCode = &tables[0].Columns[i] - break - } - } - require.NotNil(t, resultCode) - require.Equal(t, "text", resultCode.Type) -} - -func TestSearchOsqueryTablesFiltersByPlatform(t *testing.T) { - tables, err := SearchOsqueryTables("windows update history", "darwin", 20) - require.NoError(t, err) - for _, table := range tables { - require.NotEqual(t, "windows_update_history", table.Name) - } -} - -func TestSearchOsqueryTablesPreservesNullableCanonicalFields(t *testing.T) { - tables, err := SearchOsqueryTables("acpi tables", "darwin", 1) - require.NoError(t, err) - require.Len(t, tables, 1) - require.Equal(t, "acpi_tables", tables[0].Name) - require.Nil(t, tables[0].Examples) - - tables, err = SearchOsqueryTables("adobe plugins", "darwin", 1) - require.NoError(t, err) - require.Len(t, tables, 1) - require.Equal(t, "adobe_plugins", tables[0].Name) - require.NotEmpty(t, tables[0].Columns) - require.Nil(t, tables[0].Columns[0].Hidden) -} - -func TestSearchOsqueryTablesPreservesColumnPlatforms(t *testing.T) { - tables, err := SearchOsqueryTables("battery", "darwin", 1) - require.NoError(t, err) - require.Len(t, tables, 1) - require.Equal(t, "battery", tables[0].Name) - - var healthColumn *OsqueryColumn - for i := range tables[0].Columns { - if tables[0].Columns[i].Name == "health" { - healthColumn = &tables[0].Columns[i] - break - } - } - require.NotNil(t, healthColumn) - require.Equal(t, []string{"macOS"}, healthColumn.Platforms) -} - -func TestRankOsqueryTablesUsesNameOrderForEqualScores(t *testing.T) { - tables := []OsqueryTable{ - {Name: "z_table", Description: "firmware details"}, - {Name: "a_table", Description: "firmware details"}, - } - - ranked := rankOsqueryTables(tables, []string{"firmware"}, "firmware", 2) - require.Equal(t, []string{"a_table", "z_table"}, []string{ranked[0].Name, ranked[1].Name}) -} - -func TestSearchOsqueryTablesValidatesInput(t *testing.T) { - _, err := SearchOsqueryTables(" ", "all", 20) - require.ErrorContains(t, err, "query must not be empty") - - _, err = SearchOsqueryTables("updates", "freebsd", 20) - require.ErrorContains(t, err, "unsupported platform") - - _, err = SearchOsqueryTables("updates", "windows", 51) - require.ErrorContains(t, err, "limit must be between 1 and 50") - - _, err = SearchOsqueryTables(strings.Repeat("x", maxOsquerySearchQueryBytes+1), "all", 20) - require.ErrorContains(t, err, "query must not exceed 512 bytes") - - terms := make([]string, maxOsquerySearchTerms+1) - for i := range terms { - terms[i] = fmt.Sprintf("term%d", i) - } - _, err = SearchOsqueryTables(strings.Join(terms, " "), "all", 20) - require.ErrorContains(t, err, "query must not contain more than 32 searchable terms") -} - -func TestNormalizeOsqueryPlatformAliases(t *testing.T) { - tests := []struct { - platform string - expected string - }{ - {platform: "macos", expected: "darwin"}, - {platform: "chromeos", expected: "chrome"}, - } - - for _, tt := range tests { - t.Run(tt.platform, func(t *testing.T) { - platform, err := NormalizeOsqueryPlatform(tt.platform) - require.NoError(t, err) - require.Equal(t, tt.expected, platform) - }) - } -} - -func TestSearchTermsDeduplicatesTerms(t *testing.T) { - require.Equal(t, []string{"processes", "ports"}, searchTerms("processes processes ports processes", "all")) -} - -func TestParseOsquerySchemaJSONRejectsInvalidCatalog(t *testing.T) { - tests := []struct { - name string - body string - }{ - {name: "malformed json", body: `{`}, - {name: "empty schema", body: `[]`}, - {name: "missing table name", body: `[{"name":"","columns":[{"name":"value","type":"text"}]}]`}, - {name: "missing columns", body: `[{"name":"broken_table","columns":[]}]`}, - {name: "duplicate table name", body: `[{"name":"duplicate","columns":[{"name":"one","type":"text"}]},{"name":"duplicate","columns":[{"name":"two","type":"text"}]}]`}, - {name: "missing column name", body: `[{"name":"broken_table","columns":[{"name":"","type":"text"}]}]`}, - {name: "missing column type", body: `[{"name":"broken_table","columns":[{"name":"value","type":""}]}]`}, - {name: "duplicate column name", body: `[{"name":"broken_table","columns":[{"name":"value","type":"text"},{"name":"value","type":"integer"}]}]`}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, err := parseOsquerySchemaJSON([]byte(tt.body)) - - require.Error(t, err) - }) - } -} diff --git a/server/api_endpoints/api_endpoints.yml b/server/api_endpoints/api_endpoints.yml index fb1090caa88..5902abde1e1 100644 --- a/server/api_endpoints/api_endpoints.yml +++ b/server/api_endpoints/api_endpoints.yml @@ -373,9 +373,6 @@ - method: "POST" path: "/api/v1/fleet/automations/reset" display_name: "Reset policy automations" -- method: "GET" - path: "/api/v1/fleet/osquery/schema/search" - display_name: "Search osquery schema" - method: "GET" path: "/api/v1/fleet/reports" display_name: "List reports" diff --git a/server/service/handler.go b/server/service/handler.go index 4c0655e5d70..b9d1c0f6af7 100644 --- a/server/service/handler.go +++ b/server/service/handler.go @@ -388,9 +388,6 @@ func attachFleetAPIRoutes(r *mux.Router, svc fleet.Service, config config.FleetC ue.GET("/api/_version_/fleet/reports/{id:[0-9]+}", getQueryEndpoint, fleet.GetQueryRequest{}) ue.GET("/api/_version_/fleet/reports", listQueriesEndpoint, fleet.ListQueriesRequest{}) - // >>> OPENFRAME(osquery-schema-search): canonical schema context for AI-generated live queries — openframe/docs/api-osquery-schema-search.md - ue.GET("/api/_version_/fleet/osquery/schema/search", searchOsquerySchemaEndpoint, searchOsquerySchemaRequest{}) - // <<< OPENFRAME(osquery-schema-search) ue.GET("/api/_version_/fleet/reports/{id:[0-9]+}/report", getQueryReportEndpoint, fleet.GetQueryReportRequest{}) ue.POST("/api/_version_/fleet/reports", createQueryEndpoint, fleet.CreateQueryRequest{}) ue.PATCH("/api/_version_/fleet/reports/{id:[0-9]+}", modifyQueryEndpoint, fleet.ModifyQueryRequest{}) diff --git a/server/service/handler_test.go b/server/service/handler_test.go index c74cf065cd1..f6e1bb0748b 100644 --- a/server/service/handler_test.go +++ b/server/service/handler_test.go @@ -13,6 +13,7 @@ import ( "testing" "github.com/fleetdm/fleet/v4/pkg/fleethttp" + apiendpoints "github.com/fleetdm/fleet/v4/server/api_endpoints" "github.com/fleetdm/fleet/v4/server/config" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/mock" @@ -25,6 +26,37 @@ import ( "github.com/throttled/throttled/v2/store/memstore" ) +// >>> OPENFRAME(osquery-schema-retirement): schema search belongs to the AI service, not Fleet. +func TestOpenframeOsquerySchemaRouteIsNotRegistered(t *testing.T) { + ds := new(mock.Store) + svc, _ := newTestService(t, ds, nil, nil) + limitStore, err := memstore.New(0) + require.NoError(t, err) + router := MakeHandler(svc, config.TestConfig(), slog.New(slog.DiscardHandler), limitStore, nil, nil, nil).(*mux.Router) + + routes := make(map[string]bool) + err = router.Walk(func(route *mux.Route, _ *mux.Router, _ []*mux.Route) error { + path, pathErr := route.GetPathTemplate() + methods, methodsErr := route.GetMethods() + if pathErr == nil && methodsErr == nil { + for _, method := range methods { + routes[method+" "+path] = true + } + } + return nil + }) + require.NoError(t, err) + const prefix = "/api/{fleetversion:(?:v1|2022-04|latest)}/fleet" + require.True(t, routes["GET "+prefix+"/reports"]) + require.True(t, routes["POST "+prefix+"/reports"]) + require.True(t, routes["POST "+prefix+"/reports/run"]) + require.True(t, routes["GET "+prefix+"/policies/{policy_id}/hosts"]) + require.False(t, routes["GET "+prefix+"/osquery/schema/search"]) + require.False(t, apiendpoints.IsInCatalog(fleet.NewAPIEndpointFromTpl("GET", "/api/v1/fleet/osquery/schema/search").Fingerprint())) +} + +// <<< OPENFRAME(osquery-schema-retirement) + func TestAPIRoutesConflicts(t *testing.T) { ds := new(mock.Store) diff --git a/server/service/osquery_schema_openframe.go b/server/service/osquery_schema_openframe.go deleted file mode 100644 index 3ed0a57876e..00000000000 --- a/server/service/osquery_schema_openframe.go +++ /dev/null @@ -1,55 +0,0 @@ -// OPENFRAME(osquery-schema-search): searchable canonical osquery schema for OpenFrame's AI query generation. -package service - -import ( - "context" - "strings" - - "github.com/fleetdm/fleet/v4/schema" - "github.com/fleetdm/fleet/v4/server/fleet" -) - -type searchOsquerySchemaRequest struct { - Query string `query:"query"` - Platform string `query:"platform,optional"` - Limit *int `query:"limit,optional"` -} - -type searchOsquerySchemaResponse struct { - Query string `json:"query"` - Platform string `json:"platform"` - Count int `json:"count"` - Tables []schema.OsqueryTable `json:"tables"` - Err error `json:"error,omitempty"` -} - -func (r searchOsquerySchemaResponse) Error() error { return r.Err } - -func searchOsquerySchemaEndpoint(ctx context.Context, request interface{}, svc fleet.Service) (fleet.Errorer, error) { - if _, err := svc.AuthenticatedUser(ctx); err != nil { - return searchOsquerySchemaResponse{Err: err}, nil - } - - req := request.(*searchOsquerySchemaRequest) - platform, err := schema.NormalizeOsqueryPlatform(req.Platform) - if err != nil { - return searchOsquerySchemaResponse{Err: badRequest(err.Error())}, nil - } - - limit := schema.DefaultOsquerySearchLimit - if req.Limit != nil { - limit = *req.Limit - } - tables, err := schema.SearchOsqueryTables(req.Query, platform, limit) - if err != nil { - return searchOsquerySchemaResponse{Err: badRequest(err.Error())}, nil - } - - query := strings.TrimSpace(req.Query) - return searchOsquerySchemaResponse{ - Query: query, - Platform: platform, - Count: len(tables), - Tables: tables, - }, nil -} diff --git a/server/service/osquery_schema_openframe_test.go b/server/service/osquery_schema_openframe_test.go deleted file mode 100644 index f3b58408336..00000000000 --- a/server/service/osquery_schema_openframe_test.go +++ /dev/null @@ -1,114 +0,0 @@ -// OPENFRAME(osquery-schema-search): tests for the fork-only schema search endpoint. -package service - -import ( - "context" - "errors" - "log/slog" - "testing" - - "github.com/fleetdm/fleet/v4/schema" - apiendpoints "github.com/fleetdm/fleet/v4/server/api_endpoints" - "github.com/fleetdm/fleet/v4/server/config" - "github.com/fleetdm/fleet/v4/server/fleet" - "github.com/fleetdm/fleet/v4/server/mock" - mockservice "github.com/fleetdm/fleet/v4/server/mock/service" - "github.com/gorilla/mux" - "github.com/stretchr/testify/require" - "github.com/throttled/throttled/v2/store/memstore" -) - -func TestSearchOsquerySchemaEndpointReturnsAuthenticationError(t *testing.T) { - authenticationErr := errors.New("authenticate user") - svc := &mockservice.Service{ - AuthenticatedUserFunc: func(context.Context) (*fleet.User, error) { - return nil, authenticationErr - }, - } - - response, err := searchOsquerySchemaEndpoint(context.Background(), &searchOsquerySchemaRequest{ - Query: "windows update", - }, svc) - require.NoError(t, err) - require.ErrorIs(t, response.Error(), authenticationErr) -} - -func authenticatedOsquerySchemaService() fleet.Service { - return &mockservice.Service{ - AuthenticatedUserFunc: func(context.Context) (*fleet.User, error) { - return &fleet.User{}, nil - }, - } -} - -func TestSearchOsquerySchemaRouteIsRegistered(t *testing.T) { - ds := new(mock.Store) - svc, _ := newTestService(t, ds, nil, nil) - limitStore, err := memstore.New(0) - require.NoError(t, err) - router := MakeHandler(svc, config.TestConfig(), slog.New(slog.DiscardHandler), limitStore, nil, nil, nil).(*mux.Router) - - found := false - err = router.Walk(func(route *mux.Route, _ *mux.Router, _ []*mux.Route) error { - path, pathErr := route.GetPathTemplate() - if pathErr != nil || path != "/api/{fleetversion:(?:v1|2022-04|latest)}/fleet/osquery/schema/search" { - return nil - } - methods, methodsErr := route.GetMethods() - require.NoError(t, methodsErr) - found = len(methods) == 1 && methods[0] == "GET" - return nil - }) - require.NoError(t, err) - require.True(t, found) - require.True(t, apiendpoints.IsInCatalog(fleet.NewAPIEndpointFromTpl("GET", "/api/v1/fleet/osquery/schema/search").Fingerprint())) -} - -func TestSearchOsquerySchemaEndpointReturnsRankedSchema(t *testing.T) { - limit := 5 - response, err := searchOsquerySchemaEndpoint(context.Background(), &searchOsquerySchemaRequest{ - Query: "windows update result code", - Platform: "windows", - Limit: &limit, - }, authenticatedOsquerySchemaService()) - require.NoError(t, err) - - searchResponse := response.(searchOsquerySchemaResponse) - require.NoError(t, searchResponse.Error()) - require.Equal(t, "windows update result code", searchResponse.Query) - require.Equal(t, "windows", searchResponse.Platform) - require.Equal(t, len(searchResponse.Tables), searchResponse.Count) - require.Equal(t, "windows_update_history", searchResponse.Tables[0].Name) -} - -func TestSearchOsquerySchemaEndpointAppliesDefaults(t *testing.T) { - response, err := searchOsquerySchemaEndpoint(context.Background(), &searchOsquerySchemaRequest{ - Query: "os version", - }, authenticatedOsquerySchemaService()) - require.NoError(t, err) - - searchResponse := response.(searchOsquerySchemaResponse) - require.NoError(t, searchResponse.Error()) - require.Equal(t, "all", searchResponse.Platform) - require.LessOrEqual(t, searchResponse.Count, schema.DefaultOsquerySearchLimit) -} - -func TestSearchOsquerySchemaEndpointReturnsBadRequestForInvalidInput(t *testing.T) { - response, err := searchOsquerySchemaEndpoint(context.Background(), &searchOsquerySchemaRequest{ - Query: "updates", - Platform: "freebsd", - }, authenticatedOsquerySchemaService()) - require.NoError(t, err) - - searchResponse := response.(searchOsquerySchemaResponse) - require.ErrorContains(t, searchResponse.Error(), "unsupported platform") - - zero := 0 - response, err = searchOsquerySchemaEndpoint(context.Background(), &searchOsquerySchemaRequest{ - Query: "updates", - Limit: &zero, - }, authenticatedOsquerySchemaService()) - require.NoError(t, err) - searchResponse = response.(searchOsquerySchemaResponse) - require.ErrorContains(t, searchResponse.Error(), "limit must be between 1 and 50") -}