From bda30520689b2801503086e4f0793a8b4636591d Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Tue, 30 Jun 2026 18:00:00 +0200 Subject: [PATCH 1/3] expr: add HTTP helpers --- pkg/exprhelpers/expr_lib.go | 28 +++++++ pkg/exprhelpers/http.go | 140 +++++++++++++++++++++++++++++++++++ pkg/exprhelpers/http_test.go | 114 ++++++++++++++++++++++++++++ 3 files changed, 282 insertions(+) create mode 100644 pkg/exprhelpers/http.go create mode 100644 pkg/exprhelpers/http_test.go diff --git a/pkg/exprhelpers/expr_lib.go b/pkg/exprhelpers/expr_lib.go index 8fa8e084b26..707271a50ab 100644 --- a/pkg/exprhelpers/expr_lib.go +++ b/pkg/exprhelpers/expr_lib.go @@ -554,6 +554,34 @@ var exprFuncs = []exprCustomFunc{ new(func([]interface{}) time.Duration), }, }, + { + name: "HTTPGet", + function: HTTPGet, + signature: []any{ + new(func(string) (*HTTPResponse, error)), + }, + }, + { + name: "HTTPHead", + function: HTTPHead, + signature: []any{ + new(func(string) (*HTTPResponse, error)), + }, + }, + { + name: "HTTPPost", + function: HTTPPost, + signature: []any{ + new(func(string, string, string) (*HTTPResponse, error)), + }, + }, + { + name: "HTTPRequest", + function: HTTPRequest, + signature: []any{ + new(func(string, string, map[string]string, string) (*HTTPResponse, error)), + }, + }, } //go 1.20 "CutPrefix": strings.CutPrefix, diff --git a/pkg/exprhelpers/http.go b/pkg/exprhelpers/http.go new file mode 100644 index 00000000000..f0f50732bdc --- /dev/null +++ b/pkg/exprhelpers/http.go @@ -0,0 +1,140 @@ +package exprhelpers + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/crowdsecurity/crowdsec/pkg/apiclient/useragent" +) + +const ( + exprHTTPTimeout = 10 * time.Second + exprHTTPMaxBodySize = 10 << 20 // 10 MiB cap to avoid OOM on huge responses +) + +type HTTPResponse struct { + StatusCode int + Body string + Headers http.Header +} + +type httpHelperTransport struct { + next http.RoundTripper +} + +func (t *httpHelperTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Header.Get("User-Agent") == "" { + req.Header.Set("User-Agent", useragent.Default()) + } + + return t.next.RoundTrip(req) +} + +var exprHTTPClient = &http.Client{ + Timeout: exprHTTPTimeout, + Transport: &httpHelperTransport{next: http.DefaultTransport}, +} + +func doHTTPRequest(method, uri string, headers map[string]string, body io.Reader) (*HTTPResponse, error) { + req, err := http.NewRequestWithContext(context.Background(), method, uri, body) + if err != nil { + return nil, err + } + + for k, v := range headers { + req.Header.Set(k, v) + } + + resp, err := exprHTTPClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + b, err := io.ReadAll(io.LimitReader(resp.Body, exprHTTPMaxBodySize)) + if err != nil { + return nil, err + } + + return &HTTPResponse{ + StatusCode: resp.StatusCode, + Body: string(b), + Headers: resp.Header, + }, nil +} + +// HTTPGet(url string) (*HTTPResponse, error) +func HTTPGet(params ...any) (any, error) { + uri, ok := params[0].(string) + if !ok { + return nil, fmt.Errorf("invalid type for url: %T", params[0]) + } + + return doHTTPRequest(http.MethodGet, uri, nil, nil) +} + +// HTTPHead(url string) (*HTTPResponse, error) +func HTTPHead(params ...any) (any, error) { + uri, ok := params[0].(string) + if !ok { + return nil, fmt.Errorf("invalid type for url: %T", params[0]) + } + + return doHTTPRequest(http.MethodHead, uri, nil, nil) +} + +// HTTPPost(url string, contentType string, body string) (*HTTPResponse, error) +func HTTPPost(params ...any) (any, error) { + uri, ok := params[0].(string) + if !ok { + return nil, fmt.Errorf("invalid type for url: %T", params[0]) + } + + contentType, ok := params[1].(string) + if !ok { + return nil, fmt.Errorf("invalid type for contentType: %T", params[1]) + } + + body, ok := params[2].(string) + if !ok { + return nil, fmt.Errorf("invalid type for body: %T", params[2]) + } + + headers := map[string]string{"Content-Type": contentType} + + return doHTTPRequest(http.MethodPost, uri, headers, strings.NewReader(body)) +} + +// HTTPRequest(method string, url string, headers map[string]string, body string) (*HTTPResponse, error) +func HTTPRequest(params ...any) (any, error) { + method, ok := params[0].(string) + if !ok { + return nil, fmt.Errorf("invalid type for method: %T", params[0]) + } + + uri, ok := params[1].(string) + if !ok { + return nil, fmt.Errorf("invalid type for url: %T", params[1]) + } + + headers, ok := params[2].(map[string]string) + if !ok { + return nil, fmt.Errorf("invalid type for headers: %T", params[2]) + } + + body, ok := params[3].(string) + if !ok { + return nil, fmt.Errorf("invalid type for body: %T", params[3]) + } + + var reader io.Reader + if body != "" { + reader = strings.NewReader(body) + } + + return doHTTPRequest(method, uri, headers, reader) +} diff --git a/pkg/exprhelpers/http_test.go b/pkg/exprhelpers/http_test.go new file mode 100644 index 00000000000..49c4367a513 --- /dev/null +++ b/pkg/exprhelpers/http_test.go @@ -0,0 +1,114 @@ +package exprhelpers + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/expr-lang/expr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHTTPGet(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodGet, r.Method) + w.Header().Set("X-Test", "value") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("hello")) + })) + defer server.Close() + + out, err := HTTPGet(server.URL) + require.NoError(t, err) + + resp := out.(*HTTPResponse) + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "hello", resp.Body) + assert.Equal(t, "value", resp.Headers.Get("X-Test")) +} + +func TestHTTPHead(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodHead, r.Method) + w.Header().Set("X-Test", "value") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("should-be-ignored")) + })) + defer server.Close() + + out, err := HTTPHead(server.URL) + require.NoError(t, err) + + resp := out.(*HTTPResponse) + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Empty(t, resp.Body) + assert.Equal(t, "value", resp.Headers.Get("X-Test")) +} + +func TestHTTPPost(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodPost, r.Method) + assert.Equal(t, "application/json", r.Header.Get("Content-Type")) + + body, _ := io.ReadAll(r.Body) + assert.JSONEq(t, `{"foo":"bar"}`, string(body)) + + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte("created")) + })) + defer server.Close() + + out, err := HTTPPost(server.URL, "application/json", `{"foo":"bar"}`) + require.NoError(t, err) + + resp := out.(*HTTPResponse) + assert.Equal(t, http.StatusCreated, resp.StatusCode) + assert.Equal(t, "created", resp.Body) +} + +func TestHTTPRequest(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, http.MethodPut, r.Method) + assert.Equal(t, "secret", r.Header.Get("X-Api-Key")) + + body, _ := io.ReadAll(r.Body) + assert.Equal(t, "payload", string(body)) + + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte("ok")) + })) + defer server.Close() + + headers := map[string]string{"X-Api-Key": "secret"} + + out, err := HTTPRequest(http.MethodPut, server.URL, headers, "payload") + require.NoError(t, err) + + resp := out.(*HTTPResponse) + assert.Equal(t, http.StatusAccepted, resp.StatusCode) + assert.Equal(t, "ok", resp.Body) +} + +func TestHTTPRequestError(t *testing.T) { + _, err := HTTPGet("http://127.0.0.1:0") + require.Error(t, err) +} + +func TestHTTPExprIntegration(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("body")) + })) + defer server.Close() + + env := map[string]any{"url": server.URL} + + program, err := expr.Compile("HTTPGet(url).StatusCode == 200 && HTTPGet(url).Body == 'body'", GetExprOptions(env)...) + require.NoError(t, err) + + out, err := expr.Run(program, env) + require.NoError(t, err) + assert.Equal(t, true, out) +} From db7ce40403cddd63ef3ab7a0ae037c13e7b94a20 Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 1 Jul 2026 13:59:23 +0200 Subject: [PATCH 2/3] fix map type --- pkg/exprhelpers/expr_lib.go | 2 +- pkg/exprhelpers/http.go | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/exprhelpers/expr_lib.go b/pkg/exprhelpers/expr_lib.go index 707271a50ab..a38a7a5ba1f 100644 --- a/pkg/exprhelpers/expr_lib.go +++ b/pkg/exprhelpers/expr_lib.go @@ -579,7 +579,7 @@ var exprFuncs = []exprCustomFunc{ name: "HTTPRequest", function: HTTPRequest, signature: []any{ - new(func(string, string, map[string]string, string) (*HTTPResponse, error)), + new(func(string, string, map[string]any, string) (*HTTPResponse, error)), }, }, } diff --git a/pkg/exprhelpers/http.go b/pkg/exprhelpers/http.go index f0f50732bdc..60b34aef46e 100644 --- a/pkg/exprhelpers/http.go +++ b/pkg/exprhelpers/http.go @@ -109,7 +109,7 @@ func HTTPPost(params ...any) (any, error) { return doHTTPRequest(http.MethodPost, uri, headers, strings.NewReader(body)) } -// HTTPRequest(method string, url string, headers map[string]string, body string) (*HTTPResponse, error) +// HTTPRequest(method string, url string, headers map[string]any, body string) (*HTTPResponse, error) func HTTPRequest(params ...any) (any, error) { method, ok := params[0].(string) if !ok { @@ -121,11 +121,18 @@ func HTTPRequest(params ...any) (any, error) { return nil, fmt.Errorf("invalid type for url: %T", params[1]) } - headers, ok := params[2].(map[string]string) + // headers is map[string]any so that expr map literals (map[string]interface{}) + // are accepted; values are stringified. + rawHeaders, ok := params[2].(map[string]any) if !ok { return nil, fmt.Errorf("invalid type for headers: %T", params[2]) } + headers := make(map[string]string, len(rawHeaders)) + for k, v := range rawHeaders { + headers[k] = fmt.Sprint(v) + } + body, ok := params[3].(string) if !ok { return nil, fmt.Errorf("invalid type for body: %T", params[3]) From be1989a167aa4f9499144f71497e6a3c3573cafe Mon Sep 17 00:00:00 2001 From: Sebastien Blot Date: Wed, 1 Jul 2026 13:59:32 +0200 Subject: [PATCH 3/3] more tests --- pkg/exprhelpers/http_test.go | 54 ++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/pkg/exprhelpers/http_test.go b/pkg/exprhelpers/http_test.go index 49c4367a513..d44012e7604 100644 --- a/pkg/exprhelpers/http_test.go +++ b/pkg/exprhelpers/http_test.go @@ -81,7 +81,7 @@ func TestHTTPRequest(t *testing.T) { })) defer server.Close() - headers := map[string]string{"X-Api-Key": "secret"} + headers := map[string]any{"X-Api-Key": "secret"} out, err := HTTPRequest(http.MethodPut, server.URL, headers, "payload") require.NoError(t, err) @@ -97,18 +97,56 @@ func TestHTTPRequestError(t *testing.T) { } func TestHTTPExprIntegration(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + // echoes back method, Content-Type and body so expressions can assert on them + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + w.Header().Set("X-Method", r.Method) + w.Header().Set("X-Content-Type", r.Header.Get("Content-Type")) + w.Header().Set("X-Api-Key", r.Header.Get("X-Api-Key")) w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("body")) + _, _ = w.Write(body) })) defer server.Close() + tests := []struct { + name string + program string + }{ + { + name: "HTTPGet", + program: "HTTPGet(url).StatusCode == 200 && HTTPGet(url).Headers.Get('X-Method') == 'GET'", + }, + { + name: "HTTPHead", + program: "HTTPHead(url).StatusCode == 200 && HTTPHead(url).Headers.Get('X-Method') == 'HEAD'", + }, + { + name: "HTTPPost with content-type", + program: `HTTPPost(url, "application/json", '{"foo":"bar"}').Body == '{"foo":"bar"}' && ` + + `HTTPPost(url, "application/json", "x").Headers.Get("X-Content-Type") == "application/json"`, + }, + { + name: "HTTPRequest generic with headers map literal", + program: `HTTPRequest("PUT", url, {"X-Api-Key": "secret"}, "payload").Body == "payload" && ` + + `HTTPRequest("PUT", url, {"X-Api-Key": "secret"}, "payload").Headers.Get("X-Method") == "PUT" && ` + + `HTTPRequest("PUT", url, {"X-Api-Key": "secret"}, "payload").Headers.Get("X-Api-Key") == "secret"`, + }, + { + name: "HTTPRequest empty body", + program: `HTTPRequest("GET", url, {}, "").StatusCode == 200`, + }, + } + env := map[string]any{"url": server.URL} - program, err := expr.Compile("HTTPGet(url).StatusCode == 200 && HTTPGet(url).Body == 'body'", GetExprOptions(env)...) - require.NoError(t, err) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + program, err := expr.Compile(tc.program, GetExprOptions(env)...) + require.NoError(t, err) - out, err := expr.Run(program, env) - require.NoError(t, err) - assert.Equal(t, true, out) + out, err := expr.Run(program, env) + require.NoError(t, err) + assert.Equal(t, true, out) + }) + } }