From 2040e1dd89430e68286e6dd5b493d0a68ec766dc Mon Sep 17 00:00:00 2001
From: AltayAkkus <7849969+AltayAkkus@users.noreply.github.com>
Date: Sat, 18 Apr 2026 18:43:15 +0200
Subject: [PATCH 1/6] e2e tests: echoserver/imperative

---
 e2e/echoserver/imperative/assertions.go      |  81 ++++++++++++
 e2e/echoserver/imperative/assertions_test.go |  99 ++++++++++++++
 e2e/echoserver/imperative/compression.go     |  46 +++++++
 e2e/echoserver/imperative/generation.go      |  96 ++++++++++++++
 e2e/echoserver/imperative/generation_test.go | 104 +++++++++++++++
 e2e/echoserver/imperative/imperative.go      |  65 ++++++++++
 e2e/echoserver/imperative/utils.go           | 128 +++++++++++++++++++
 e2e/echoserver/imperative/utils_test.go      | 115 +++++++++++++++++
 8 files changed, 734 insertions(+)
 create mode 100644 e2e/echoserver/imperative/assertions.go
 create mode 100644 e2e/echoserver/imperative/assertions_test.go
 create mode 100644 e2e/echoserver/imperative/compression.go
 create mode 100644 e2e/echoserver/imperative/generation.go
 create mode 100644 e2e/echoserver/imperative/generation_test.go
 create mode 100644 e2e/echoserver/imperative/imperative.go
 create mode 100644 e2e/echoserver/imperative/utils.go
 create mode 100644 e2e/echoserver/imperative/utils_test.go

diff --git a/e2e/echoserver/imperative/assertions.go b/e2e/echoserver/imperative/assertions.go
new file mode 100644
index 0000000..3378e13
--- /dev/null
+++ b/e2e/echoserver/imperative/assertions.go
@@ -0,0 +1,81 @@
+package imperative
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+)
+
+func AssertRequest(req *http.Request, body []byte) error {
+	// parse imperative from request header
+	imp, err := ParseImperative(req.Header.Get(ImperativeHeader))
+	if err != nil {
+		return fmt.Errorf("assertRequest: parse imperative: %v", err)
+	}
+	if err := imp.Validate(); err != nil {
+		return fmt.Errorf("assertRequest: validate imperative: %v", err)
+	}
+
+	// check method
+	if req.Method != imp.Request.Method {
+		return fmt.Errorf("request method mismatch: expected %s, got %s", imp.Request.Method, req.Method)
+	}
+	// check path
+	if req.URL.Path != imp.Request.Path {
+		return fmt.Errorf("request path mismatch: expected %s, got %s", imp.Request.Path, req.URL.Path)
+	}
+	// check headers
+	for key, value := range imp.Request.Headers {
+		if req.Header.Get(key) != value {
+			return fmt.Errorf("request header mismatch: expected %s, got %s", value, req.Header.Get(key))
+		}
+	}
+	// calculate specified body
+	specifiedBody, err := imp.Request.Body.Build()
+	if err != nil {
+		return fmt.Errorf("assertRequest: build body: %v", err)
+	}
+	// compare against passed body
+	if !bytes.Equal(specifiedBody, body) {
+		return fmt.Errorf("request body mismatch: expected %d bytes, got %d bytes", len(specifiedBody), len(body))
+	}
+	// imperative ok, method ok, path ok, headers ok, body ok
+	return nil
+}
+
+func AssertResponse(resp *http.Response, body []byte) error {
+	// read imperative from response header
+	imp, err := ParseImperative(resp.Header.Get(ImperativeHeader))
+	if err != nil {
+		return fmt.Errorf("assertRequest: parse imperative: %v", err)
+	}
+	if err := imp.Validate(); err != nil {
+		return fmt.Errorf("assertResponse: validate imperative: %v", err)
+	}
+
+	// check status code
+	if resp.StatusCode != imp.Response.StatusCode {
+		return fmt.Errorf("response status code mismatch: expected %d, got %d", imp.Response.StatusCode, resp.StatusCode)
+	}
+	// check headers
+	for key, value := range imp.Response.Headers {
+		if resp.Header.Get(key) != value {
+			return fmt.Errorf("response header mismatch: expected %s, got %s", value, resp.Header.Get(key))
+		}
+	}
+	// return early if the imperative aborts the connection
+	if imp.Transport.Abort {
+		return nil
+	}
+	// calculate specified body
+	specifiedBody, err := imp.Response.Body.Build()
+	if err != nil {
+		return fmt.Errorf("assertResponse: build body: %v", err)
+	}
+	// compare against passed body
+	if !bytes.Equal(specifiedBody, body) {
+		return fmt.Errorf("response body mismatch: expected %d bytes, got %d bytes", len(specifiedBody), len(body))
+	}
+	// imperative ok, status code ok, headers ok, body ok
+	return nil
+}
diff --git a/e2e/echoserver/imperative/assertions_test.go b/e2e/echoserver/imperative/assertions_test.go
new file mode 100644
index 0000000..4bf0267
--- /dev/null
+++ b/e2e/echoserver/imperative/assertions_test.go
@@ -0,0 +1,99 @@
+package imperative
+
+import (
+	"io"
+	"testing"
+)
+
+func TestAssertRequest(t *testing.T) {
+	imp := Imperative{Response: Response{StatusCode: 200}, Request: Request{Method: "GET", Path: "/", Headers: map[string]string{"Content-Type": "application/json"}, Body: Body{Length: 1024, Encoding: EncodingUTF8, Seed: 42}}}
+	req, err := imp.BuildRequest("")
+	if err != nil {
+		t.Errorf("BuildRequest: %v", err)
+	}
+	body, err := io.ReadAll(req.Body)
+	if err != nil {
+		t.Errorf("ReadAll: %v", err)
+	}
+
+	if err := AssertRequest(req, body); err != nil {
+		t.Errorf("AssertRequest: %v", err)
+	}
+}
+
+func TestAssertRequest_InvalidBody(t *testing.T) {
+	imp := Imperative{Response: Response{StatusCode: 200}, Request: Request{Method: "GET", Path: "/", Headers: map[string]string{"Content-Type": "application/json"}, Body: Body{Length: 1024, Encoding: EncodingUTF8, Seed: 42}}}
+	req, err := imp.BuildRequest("")
+	if err != nil {
+		t.Errorf("BuildRequest: %v", err)
+	}
+	body, err := io.ReadAll(req.Body)
+	if err != nil {
+		t.Errorf("ReadAll: %v", err)
+	}
+	// flip a single bit in the body
+	body[512] ^= 1
+
+	if err := AssertRequest(req, body); err == nil {
+		t.Errorf("AssertRequest should have failed")
+	}
+}
+
+func TestAssertRequest_InvalidHeaders(t *testing.T) {
+	imp := Imperative{Response: Response{StatusCode: 200}, Request: Request{Method: "GET", Path: "/", Headers: map[string]string{"Content-Type": "application/json"}, Body: Body{Length: 1024, Encoding: EncodingUTF8, Seed: 42}}}
+	req, err := imp.BuildRequest("")
+	if err != nil {
+		t.Errorf("BuildRequest: %v", err)
+	}
+	body, err := io.ReadAll(req.Body)
+	if err != nil {
+		t.Errorf("ReadAll: %v", err)
+	}
+	// overwrite the content type
+	req.Header.Set("Content-Type", "application/octet-stream")
+
+	if err := AssertRequest(req, body); err == nil {
+		t.Errorf("AssertRequest should have failed")
+	}
+}
+
+func TestAssertRequest_InvalidPath(t *testing.T) {
+	imp := Imperative{Response: Response{StatusCode: 200}, Request: Request{Method: "GET", Path: "/", Headers: map[string]string{"Content-Type": "application/json"}, Body: Body{Length: 1024, Encoding: EncodingUTF8, Seed: 42}}}
+	req, err := imp.BuildRequest("")
+	if err != nil {
+		t.Errorf("BuildRequest: %v", err)
+	}
+	body, err := io.ReadAll(req.Body)
+	if err != nil {
+		t.Errorf("ReadAll: %v", err)
+	}
+	// overwrite the path
+	req.URL.Path = "/invalid"
+
+	if err := AssertRequest(req, body); err == nil {
+		t.Errorf("AssertRequest should have failed")
+	}
+}
+
+func TestAssertRequest_InvalidMethod(t *testing.T) {
+	imp := Imperative{Response: Response{StatusCode: 200}, Request: Request{Method: "POST", Path: "/", Headers: map[string]string{"Content-Type": "application/json"}, Body: Body{Length: 1024, Encoding: EncodingUTF8, Seed: 42}}}
+	req, err := imp.BuildRequest("")
+	if err != nil {
+		t.Errorf("BuildRequest: %v", err)
+	}
+	body, err := io.ReadAll(req.Body)
+	if err != nil {
+		t.Errorf("ReadAll: %v", err)
+	}
+	// overwrite the method
+	req.Method = "GET"
+
+	if err := AssertRequest(req, body); err == nil {
+		t.Errorf("AssertRequest should have failed")
+	}
+}
+
+// where are the tests for the response assertions?
+// since it doesn't make sense to generate a http.Response from an Imperative
+// because the http.Handler uses the http.ResponseWriter to write the response
+// we test the response building logic in the ../server_test.go file :)
diff --git a/e2e/echoserver/imperative/compression.go b/e2e/echoserver/imperative/compression.go
new file mode 100644
index 0000000..22c7009
--- /dev/null
+++ b/e2e/echoserver/imperative/compression.go
@@ -0,0 +1,46 @@
+package imperative
+
+import (
+	"bytes"
+	"compress/gzip"
+	"compress/zlib"
+
+	"github.com/klauspost/compress/zstd"
+)
+
+// Compress compresses data using the named encoding.
+func Compress(data []byte, encoding string) ([]byte, error) {
+	var buf bytes.Buffer
+	switch encoding {
+	case "gzip":
+		w := gzip.NewWriter(&buf)
+		if _, err := w.Write(data); err != nil {
+			return nil, err
+		}
+		if err := w.Close(); err != nil {
+			return nil, err
+		}
+	case "deflate":
+		w := zlib.NewWriter(&buf)
+		if _, err := w.Write(data); err != nil {
+			return nil, err
+		}
+		if err := w.Close(); err != nil {
+			return nil, err
+		}
+	case "zstd":
+		w, err := zstd.NewWriter(&buf)
+		if err != nil {
+			return nil, err
+		}
+		if _, err := w.Write(data); err != nil {
+			return nil, err
+		}
+		if err := w.Close(); err != nil {
+			return nil, err
+		}
+	default:
+		return data, nil
+	}
+	return buf.Bytes(), nil
+}
diff --git a/e2e/echoserver/imperative/generation.go b/e2e/echoserver/imperative/generation.go
new file mode 100644
index 0000000..3a636a7
--- /dev/null
+++ b/e2e/echoserver/imperative/generation.go
@@ -0,0 +1,96 @@
+package imperative
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+
+	"bytes"
+)
+
+func (bdy Body) Validate() error {
+	if bdy.Length != 0 && bdy.Filepath != "" {
+		return fmt.Errorf("imperative: body length and filepath cannot be set at the same time. Pick one or the other.")
+	}
+	if bdy.Encoding != "" && bdy.Encoding != EncodingUTF8 && bdy.Encoding != EncodingBinary {
+		return fmt.Errorf("imperative: invalid value for encoding, must be either empty, %s or %s", EncodingUTF8, EncodingBinary)
+	}
+	if bdy.Length > 0 && bdy.Seed == 0 {
+		return fmt.Errorf("imperative: set the seed explicitly to prevent duplicate payload digests")
+	}
+	return nil
+}
+
+// Builds a body from specification
+// If neither Filepath nor Length is set, returns empty []byte, nil
+func (bdy Body) Build() ([]byte, error) {
+	var body []byte
+	var err error
+
+	if bdy.Filepath != "" {
+		// should be page cached by kernel
+		body, err = os.ReadFile(bdy.Filepath)
+		if err != nil {
+			return nil, err
+		}
+	} else if bdy.Length > 0 {
+		if bdy.Encoding == EncodingBinary {
+			body = binaryBody(bdy.Length, bdy.Seed)
+		} else {
+			body = stringBody(bdy.Length, bdy.Seed)
+		}
+	}
+
+	if bdy.Compress != "" && len(body) > 0 {
+		compressed, err := Compress(body, bdy.Compress)
+		if err != nil {
+			return nil, fmt.Errorf("compress: %w", err)
+		}
+		body = compressed
+	}
+
+	return body, nil
+}
+
+// Returns deterministic binary slice
+func binaryBody(n int, seed int) []byte {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = byte((i + seed) % 256)
+	}
+	return b
+}
+
+// Returns deterministic string as []byte
+func stringBody(n int, seed int) []byte {
+	const alphabet = "abcdefghijklmnopqrstuvwxyz0123456789"
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = alphabet[(i+seed)%len(alphabet)]
+	}
+	return b
+}
+
+// Returns a http.Request from an Imperative
+func (imp Imperative) BuildRequest(baseURL string) (*http.Request, error) {
+	if err := imp.Validate(); err != nil {
+		return nil, err
+	}
+	body, err := imp.Request.Body.Build()
+	if err != nil {
+		return nil, err
+	}
+	req, err := http.NewRequest(imp.Request.Method, fmt.Sprintf("%s%s", baseURL, imp.Request.Path), bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	// set headers
+	for key, value := range imp.Request.Headers {
+		req.Header.Set(key, value)
+	}
+	// set imperative header
+	req.Header.Set(ImperativeHeader, imp.Encode())
+
+	// imp validated, body built, headers set
+	return req, nil
+}
diff --git a/e2e/echoserver/imperative/generation_test.go b/e2e/echoserver/imperative/generation_test.go
new file mode 100644
index 0000000..8eb0fea
--- /dev/null
+++ b/e2e/echoserver/imperative/generation_test.go
@@ -0,0 +1,104 @@
+package imperative
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestBody_Validate(t *testing.T) {
+	imp := Imperative{Response: Response{StatusCode: 200, Body: Body{Length: 1024, Filepath: "test.txt", Seed: 42}}}
+	if err := imp.Response.Body.Validate(); err == nil {
+		t.Errorf("filepath and length set at the same time should error")
+	}
+	imp = Imperative{Response: Response{StatusCode: 200, Body: Body{Length: 1024, Encoding: "utf-8", Seed: 42}}}
+	if err := imp.Response.Body.Validate(); err == nil {
+		t.Errorf("encoding set to unknown value should error")
+	}
+	imp = Imperative{Response: Response{StatusCode: 200, Body: Body{Length: 1024, Encoding: EncodingUTF8}}}
+	if err := imp.Response.Body.Validate(); err == nil {
+		t.Errorf("unset seed should error")
+	}
+}
+func TestBody_Build(t *testing.T) {
+	// utf-8 body
+	imp1 := Imperative{Response: Response{StatusCode: 200, Body: Body{Length: 1024, Encoding: EncodingUTF8, Seed: 42}}}
+	imp2 := Imperative{Response: Response{StatusCode: 404, Body: Body{Length: 1024, Encoding: EncodingUTF8, Seed: 42}}}
+	body1, err1 := imp1.Response.Body.Build()
+	if err1 != nil {
+		t.Errorf("Build() utf-8 = %v, want nil", err1)
+	}
+	body2, err2 := imp2.Response.Body.Build()
+	if err2 != nil {
+		t.Errorf("Build() utf-8 = %v, want nil", err2)
+	}
+	if !bytes.Equal(body1, body2) {
+		t.Errorf("Build() utf-8 = %v, want %v", string(body1), string(body2))
+	}
+
+	// binary body
+	imp3 := Imperative{Response: Response{StatusCode: 200, Body: Body{Length: 1024, Encoding: EncodingBinary, Seed: 42}}}
+	imp4 := Imperative{Response: Response{StatusCode: 404, Body: Body{Length: 1024, Encoding: EncodingBinary, Seed: 42}}}
+	body3, err3 := imp3.Response.Body.Build()
+	if err3 != nil {
+		t.Errorf("Build() binary = %v, want nil", err3)
+	}
+	body4, err4 := imp4.Response.Body.Build()
+	if err4 != nil {
+		t.Errorf("Build() binary = %v, want nil", err4)
+	}
+	if !bytes.Equal(body3, body4) {
+		t.Errorf("Build() binary = %v, want %v", string(body3), string(body4))
+	}
+
+	// empty body should return empty []byte
+	imp5 := Imperative{Response: Response{StatusCode: 200, Body: Body{}}}
+	body5, err5 := imp5.Response.Body.Build()
+	if err5 != nil {
+		t.Errorf("Build() empty = %v, want nil", err5)
+	}
+	if len(body5) != 0 {
+		t.Errorf("Build() empty = %v, want empty", string(body5))
+	}
+}
+
+func TestBody_Build_Compress(t *testing.T) {
+	// utf-8 body
+	imp1 := Imperative{Response: Response{StatusCode: 200, Body: Body{Length: 1024, Encoding: EncodingUTF8, Seed: 42, Compress: "gzip"}}}
+	imp2 := Imperative{Response: Response{StatusCode: 404, Body: Body{Length: 1024, Encoding: EncodingUTF8, Seed: 42, Compress: "gzip"}}}
+	body1, err1 := imp1.Response.Body.Build()
+	if err1 != nil {
+		t.Errorf("Build() utf-8 = %v, want nil", err1)
+	}
+	body2, err2 := imp2.Response.Body.Build()
+	if err2 != nil {
+		t.Errorf("Build() utf-8 = %v, want nil", err2)
+	}
+	if !bytes.Equal(body1, body2) {
+		t.Errorf("Build() utf-8 = %v, want %v", string(body1), string(body2))
+	}
+
+	// binary body
+	imp3 := Imperative{Response: Response{StatusCode: 200, Body: Body{Length: 1024, Encoding: EncodingBinary, Seed: 42, Compress: "zstd"}}}
+	imp4 := Imperative{Response: Response{StatusCode: 404, Body: Body{Length: 1024, Encoding: EncodingBinary, Seed: 42, Compress: "zstd"}}}
+	body3, err3 := imp3.Response.Body.Build()
+	if err3 != nil {
+		t.Errorf("Build() binary = %v, want nil", err3)
+	}
+	body4, err4 := imp4.Response.Body.Build()
+	if err4 != nil {
+		t.Errorf("Build() binary = %v, want nil", err4)
+	}
+	if !bytes.Equal(body3, body4) {
+		t.Errorf("Build() binary = %v, want %v", string(body3), string(body4))
+	}
+
+	// empty body should return empty []byte
+	imp5 := Imperative{Response: Response{StatusCode: 200, Body: Body{Compress: "gzip"}}}
+	body5, err5 := imp5.Response.Body.Build()
+	if err5 != nil {
+		t.Errorf("Build() empty = %v, want nil", err5)
+	}
+	if len(body5) != 0 {
+		t.Errorf("Build() empty = %v, want empty", string(body5))
+	}
+}
diff --git a/e2e/echoserver/imperative/imperative.go b/e2e/echoserver/imperative/imperative.go
new file mode 100644
index 0000000..4ef5d8a
--- /dev/null
+++ b/e2e/echoserver/imperative/imperative.go
@@ -0,0 +1,65 @@
+package imperative
+
+type Imperative struct {
+	// Describes the HTTP response the echo server shall produce
+	Response Response `json:"response,omitempty"`
+
+	// Opt. describes the request the echo server shall receive and assert
+	Request Request `json:"request,omitempty"`
+
+	// Opt. describes transport channel behaviour the echo server shall produce (e.g. close after bytes)
+	Transport Transport `json:"transport,omitempty"`
+}
+
+type Response struct {
+	// StatusCode is the HTTP status code.
+	StatusCode int `json:"statusCode,omitempty"`
+
+	// Headers are additional response headers set verbatim, allowing tests
+	// to verify that gowarc preserves arbitrary response headers faithfully.
+	Headers map[string]string `json:"headers,omitempty"`
+
+	// Body configures the response body content and encoding.
+	Body Body `json:"body,omitempty"`
+}
+
+type Request struct {
+	Path    string            `json:"path,omitempty"`
+	Headers map[string]string `json:"headers,omitempty"` // a list of expected headers
+	Method  string            `json:"method,omitempty"`
+	// Body configures the HTTP request entity body (see [Imperative.RequestBody]).
+	Body Body `json:"body,omitempty"`
+}
+
+type Transport struct {
+	// Abort instructs the HTTP server to forcibly close the connection
+	Abort bool `json:"abort,omitempty"`
+	// Delay instructs the HTTP server to delay the response by n milliseconds
+	Delay int `json:"delay,omitempty"`
+}
+
+const (
+	EncodingUTF8   = ""       // printable characters (a–z, 0–9 cycling) — default
+	EncodingBinary = "binary" // raw bytes (0x00–0xFF cycling)
+)
+
+type Body struct {
+	// Length generates a deterministic body of exactly N bytes.
+	Length int `json:"length,omitempty"`
+
+	// Encoding controls the byte pattern when Length > 0:
+	//   "" or "utf-8": printable characters (a–z, 0–9 cycling)
+	//   "binary":      raw byte values (0x00–0xFF cycling)
+	Encoding string `json:"encoding,omitempty"`
+
+	// Compress applies HTTP content-encoding before sending the body.
+	// Supported values: "gzip", "deflate", "zstd". Empty means no compression.
+	Compress string `json:"compress,omitempty"`
+
+	// Filepath serves a static file as the response body.
+	// Path is relative to the echo server's configured test data directory.
+	Filepath string `json:"filepath,omitempty"`
+
+	// A random seed to generate a deterministic body
+	Seed int `json:"seed,omitempty"`
+}
diff --git a/e2e/echoserver/imperative/utils.go b/e2e/echoserver/imperative/utils.go
new file mode 100644
index 0000000..e09f57e
--- /dev/null
+++ b/e2e/echoserver/imperative/utils.go
@@ -0,0 +1,128 @@
+package imperative
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"math/rand/v2"
+)
+
+// ImperativeHeader is the request header that carries the JSON-encoded [Imperative].
+const ImperativeHeader = "X-Echo-Imperative"
+
+// Parses a Imperative from a JSON string
+func ParseImperative(s string) (Imperative, error) {
+	var d Imperative
+	if err := json.Unmarshal([]byte(s), &d); err != nil {
+		return d, err
+	}
+	return d, nil
+}
+
+// Encodes a Imperative to JSON
+func (imp Imperative) Encode() string {
+	b, _ := json.Marshal(imp)
+	return string(b)
+}
+
+// Hashes the Imperative
+func (imp Imperative) Hash() string {
+	hash := sha256.New()
+	hash.Write([]byte(imp.Encode()))
+	return hex.EncodeToString(hash.Sum(nil))
+}
+
+// Multiply the Imperative by n times
+func (imp Imperative) Multiply(n int) Imperatives {
+	out := make(Imperatives, n)
+	for i := range out {
+		out[i] = imp
+	}
+	return out
+}
+
+// Aborts returns true if the Imperative is aborted
+func (imp Imperative) Aborts() bool {
+	return imp.Transport.Abort
+}
+
+// Validate a single Imperative
+func (imp Imperative) Validate() error {
+	if imp.Response.StatusCode == 0 {
+		return fmt.Errorf("imperative: response status code is required")
+	}
+	if err := imp.Response.Body.Validate(); err != nil {
+		return err
+	}
+	if err := imp.Request.Body.Validate(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Imperatives is a slice of [Imperative] values
+type Imperatives []Imperative
+
+// Fills the Seed field with random positive numbers
+func (imps Imperatives) Seed(baseSeed uint64) Imperatives {
+	out := make(Imperatives, len(imps))
+	copy(out, imps)
+	for i := range out {
+		r := rand.New(rand.NewPCG(baseSeed, uint64(i)))
+		if out[i].Response.Body.Length > 0 {
+			out[i].Response.Body.Seed = r.IntN(1_000_000_000)
+		}
+		if out[i].Request.Body.Length > 0 {
+			out[i].Request.Body.Seed = r.IntN(1_000_000_000)
+		}
+	}
+	return out
+}
+
+// Randomly shuffles the imperatives in place
+func (imps Imperatives) Shuffle(seed uint64) Imperatives {
+	out := make(Imperatives, len(imps))
+	copy(out, imps)
+
+	// PCG requires two seeds, but we just set the stream selector to 0
+	src := rand.NewPCG(seed, 0)
+	r := rand.New(src)
+
+	r.Shuffle(len(out), func(i, j int) {
+		out[i], out[j] = out[j], out[i]
+	})
+
+	return out
+}
+
+// Hash all imperatives and return a map of hash -> []Imperative
+func (imps Imperatives) Hash() map[string][]Imperative {
+	out := make(map[string][]Imperative)
+	for _, imp := range imps {
+		h := imp.Hash()
+		out[h] = append(out[h], imp)
+	}
+	return out
+}
+
+// Multiply all imperatives by n times
+func (imps Imperatives) Multiply(n int) Imperatives {
+	out := make(Imperatives, 0, len(imps)*n)
+	for _, imp := range imps {
+		for i := 0; i < n; i++ {
+			out = append(out, imp)
+		}
+	}
+	return out
+}
+
+// Validate all imperatives
+func (imps Imperatives) Validate() error {
+	for _, imp := range imps {
+		if err := imp.Validate(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
diff --git a/e2e/echoserver/imperative/utils_test.go b/e2e/echoserver/imperative/utils_test.go
new file mode 100644
index 0000000..0ffcde8
--- /dev/null
+++ b/e2e/echoserver/imperative/utils_test.go
@@ -0,0 +1,115 @@
+package imperative
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestImperative_Multiply(t *testing.T) {
+	imp := Imperative{Response: Response{StatusCode: 200}}
+	got := imp.Multiply(2)
+	if len(got) != 2 {
+		t.Errorf("Multiply() = %d, want 2", len(got))
+	}
+	for _, imp := range got {
+		if imp.Response.StatusCode != 200 {
+			t.Errorf("Multiply() = %d, want 200", imp.Response.StatusCode)
+		}
+	}
+}
+
+func TestImperative_Seed(t *testing.T) {
+	imp := Imperative{
+		Request:  Request{Path: "/test", Method: "GET", Body: Body{Length: 1024, Encoding: EncodingUTF8, Seed: 1_000_000_000 + 42}},
+		Response: Response{StatusCode: 200},
+	}
+	imps := imp.Multiply(42)
+	got := imps.Seed(1337)
+	for _, imp := range got {
+		if imp.Request.Body.Seed <= 0 {
+			t.Errorf("Seed() = %d, want non-zero", imp.Response.Body.Seed)
+		}
+		if imp.Request.Body.Seed >= 1_000_000_000 {
+			t.Errorf("Seed() = %d, generated seed should be less or equal to 1_000_000_000", imp.Response.Body.Seed)
+		}
+		// response body has no length, so no seed should be set
+		if imp.Response.Body.Seed != 0 {
+			t.Errorf("Seed() = %d, want 0", imp.Response.Body.Seed)
+		}
+	}
+}
+
+func TestImperative_Hash(t *testing.T) {
+	imp1 := Imperative{Response: Response{StatusCode: 200}, Request: Request{Path: "/test", Method: "GET", Body: Body{Length: 1024, Encoding: EncodingUTF8, Seed: 42}}}
+	hash1 := imp1.Hash()
+	if hash1 == "" {
+		t.Errorf("Hash() = %s, want non-empty string", hash1)
+	}
+	// re-encode and hash again
+	imp2, err := ParseImperative(imp1.Encode())
+	if err != nil {
+		t.Errorf("ParseImperative() = %v, want nil", err)
+	}
+	hash2 := imp2.Hash()
+	if hash1 != hash2 {
+		t.Errorf("Hash() = %s, want %s", hash2, hash1)
+	}
+}
+
+func TestImperatives_Hash(t *testing.T) {
+	imp := Imperative{Response: Response{StatusCode: 200}, Request: Request{Path: "/test", Method: "GET", Body: Body{Length: 1024, Encoding: EncodingUTF8, Seed: 42}}}
+	imps := imp.Multiply(42).Seed(1337)
+	hashMap := imps.Hash()
+	if len(hashMap) != 42 {
+		t.Errorf("Hash() = %d, want 42", len(hashMap))
+	}
+	for _, imp := range imps {
+		got := hashMap[imp.Hash()]
+		for _, gotImp := range got {
+			if !reflect.DeepEqual(imp, gotImp) {
+				t.Errorf("Hash() = %s, want %s", imp.Encode(), gotImp.Encode())
+			}
+		}
+	}
+
+}
+
+func TestImperatives_Multiply(t *testing.T) {
+	imps := Imperatives{
+		{Response: Response{StatusCode: 200}},
+		{Response: Response{StatusCode: 404}},
+		{Response: Response{StatusCode: 500}},
+	}
+	got := imps.Multiply(2)
+	if len(got) != 6 {
+		t.Errorf("Multiply() = %d, want 4", len(got))
+	}
+	counts := make(map[int]int)
+	for _, imp := range got {
+		counts[imp.Response.StatusCode]++
+	}
+
+	if counts[200] != 2 || counts[404] != 2 || counts[500] != 2 {
+		t.Errorf("Multiply() = %v, want 2 for each status code", counts)
+	}
+}
+
+func TestImperative_Validate(t *testing.T) {
+	imp := Imperative{Response: Response{StatusCode: 200, Body: Body{Length: 1024, Filepath: "test.txt", Seed: 42}}}
+
+	if err := imp.Validate(); err == nil {
+		t.Errorf("filepath and length set at the same time should error")
+	}
+	imp = Imperative{Response: Response{StatusCode: 200, Body: Body{Length: 1024, Encoding: "utf-8", Seed: 42}}}
+	if err := imp.Validate(); err == nil {
+		t.Errorf("encoding set to unknown value should error")
+	}
+	imp = Imperative{Request: Request{Body: Body{Length: 1024, Encoding: EncodingUTF8, Seed: 42}}, Response: Response{StatusCode: 200}}
+	if err := imp.Validate(); err != nil {
+		t.Errorf("request encoding set to valid value should not error")
+	}
+	imp = Imperative{Request: Request{Body: Body{Length: 1024, Encoding: EncodingBinary}}, Response: Response{StatusCode: 200}}
+	if err := imp.Validate(); err == nil {
+		t.Errorf("no seed set should error")
+	}
+}

From 0dc86ec8144dc51e88ba134826a638ce58aec500 Mon Sep 17 00:00:00 2001
From: AltayAkkus <7849969+AltayAkkus@users.noreply.github.com>
Date: Sat, 18 Apr 2026 18:43:35 +0200
Subject: [PATCH 2/6] e2e tests: echoserver

---
 e2e/echoserver/server.go      | 104 +++++++++++++++++
 e2e/echoserver/server_test.go | 202 ++++++++++++++++++++++++++++++++++
 2 files changed, 306 insertions(+)
 create mode 100644 e2e/echoserver/server.go
 create mode 100644 e2e/echoserver/server_test.go

diff --git a/e2e/echoserver/server.go b/e2e/echoserver/server.go
new file mode 100644
index 0000000..ac5a8c6
--- /dev/null
+++ b/e2e/echoserver/server.go
@@ -0,0 +1,104 @@
+package echoserver
+
+import (
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/internetarchive/gowarc/e2e/echoserver/imperative"
+)
+
+type EchoServer struct {
+	*httptest.Server
+}
+
+// shorthand to create a IPv4 local server
+func New(t *testing.T, tls, http2 bool) *EchoServer { return newOn(t, "127.0.0.1", tls, http2) }
+
+// shorthand to create a IPv6 local server
+func NewIPv6(t *testing.T, tls, http2 bool) *EchoServer { return newOn(t, "::1", tls, http2) }
+
+func newOn(t *testing.T, host string, tls, http2 bool) *EchoServer {
+	t.Helper()
+	// port is automatically assigned by the OS
+	ln, err := net.Listen("tcp", net.JoinHostPort(host, "0"))
+	if err != nil {
+		t.Fatalf("echoserver: listen %s: %v", host, err)
+	}
+	ts := httptest.NewUnstartedServer(newHandler(t))
+	// close the listener to avoid port conflicts
+	_ = ts.Listener.Close()
+	// set the listener to the new one
+	ts.Listener = ln
+	ts.EnableHTTP2 = http2
+	if tls {
+		ts.StartTLS()
+	} else {
+		ts.Start()
+	}
+	t.Cleanup(ts.Close)
+	return &EchoServer{ts}
+}
+
+func newHandler(t *testing.T) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		if err != nil {
+			t.Errorf("echoserver: read body: %v", err)
+			return
+		}
+		imp, err := imperative.ParseImperative(r.Header.Get(imperative.ImperativeHeader))
+		if err != nil {
+			t.Errorf("echoserver: parse imperative: %v", err)
+			return
+		}
+		// validate imperative
+		if err := imp.Validate(); err != nil {
+			t.Errorf("echoserver: validate imperative: %v", err)
+			return
+		}
+		// assert request
+		if err := imperative.AssertRequest(r, body); err != nil {
+			t.Errorf("echoserver: assert request: %v", err)
+			return
+		}
+		// set headers (overwrite)
+		for k, v := range imp.Response.Headers {
+			w.Header().Set(k, v)
+		}
+		// always echo back the imperative in response headers
+		w.Header().Set(imperative.ImperativeHeader, imp.Encode())
+
+		// opt. sleep before sending headers
+		// go's scheduler parks handler goroutine automatically with time.Sleep
+		if imp.Transport.Delay > 0 {
+			time.Sleep(time.Duration(imp.Transport.Delay) * time.Millisecond)
+		}
+
+		// send headers with status code
+		if imp.Response.StatusCode != 0 {
+			w.WriteHeader(imp.Response.StatusCode)
+		}
+
+		// opt. yank the connection after sending the headers
+		if imp.Transport.Abort {
+			panic(http.ErrAbortHandler)
+		}
+
+		// build response body
+		specifiedBody, err := imp.Response.Body.Build()
+		if err != nil {
+			t.Errorf("echoserver: build response body: %v", err)
+			return
+		}
+		// write body
+		if len(specifiedBody) > 0 {
+			if _, err := w.Write(specifiedBody); err != nil {
+				t.Errorf("echoserver: write body: %v", err)
+			}
+		}
+	}
+}
diff --git a/e2e/echoserver/server_test.go b/e2e/echoserver/server_test.go
new file mode 100644
index 0000000..d526bb1
--- /dev/null
+++ b/e2e/echoserver/server_test.go
@@ -0,0 +1,202 @@
+package echoserver
+
+import (
+	"crypto/tls"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/internetarchive/gowarc/e2e/echoserver/imperative"
+)
+
+type serverStack struct {
+	name string
+	tls  bool
+	h2   bool
+	ipv6 bool // false → 127.0.0.1, true → ::1
+}
+
+// All combinations of TLS, HTTP/2, IPv4 and IPv6
+var serverStacks = []serverStack{
+	{name: "plain_H1_IPv4", tls: false, h2: false, ipv6: false},
+	{name: "plain_H1_IPv6", tls: false, h2: false, ipv6: true},
+	{name: "plain_H2_IPv4", tls: false, h2: true, ipv6: false},
+	{name: "plain_H2_IPv6", tls: false, h2: true, ipv6: true},
+	{name: "TLS_H1_IPv4", tls: true, h2: false, ipv6: false},
+	{name: "TLS_H1_IPv6", tls: true, h2: false, ipv6: true},
+	{name: "TLS_H2_IPv4", tls: true, h2: true, ipv6: false},
+	{name: "TLS_H2_IPv6", tls: true, h2: true, ipv6: true},
+}
+
+// TestServer runs the same imperative scenarios on every server stack (TLS × HTTP/2 flag × IPv4/IPv6).
+// This should prove that the echoserver follows the imperative specification correctly.
+func TestServer(t *testing.T) {
+	for _, st := range serverStacks {
+		st := st
+		t.Run(st.name, func(t *testing.T) {
+			t.Parallel()
+			srv := newServerForStack(t, st)
+			baseURL := srv.URL
+			client := httpClient()
+
+			t.Run("GET /huge.pdf (10MB)", func(t *testing.T) {
+				imp := imperative.Imperative{
+					Request:  imperative.Request{Method: "GET", Path: "/huge.pdf"},
+					Response: imperative.Response{StatusCode: 200, Body: imperative.Body{Length: 1024 * 1024 * 10, Encoding: imperative.EncodingBinary, Seed: 42}, Headers: map[string]string{"Content-Type": "application/pdf"}},
+				}
+				req, err := imp.BuildRequest(baseURL)
+				if err != nil {
+					t.Fatalf("BuildRequest: %v", err)
+				}
+				resp, body, err := doAndReadBody(client, req)
+				if err != nil {
+					t.Fatalf("doAndReadBody: %v", err)
+				}
+				if err := imperative.AssertResponse(resp, body); err != nil {
+					t.Errorf("AssertResponse: %v", err)
+				}
+				// flip a single bit in the body
+				body[512] ^= 1
+				if err := imperative.AssertResponse(resp, body); err == nil {
+					t.Errorf("AssertResponse should have failed with body mismatch")
+				}
+			})
+
+			t.Run("GET /foo DELAY", func(t *testing.T) {
+				imp := imperative.Imperative{
+					Request:   imperative.Request{Method: "GET", Path: "/foo"},
+					Response:  imperative.Response{StatusCode: 404, Body: imperative.Body{Length: 1024 * 1024, Encoding: imperative.EncodingUTF8, Seed: 42}, Headers: map[string]string{"Content-Type": "text/plain"}},
+					Transport: imperative.Transport{Delay: 1000},
+				}
+				req, err := imp.BuildRequest(baseURL)
+				if err != nil {
+					t.Fatalf("BuildRequest: %v", err)
+				}
+				start := time.Now()
+				resp, body, err := doAndReadBody(client, req)
+				elapsed := time.Since(start)
+				if elapsed < 1000*time.Millisecond {
+					t.Errorf("expected delay ≥ 1000ms, got %v", elapsed)
+				}
+				if err != nil {
+					t.Fatalf("doAndReadBody: %v", err)
+				}
+				if err := imperative.AssertResponse(resp, body); err != nil {
+					t.Errorf("AssertResponse: %v", err)
+				}
+				// mutate a header value, assert that it fails
+				resp.Header.Set("Content-Type", "application/octet-stream")
+				if err := imperative.AssertResponse(resp, body); err == nil {
+					t.Errorf("AssertResponse should have failed with header mismatch")
+				}
+			})
+
+			t.Run("POST /users", func(t *testing.T) {
+				imp := imperative.Imperative{
+					Request:  imperative.Request{Method: "POST", Path: "/users", Headers: map[string]string{"Content-Type": "application/json"}, Body: imperative.Body{Length: 1024 * 512, Encoding: imperative.EncodingUTF8, Seed: 42}},
+					Response: imperative.Response{StatusCode: 200, Body: imperative.Body{Length: 1024 * 1024, Encoding: imperative.EncodingUTF8, Seed: 42}, Headers: map[string]string{"Content-Type": "application/json"}},
+				}
+				req, err := imp.BuildRequest(baseURL)
+				if err != nil {
+					t.Fatalf("BuildRequest: %v", err)
+				}
+				resp, body, err := doAndReadBody(client, req)
+				if err != nil {
+					t.Fatalf("doAndReadBody: %v", err)
+				}
+				if err := imperative.AssertResponse(resp, body); err != nil {
+					t.Errorf("AssertResponse: %v", err)
+				}
+			})
+
+			t.Run("POST /users ABORT", func(t *testing.T) {
+				imp := imperative.Imperative{
+					Request:   imperative.Request{Method: "POST", Path: "/users", Headers: map[string]string{"Content-Type": "application/json"}, Body: imperative.Body{Length: 1024 * 512, Encoding: imperative.EncodingUTF8, Seed: 42}},
+					Response:  imperative.Response{StatusCode: 200, Body: imperative.Body{Length: 1024, Encoding: imperative.EncodingUTF8, Seed: 42}, Headers: map[string]string{"Content-Type": "application/json"}},
+					Transport: imperative.Transport{Abort: true},
+				}
+				req, err := imp.BuildRequest(baseURL)
+				if err != nil {
+					t.Fatalf("BuildRequest: %v", err)
+				}
+				resp, body, err := doAndReadBody(client, req)
+				if err == nil {
+					t.Fatal("doAndReadBody: expected EOF-style error, got nil")
+				}
+				// HTTP/1.1 raises EOF, HTTP/2 raises INTERNAL_ERROR stream error
+				if !strings.Contains(err.Error(), "EOF") && !strings.Contains(err.Error(), "INTERNAL_ERROR") {
+					t.Errorf("doAndReadBody: expected abort/EOF-style error, got %v", err)
+				}
+				if resp != nil && body != nil {
+					t.Errorf("doAndReadBody: on abort expected no usable resp/body, got resp=%v len(body)=%d", resp != nil, len(body))
+				}
+			})
+
+			t.Run("GET local file", func(t *testing.T) {
+				// create local file in temp dir
+				tempDir := t.TempDir()
+				tempFile := filepath.Join(tempDir, "2MB.jpg")
+				// unique size because this what we will compare against
+				if err := os.WriteFile(tempFile, make([]byte, 2*1024*1024+1337), 0644); err != nil {
+					t.Fatalf("os.WriteFile: %v", err)
+				}
+
+				imp := imperative.Imperative{
+					Request:  imperative.Request{Method: "GET", Path: "/2MB.jpg"},
+					Response: imperative.Response{StatusCode: 200, Body: imperative.Body{Filepath: tempFile}, Headers: map[string]string{"Content-Type": "image/jpeg"}},
+				}
+				req, err := imp.BuildRequest(baseURL)
+				if err != nil {
+					t.Fatalf("BuildRequest: %v", err)
+				}
+				resp, body, err := doAndReadBody(client, req)
+				if err != nil {
+					t.Fatalf("doAndReadBody: %v", err)
+				}
+				if err := imperative.AssertResponse(resp, body); err != nil {
+					t.Errorf("AssertResponse: %v", err)
+				}
+				// sanity check if the file is correctly read
+				if len(body) != 2*1024*1024+1337 {
+					t.Errorf("body should be 2MB + 1337 bytes, got %d bytes", len(body))
+				}
+
+			})
+		})
+	}
+}
+
+// doAndReadBody sends the request and returns the raw response body bytes.
+func doAndReadBody(client *http.Client, req *http.Request) (*http.Response, []byte, error) {
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, nil, err
+	}
+	body, err := io.ReadAll(resp.Body)
+	resp.Body.Close()
+	if err != nil {
+		return nil, nil, err
+	}
+	return resp, body, nil
+}
+
+// creates a new echoserver for the given configuration
+func newServerForStack(t *testing.T, st serverStack) *EchoServer {
+	t.Helper()
+	if st.ipv6 {
+		return NewIPv6(t, st.tls, st.h2)
+	}
+	return New(t, st.tls, st.h2)
+}
+
+// shorthand for our http.Client
+func httpClient() *http.Client {
+	return &http.Client{Transport: &http.Transport{
+		TLSClientConfig:   &tls.Config{InsecureSkipVerify: true},
+		ForceAttemptHTTP2: true,
+	}}
+}

From 0e699ba24ef923143b9728d565299228f0a8991d Mon Sep 17 00:00:00 2001
From: AltayAkkus <7849969+AltayAkkus@users.noreply.github.com>
Date: Sat, 18 Apr 2026 18:43:58 +0200
Subject: [PATCH 3/6] e2e tests: warcvalidator

---
 e2e/warcvalidator/digest.go             | 136 ++++++++++++++++
 e2e/warcvalidator/parse.go              |  39 +++++
 e2e/warcvalidator/read.go               |  84 ++++++++++
 e2e/warcvalidator/revisit.go            |  46 ++++++
 e2e/warcvalidator/utils.go              |  25 +++
 e2e/warcvalidator/warcvalidator.go      | 158 +++++++++++++++++++
 e2e/warcvalidator/warcvalidator_test.go | 200 ++++++++++++++++++++++++
 7 files changed, 688 insertions(+)
 create mode 100644 e2e/warcvalidator/digest.go
 create mode 100644 e2e/warcvalidator/parse.go
 create mode 100644 e2e/warcvalidator/read.go
 create mode 100644 e2e/warcvalidator/revisit.go
 create mode 100644 e2e/warcvalidator/utils.go
 create mode 100644 e2e/warcvalidator/warcvalidator.go
 create mode 100644 e2e/warcvalidator/warcvalidator_test.go

diff --git a/e2e/warcvalidator/digest.go b/e2e/warcvalidator/digest.go
new file mode 100644
index 0000000..626b21b
--- /dev/null
+++ b/e2e/warcvalidator/digest.go
@@ -0,0 +1,136 @@
+package warcvalidator
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	warc "github.com/internetarchive/gowarc"
+)
+
+// validateRecordDigests checks WARC-Block-Digest and WARC-Payload-Digest where applicable.
+// Logic follows cmd/warc/verify (block always when present; payload for full HTTP payloads only).
+// Revisit records keep a truncated block — block digest still must match stored bytes;
+// payload digest in the header must match the referred response and is checked in ValidateRevisitAgainstOriginal.
+func validateRecordDigests(rec *warc.Record, filePath string) error {
+	id := rec.Header.Get("WARC-Record-ID")
+	if err := validateWARCVersion(rec, filePath, id); err != nil {
+		return err
+	}
+	if err := validateBlockDigest(rec, filePath, id); err != nil {
+		return err
+	}
+	if rec.Header.Get("WARC-Type") == "revisit" {
+		// we cannot validate the payload digest of a revisit record
+		// because there is no payload to digest
+		return nil
+	}
+	return validatePayloadDigest(rec, filePath, id)
+}
+
+func validateWARCVersion(rec *warc.Record, filePath, recordID string) error {
+	v := rec.Version
+	if strings.ContainsAny(v, "\r\n") {
+		return fmt.Errorf("%s: record %s: WARC version contains invalid characters", filePath, recordID)
+	}
+	if v != "WARC/1.0" && v != "WARC/1.1" {
+		return fmt.Errorf("%s: record %s: invalid WARC version %q (want WARC/1.0 or WARC/1.1)", filePath, recordID, v)
+	}
+	return nil
+}
+
+func validateBlockDigest(rec *warc.Record, filePath, recordID string) error {
+	blockDigest := rec.Header.Get("WARC-Block-Digest")
+	if blockDigest == "" {
+		return fmt.Errorf("%s: record %s: WARC-Block-Digest is missing", filePath, recordID)
+	}
+	prefix := strings.SplitN(blockDigest, ":", 2)[0]
+	if !warc.IsDigestSupported(prefix) {
+		return fmt.Errorf("%s: record %s: unsupported WARC-Block-Digest algorithm %q", filePath, recordID, prefix)
+	}
+	if _, err := rec.Content.Seek(0, io.SeekStart); err != nil {
+		return fmt.Errorf("%s: record %s: seek content: %w", filePath, recordID, err)
+	}
+	defer rec.Content.Seek(0, io.SeekStart)
+
+	algo := warc.GetDigestFromPrefix(prefix)
+	got, err := warc.GetDigest(rec.Content, algo)
+	if err != nil {
+		return fmt.Errorf("%s: record %s: block digest compute: %w", filePath, recordID, err)
+	}
+	if got != blockDigest {
+		return fmt.Errorf("%s: record %s: WARC-Block-Digest mismatch: header %q recomputed %q", filePath, recordID, blockDigest, got)
+	}
+	return nil
+}
+
+func validatePayloadDigest(rec *warc.Record, filePath, recordID string) error {
+	want := rec.Header.Get("WARC-Payload-Digest")
+	if want == "" {
+		return nil
+	}
+	ct := rec.Header.Get("Content-Type")
+	switch {
+	case strings.Contains(ct, "msgtype=response"):
+		return validateHTTPResponsePayloadDigest(rec, filePath, recordID, want)
+	case strings.Contains(ct, "msgtype=request"):
+		return validateHTTPRequestPayloadDigest(rec, filePath, recordID, want)
+	default:
+		return nil
+	}
+}
+
+func validateHTTPResponsePayloadDigest(rec *warc.Record, filePath, recordID, want string) error {
+	if _, err := rec.Content.Seek(0, io.SeekStart); err != nil {
+		return fmt.Errorf("%s: record %s: seek content: %w", filePath, recordID, err)
+	}
+	resp, err := http.ReadResponse(bufio.NewReader(rec.Content), nil)
+	if err != nil {
+		return fmt.Errorf("%s: record %s: read HTTP response: %w", filePath, recordID, err)
+	}
+	defer resp.Body.Close()
+	defer rec.Content.Seek(0, io.SeekStart)
+
+	if resp.Header.Get("X-Crawler-Transfer-Encoding") != "" || resp.Header.Get("X-Crawler-Content-Encoding") != "" {
+		return fmt.Errorf("%s: record %s: crawler transport/content-encoding headers prevent payload digest verification", filePath, recordID)
+	}
+	prefix := strings.SplitN(want, ":", 2)[0]
+	if !warc.IsDigestSupported(prefix) {
+		return fmt.Errorf("%s: record %s: unsupported WARC-Payload-Digest algorithm %q", filePath, recordID, prefix)
+	}
+	got, err := warc.GetDigest(resp.Body, warc.GetDigestFromPrefix(prefix))
+	if err != nil {
+		return fmt.Errorf("%s: record %s: payload digest compute: %w", filePath, recordID, err)
+	}
+	if got != want {
+		return fmt.Errorf("%s: record %s: WARC-Payload-Digest mismatch: header %q recomputed %q", filePath, recordID, want, got)
+	}
+	return nil
+}
+
+func validateHTTPRequestPayloadDigest(rec *warc.Record, filePath, recordID, want string) error {
+	if _, err := rec.Content.Seek(0, io.SeekStart); err != nil {
+		return fmt.Errorf("%s: record %s: seek content: %w", filePath, recordID, err)
+	}
+	req, err := http.ReadRequest(bufio.NewReader(rec.Content))
+	if err != nil {
+		return fmt.Errorf("%s: record %s: read HTTP request: %w", filePath, recordID, err)
+	}
+	defer req.Body.Close()
+	defer rec.Content.Seek(0, io.SeekStart)
+
+	prefix := strings.SplitN(want, ":", 2)[0]
+	if !warc.IsDigestSupported(prefix) {
+		return fmt.Errorf("%s: record %s: unsupported WARC-Payload-Digest algorithm %q", filePath, recordID, prefix)
+	}
+	got, err := warc.GetDigest(req.Body, warc.GetDigestFromPrefix(prefix))
+	if err != nil {
+		return fmt.Errorf("%s: record %s: request payload digest compute: %w", filePath, recordID, err)
+	}
+	if got != want {
+		return fmt.Errorf("%s: record %s: WARC-Payload-Digest mismatch: header %q recomputed %q", filePath, recordID, want, got)
+	}
+	return nil
+}
diff --git a/e2e/warcvalidator/parse.go b/e2e/warcvalidator/parse.go
new file mode 100644
index 0000000..1ff3cd8
--- /dev/null
+++ b/e2e/warcvalidator/parse.go
@@ -0,0 +1,39 @@
+package warcvalidator
+
+import (
+	"bufio"
+	"io"
+	"net/http"
+
+	"github.com/internetarchive/gowarc/e2e/echoserver/imperative"
+	"github.com/internetarchive/gowarc/pkg/spooledtempfile"
+)
+
+// parseHTTPResponse parses HTTP response from WARC content.
+func parseHTTPResponse(content spooledtempfile.ReadSeekCloser) (*http.Response, error) {
+	content.Seek(0, io.SeekStart)
+	return http.ReadResponse(bufio.NewReader(content), nil)
+}
+
+// parseHTTPRequest parses HTTP request from WARC content.
+// When passing a revisit record, reading the body will cause a panic!
+func parseHTTPRequest(content spooledtempfile.ReadSeekCloser) (*http.Request, error) {
+	content.Seek(0, io.SeekStart)
+	return http.ReadRequest(bufio.NewReader(content))
+}
+
+func imperativeFromRequest(request *http.Request) (imperative.Imperative, error) {
+	imp, err := imperative.ParseImperative(request.Header.Get(imperative.ImperativeHeader))
+	if err != nil {
+		return imperative.Imperative{}, err
+	}
+	return imp, nil
+}
+
+func imperativeFromResponse(response *http.Response) (imperative.Imperative, error) {
+	imp, err := imperative.ParseImperative(response.Header.Get(imperative.ImperativeHeader))
+	if err != nil {
+		return imperative.Imperative{}, err
+	}
+	return imp, nil
+}
diff --git a/e2e/warcvalidator/read.go b/e2e/warcvalidator/read.go
new file mode 100644
index 0000000..9c56df7
--- /dev/null
+++ b/e2e/warcvalidator/read.go
@@ -0,0 +1,84 @@
+package warcvalidator
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+
+	warc "github.com/internetarchive/gowarc"
+)
+
+// readWARCs reads all WARC files from a directory and returns a map of records keyed by WARC-Record-ID.
+func readWARCs(dir string) (map[string]*warc.Record, error) {
+	files, err := filepath.Glob(filepath.Join(dir, "*"))
+	if err != nil {
+		return nil, err
+	}
+
+	if len(files) == 0 {
+		return nil, nil
+	}
+
+	sort.Strings(files)
+
+	allRecords := make(map[string]*warc.Record)
+
+	for _, file := range files {
+		stat, err := os.Stat(file)
+		if err != nil || stat.IsDir() {
+			continue
+		}
+
+		records, err := readWARCFile(file)
+		if err != nil {
+			return nil, err
+		}
+
+		for id, rec := range records {
+			allRecords[id] = rec
+		}
+	}
+
+	return allRecords, nil
+}
+
+// readWARCFile reads a single WARC file and returns its records as a map keyed by WARC-Record-ID.
+func readWARCFile(file string) (map[string]*warc.Record, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	reader, err := warc.NewReader(f)
+	if err != nil {
+		return nil, err
+	}
+
+	records := make(map[string]*warc.Record)
+
+	for {
+		rec, err := reader.ReadRecord()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		if err := validateRecordDigests(rec, file); err != nil {
+			rec.Content.Close()
+			return nil, err
+		}
+		// we only care about request, response and revisit records
+		// info records are dropped (digests still validated above)
+		if isInfoRecord(rec) {
+			rec.Content.Close()
+			continue
+		}
+		recordID := rec.Header.Get("WARC-Record-ID")
+		records[recordID] = rec
+	}
+
+	return records, nil
+}
diff --git a/e2e/warcvalidator/revisit.go b/e2e/warcvalidator/revisit.go
new file mode 100644
index 0000000..d2c9be2
--- /dev/null
+++ b/e2e/warcvalidator/revisit.go
@@ -0,0 +1,46 @@
+package warcvalidator
+
+import (
+	"fmt"
+
+	warc "github.com/internetarchive/gowarc"
+)
+
+func validateRevisit(recordMap map[string]*warc.Record, recordID string) (*warc.Record, error) {
+	// lookup revisit in recordMap
+	revisit := recordMap[recordID]
+	if revisit == nil {
+		// we panic because this should never happen
+		panic(fmt.Sprintf("revisit record not found for record: %s", recordID))
+	}
+	// read WARC-Refers-To header
+	refersToID := revisit.Header.Get("WARC-Refers-To")
+	if refersToID == "" {
+		return nil, ValidationError{
+			Message:           fmt.Sprintf("No WARC-Refers-To header found in revisit record: %s", recordID),
+			RelatedImperative: nil,
+			Record:            revisit,
+		}
+	}
+	// lookup referred-to record in recordMap
+	refersToRecord := recordMap[refersToID]
+	if refersToRecord == nil {
+		return nil, ValidationError{
+			Message:           fmt.Sprintf("refers to record not found for revisit: %s -> %s", recordID, refersToID),
+			RelatedImperative: nil,
+			Record:            revisit,
+		}
+	}
+	// digests were already validated when reading the records from file
+	// we check if the payload digests match
+	if revisit.Header.Get("WARC-Payload-Digest") != refersToRecord.Header.Get("WARC-Payload-Digest") {
+		return nil, ValidationError{
+			Message:           fmt.Sprintf("WARC-Payload-Digest mismatch for revisit: %s -> %s", recordID, refersToID),
+			RelatedImperative: nil,
+			Record:            revisit,
+		}
+	}
+
+	// return the referred-to record
+	return refersToRecord, nil
+}
diff --git a/e2e/warcvalidator/utils.go b/e2e/warcvalidator/utils.go
new file mode 100644
index 0000000..26ae9d4
--- /dev/null
+++ b/e2e/warcvalidator/utils.go
@@ -0,0 +1,25 @@
+package warcvalidator
+
+import (
+	warc "github.com/internetarchive/gowarc"
+)
+
+// true if WARC-Type is "request"
+func isRequestRecord(rec *warc.Record) bool {
+	return rec.Header.Get("WARC-Type") == "request"
+}
+
+// true if WARC-Type is "response"
+func isResponseRecord(rec *warc.Record) bool {
+	return rec.Header.Get("WARC-Type") == "response"
+}
+
+// true if WARC-Type is "revisit"
+func isRevisitRecord(rec *warc.Record) bool {
+	return rec.Header.Get("WARC-Type") == "revisit"
+}
+
+// true if WARC-Type is not "request", "response" or "revisit"
+func isInfoRecord(rec *warc.Record) bool {
+	return !(isResponseRecord(rec) || isRequestRecord(rec) || isRevisitRecord(rec))
+}
diff --git a/e2e/warcvalidator/warcvalidator.go b/e2e/warcvalidator/warcvalidator.go
new file mode 100644
index 0000000..7aa75cb
--- /dev/null
+++ b/e2e/warcvalidator/warcvalidator.go
@@ -0,0 +1,158 @@
+package warcvalidator
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strings"
+
+	warc "github.com/internetarchive/gowarc"
+	"github.com/internetarchive/gowarc/e2e/echoserver/imperative"
+)
+
+func (e ValidationError) Error() string { return e.Message }
+
+type ValidationError struct {
+	Message           string
+	RelatedImperative *imperative.Imperative // is nil if not related to an imperative
+	Record            *warc.Record
+}
+
+func newValidationError(record *warc.Record, imp *imperative.Imperative, format string, args ...any) ValidationError {
+	return ValidationError{
+		Message:           fmt.Sprintf(format, args...),
+		RelatedImperative: imp,
+		Record:            record,
+	}
+}
+
+type Counts struct {
+	Request  int
+	Response int
+	Revisit  int
+}
+
+func (c Counts) Increment(recordType string) Counts {
+	switch recordType {
+	case "request":
+		c.Request++
+	case "response":
+		c.Response++
+	case "revisit":
+		c.Revisit++
+	default:
+		panic(fmt.Sprintf("invalid record type: %s", recordType))
+	}
+	return c
+}
+
+// Check validates the WARC files in the given directory against the given imperatives.
+// On success it returns (nil or empty slice, nil). Validation findings are returned as
+// a slice; I/O or read failures return (_, err).
+func Check(outDir string) ([]ValidationError, map[string]Counts) {
+	var validationErrs []ValidationError
+	tracker := make(map[string]Counts)
+
+	fmt.Printf("Checking %s\n", outDir)
+	records, err := readWARCs(outDir)
+	if err != nil {
+		validationErrs = append(validationErrs, newValidationError(nil, nil, "Failed to read WARC files: %s", err))
+		return validationErrs, nil
+	}
+	fmt.Printf("Found %d records in %s\n", len(records), outDir)
+	for _, record := range records {
+		if isRequestRecord(record) {
+			request, err := parseHTTPRequest(record.Content)
+			if err != nil {
+				validationErrs = append(validationErrs, newValidationError(record, nil, "Failed to parse request: %s", err))
+				continue
+			}
+			imp, err := imperativeFromRequest(request)
+			if err != nil {
+				validationErrs = append(validationErrs, newValidationError(record, nil, "Failed to parse imperative from request: %s", err))
+				continue
+			}
+			tracker[imp.Hash()] = tracker[imp.Hash()].Increment("request")
+
+			body, err := io.ReadAll(request.Body)
+			if err != nil {
+				validationErrs = append(validationErrs, newValidationError(record, &imp, "Failed to read request body: %s", err))
+				continue
+			}
+			err = imperative.AssertRequest(request, body)
+			if err != nil {
+				validationErrs = append(validationErrs, newValidationError(record, &imp, "Failed to assert request: %s", err))
+				continue
+			}
+		}
+		if isResponseRecord(record) {
+			response, err := parseHTTPResponse(record.Content)
+			if err != nil {
+				validationErrs = append(validationErrs, newValidationError(record, nil, "Failed to parse response: %s", err))
+				continue
+			}
+			imp, err := imperativeFromResponse(response)
+			if err != nil {
+				validationErrs = append(validationErrs, newValidationError(record, nil, "Failed to parse imperative from response: %s", err))
+				continue
+			}
+			tracker[imp.Hash()] = tracker[imp.Hash()].Increment("response")
+
+			body, err := io.ReadAll(response.Body)
+			if err != nil {
+				validationErrs = append(validationErrs, newValidationError(record, &imp, "Failed to read response body: %s", err))
+				continue
+			}
+			err = imperative.AssertResponse(response, body)
+			if err != nil {
+				validationErrs = append(validationErrs, newValidationError(record, &imp, "Failed to assert response: %s", err))
+				continue
+			}
+		}
+		if isRevisitRecord(record) {
+			// only validates if the referred-to record exists, and payload digests match
+			referredToRecord, err := validateRevisit(records, record.Header.Get("WARC-Record-ID"))
+			if err != nil {
+				validationErrs = append(validationErrs, newValidationError(record, nil, "Failed to validate revisit: %s", err))
+				continue
+			}
+
+			// calculate the body from the imperative, digest it, and compare it to the revisit record's payload digest
+
+			response, err := parseHTTPResponse(record.Content)
+			if err != nil {
+				validationErrs = append(validationErrs, newValidationError(referredToRecord, nil, "Failed to parse revisit: %s", err))
+				continue
+			}
+			// get the imperative from the response
+			imp, err := imperativeFromResponse(response)
+			if err != nil {
+				validationErrs = append(validationErrs, newValidationError(referredToRecord, nil, "Failed to parse imperative from revisit: %s", err))
+				continue
+			}
+			tracker[imp.Hash()] = tracker[imp.Hash()].Increment("revisit")
+
+			// calculate the digest of the expected response body
+			calculatedBody, err := imp.Response.Body.Build()
+			if err != nil {
+				validationErrs = append(validationErrs, newValidationError(referredToRecord, &imp, "Failed to build response body: %s", err))
+				continue
+			}
+			// calculate digest
+			// digest contains the correct prefix
+			digestAlgo := warc.GetDigestFromPrefix(strings.SplitN(record.Header.Get("WARC-Payload-Digest"), ":", 2)[0])
+			digest, err := warc.GetDigest(bytes.NewReader(calculatedBody), digestAlgo)
+			if err != nil {
+				validationErrs = append(validationErrs, newValidationError(referredToRecord, &imp, "Failed to calculate digest of response body: %s", err))
+				continue
+			}
+			// compare the prefix + digest of the calculated response body with the prefix + digest of the response body
+			if digest != record.Header.Get("WARC-Payload-Digest") {
+				validationErrs = append(validationErrs, newValidationError(referredToRecord, &imp, "WARC-Payload-Digest mismatch: %s != %s", digest, response.Header.Get("WARC-Payload-Digest")))
+				continue
+			}
+		}
+	}
+
+	return validationErrs, tracker
+}
diff --git a/e2e/warcvalidator/warcvalidator_test.go b/e2e/warcvalidator/warcvalidator_test.go
new file mode 100644
index 0000000..d456a5d
--- /dev/null
+++ b/e2e/warcvalidator/warcvalidator_test.go
@@ -0,0 +1,200 @@
+package warcvalidator
+
+import (
+	"io"
+	"math/rand/v2"
+	"testing"
+
+	warc "github.com/internetarchive/gowarc"
+	"github.com/internetarchive/gowarc/e2e/echoserver"
+	"github.com/internetarchive/gowarc/e2e/echoserver/imperative"
+)
+
+// TestCheck runs the Check method with real WARC files from gowarc.
+func TestCheck(t *testing.T) {
+	// Create output directories
+	outDir := t.TempDir()
+	spoolDir := t.TempDir()
+
+	// Create and start echo server
+	srv := echoserver.New(t, true, true)
+
+	// Create gowarc client
+	rs := warc.NewRotatorSettings()
+	rs.OutputDirectory = outDir
+	rs.Prefix = "TEST"
+	rs.WARCSize = 10 << 20
+	rs.WARCWriterPoolSize = 1
+
+	client, err := warc.NewWARCWritingHTTPClient(warc.HTTPClientSettings{
+		RotatorSettings: rs,
+		TempDir:         spoolDir,
+		VerifyCerts:     false,
+		DedupeOptions:   warc.DedupeOptions{LocalDedupe: true, SizeThreshold: 512},
+	})
+	if err != nil {
+		t.Fatalf("Failed to create WARC client: %v", err)
+	}
+
+	// Drain error channel
+	go func() {
+		for e := range client.ErrChan {
+			t.Logf("WARC error: %v (func: %s)", e.Err, e.Func)
+		}
+	}()
+
+	// Generate some imperatives
+	var imps imperative.Imperatives
+	for _ = range 100 {
+		imps = append(imps, imperative.Imperative{
+			Response: imperative.Response{
+				StatusCode: randomDraw(200, 201, 404, 500),
+				Headers:    randomHeaders(),
+				Body: imperative.Body{
+					Length: rand.IntN(1024 * 1024 * 5), // 5MB
+				},
+			},
+			Request: imperative.Request{
+				Path:    randomPath(),
+				Headers: randomHeaders(),
+				Method:  randomDraw("GET", "POST", "PUT", "DELETE", "OPTIONS"),
+				Body: imperative.Body{
+					Length: rand.IntN(1024 * 1024 * 1), // 1MB
+				},
+			},
+			Transport: imperative.Transport{
+				Abort: false,
+			},
+		})
+	}
+
+	// seed & shuffle
+	imps = imps.Seed(1337).Shuffle(42)
+	// double a few imperatives to test with duplicates
+	imps = append(imps, imps[0].Multiply(10)...)
+	// additional imperative to test with unfinished imperatives
+	_ = append(imps, imperative.Imperative{
+		Request: imperative.Request{
+			Path:    "/test",
+			Headers: map[string]string{"Content-Type": "text/html"},
+			Method:  "GET",
+		},
+		Response: imperative.Response{
+			StatusCode: 200,
+			Headers:    map[string]string{"Content-Type": "text/html"},
+			Body: imperative.Body{
+				Length: 1024,
+				Seed:   42,
+			},
+		},
+	})
+
+	// send requests using gowarc client
+	for _, imp := range imps {
+		req, err := imp.BuildRequest(srv.URL)
+		if err != nil {
+			t.Fatalf("Failed to build request: %v", err)
+		}
+		resp, err := client.Do(req)
+		if err != nil {
+			if imp.Aborts() {
+				t.Logf("request error (expected when server aborts connection): %v", err)
+				continue
+			}
+			t.Fatalf("Failed to send request: %v", err)
+		}
+
+		body, err := io.ReadAll(resp.Body)
+		if closeErr := resp.Body.Close(); closeErr != nil && err == nil {
+			err = closeErr
+		}
+		if err != nil {
+			if imp.Aborts() {
+				t.Logf("read response error (expected when server aborts connection): %v", err)
+				continue
+			}
+			t.Fatalf("Failed to read response body: %v", err)
+		}
+		if err := imperative.AssertResponse(resp, body); err != nil {
+			t.Fatalf("Failed to assert response: %v", err)
+		}
+	}
+
+	if err := client.Close(); err != nil {
+		t.Fatalf("Failed to close client: %v", err)
+	}
+
+	// Run warcvalidator.Check
+	errs, tracker := Check(outDir)
+	if len(errs) > 0 {
+		t.Fatalf("Check failed: %v", errs)
+	}
+
+	// log the tracker for debugging
+	for impHash, count := range tracker {
+		t.Logf("ImpHash: %s, Count: %+v\n", impHash, count)
+	}
+
+	// loop through all imperatives, look them up in the tracker and check if the counts are correct
+	for hash, imps := range imps.Hash() {
+		// how many imperatives with that hash are in our imps slice
+		numImperatives := len(imps)
+		// look up the counter for this imperative
+		count := tracker[hash]
+
+		// tifo: gowarc does not store request records for failed connections!
+		if count.Request != numImperatives && !imps[0].Aborts() {
+			t.Errorf("Request count mismatch: 1 got %d want %d", count.Request, 1)
+		}
+		// if the imperative does not abort, there should be one response
+		// the other one's should be revisits
+		if count.Response != 1 && !imps[0].Aborts() {
+			t.Errorf("Response count mismatch: 2 got %d want %d", count.Response, 1)
+		}
+		if count.Response != 0 && imps[0].Aborts() {
+			t.Errorf("Response count mismatch: imp aborts, got %d want 0", count.Response)
+		}
+
+		// yeah, complex logic here...
+		if count.Revisit != numImperatives-1 && !imps[0].Aborts() || imps[0].Aborts() && count.Revisit != 0 {
+			t.Errorf("Revisit count mismatch: 3 got %d want %d", count.Revisit, 1)
+		}
+	}
+}
+
+func stringWithCharset(length int, charset string) string {
+	seededRand := rand.New(rand.NewPCG(1337, 42))
+	b := make([]byte, length)
+	for i := range b {
+		b[i] = charset[seededRand.IntN(len(charset))]
+	}
+	return string(b)
+}
+
+const headerCharset = "abcdefghijklmnopqrstuvwxyz0123456789-_"
+
+// Returns a random map[string]string with header compatible values
+func randomHeaders() map[string]string {
+	headers := make(map[string]string)
+	for i := 0; i < rand.IntN(3); i++ {
+		headers[stringWithCharset(1+rand.IntN(16), headerCharset)] = stringWithCharset(1+rand.IntN(32), headerCharset)
+	}
+	return headers
+}
+
+const pathCharset = headerCharset + "/"
+
+// Returns a random path string
+func randomPath() string {
+	return "/" + stringWithCharset(1+rand.IntN(16), pathCharset)
+	// could also generate random query string
+}
+
+// randomDraw returns one random element from a slice of choices.
+func randomDraw[T any](choices ...T) T {
+	if len(choices) == 0 {
+		var z T
+		return z
+	}
+	return choices[rand.IntN(len(choices))]
+}

From 956af4a2f23312e5005b80e0dbcbb03aa9661330 Mon Sep 17 00:00:00 2001
From: AltayAkkus <7849969+AltayAkkus@users.noreply.github.com>
Date: Sat, 18 Apr 2026 18:44:14 +0200
Subject: [PATCH 4/6] e2e tests: proxy

---
 e2e/proxy/socks5.go      | 45 ++++++++++++++++++++++++++
 e2e/proxy/socks5_test.go | 68 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 113 insertions(+)
 create mode 100644 e2e/proxy/socks5.go
 create mode 100644 e2e/proxy/socks5_test.go

diff --git a/e2e/proxy/socks5.go b/e2e/proxy/socks5.go
new file mode 100644
index 0000000..0923e54
--- /dev/null
+++ b/e2e/proxy/socks5.go
@@ -0,0 +1,45 @@
+package proxy
+
+import (
+	"context"
+	"errors"
+	"net"
+	"sync/atomic"
+	"testing"
+
+	"github.com/things-go/go-socks5"
+)
+
+// socks5.RuleSet that permits all connections and increments a counter on every request.
+type countingRuleSet struct {
+	count atomic.Int64
+}
+
+func (r *countingRuleSet) Allow(ctx context.Context, req *socks5.Request) (context.Context, bool) {
+	r.count.Add(1)
+	return ctx, true
+}
+
+// NewSOCKS5Proxy starts a SOCKS5 proxy on a random port and returns its
+// address, a connection counter, and a cleanup function that stops the server.
+func NewSOCKS5Proxy(t *testing.T) (addr string, count *atomic.Int64) {
+	t.Helper()
+
+	rule := &countingRuleSet{}
+	server := socks5.NewServer(socks5.WithRule(rule))
+
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("failed to listen for proxy: %v", err)
+	}
+
+	go func() {
+		if err := server.Serve(listener); err != nil && !errors.Is(err, net.ErrClosed) {
+			panic(err)
+		}
+	}()
+
+	t.Cleanup(func() { listener.Close() })
+
+	return listener.Addr().String(), &rule.count
+}
diff --git a/e2e/proxy/socks5_test.go b/e2e/proxy/socks5_test.go
new file mode 100644
index 0000000..fd22f48
--- /dev/null
+++ b/e2e/proxy/socks5_test.go
@@ -0,0 +1,68 @@
+package proxy
+
+import (
+	"crypto/tls"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+)
+
+func newTestServer(t *testing.T) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	srv.EnableHTTP2 = false
+	srv.StartTLS()
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func testClient(proxyURL *url.URL) *http.Client {
+	return &http.Client{
+		Transport: &http.Transport{
+			Proxy:             http.ProxyURL(proxyURL),
+			DisableKeepAlives: true, // disable keep-alive to avoid connection re-use
+			TLSClientConfig:   &tls.Config{InsecureSkipVerify: true},
+		}}
+}
+
+func TestSOCKS5Proxy(t *testing.T) {
+	addr, count := NewSOCKS5Proxy(t)
+	proxyURL, _ := url.Parse("socks5://" + addr)
+	client := testClient(proxyURL)
+
+	srv := newTestServer(t)
+
+	resp, err := client.Get(srv.URL)
+	if err != nil {
+		t.Fatalf("request to %s failed: %v", srv.URL, err)
+	}
+	resp.Body.Close()
+
+	if got := count.Load(); got != 1 {
+		t.Errorf("expected 1 proxied connection, got %d", got)
+	}
+}
+
+func TestSOCKS5ProxyCountsEachConnection(t *testing.T) {
+	addr, count := NewSOCKS5Proxy(t)
+
+	proxyURL, _ := url.Parse("socks5://" + addr)
+	client := testClient(proxyURL)
+
+	srv := newTestServer(t)
+	requests := 5
+	for i := 0; i < requests; i++ {
+		resp, err := client.Get(srv.URL)
+		if err != nil {
+			t.Fatalf("request to %s failed: %v", srv.URL, err)
+		}
+		resp.Body.Close()
+	}
+
+	if got := count.Load(); got != int64(requests) {
+		t.Errorf("expected %d proxied connections, got %d", requests, got)
+	}
+}

From 0c77605ebdf5125d67022eb6481173f0d0c1125d Mon Sep 17 00:00:00 2001
From: AltayAkkus <7849969+AltayAkkus@users.noreply.github.com>
Date: Sat, 18 Apr 2026 18:44:30 +0200
Subject: [PATCH 5/6] e2e tests: initial load test

---
 e2e/load_test.go | 302 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 302 insertions(+)
 create mode 100644 e2e/load_test.go

diff --git a/e2e/load_test.go b/e2e/load_test.go
new file mode 100644
index 0000000..3b33f95
--- /dev/null
+++ b/e2e/load_test.go
@@ -0,0 +1,302 @@
+package e2e
+
+import (
+	"bytes"
+	"io"
+	"math/rand/v2"
+	"os"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	warc "github.com/internetarchive/gowarc"
+	"github.com/internetarchive/gowarc/e2e/echoserver"
+	"github.com/internetarchive/gowarc/e2e/echoserver/imperative"
+	"github.com/internetarchive/gowarc/e2e/proxy"
+	"github.com/internetarchive/gowarc/e2e/warcvalidator"
+)
+
+// EchoServer configuration stacks
+// 8 server stacks = plain/TLS × HTTP1/HTTP2 × IPv4/IPv6.
+var loadStacks = []struct {
+	name          string
+	tls, h2, ipv6 bool
+}{
+	{"plain_H1_IPv4", false, false, false},
+	//{"plain_H1_IPv6", false, false, true},
+	{"plain_H2_IPv4", false, true, false},
+	//{"plain_H2_IPv6", false, true, true},
+	{"TLS_H1_IPv4", true, false, false},
+	//{"TLS_H1_IPv6", true, false, true},
+	{"TLS_H2_IPv4", true, true, false},
+	//{"TLS_H2_IPv6", true, true, true},
+}
+
+// Set this env var to route the "direct" subtest's traffic through a
+// proxy (e.g. Proxyman at http://127.0.0.1:9090) so request/responses can be
+// inspected live. Empty = no proxy.
+const proxyEnvVar = "GOWARC_E2E_PROXY"
+
+// Runs twice: once without a proxy (opt. GOWARC_E2E_PROXY for debugging),
+// and once through our counting SOCKS5 proxy.
+func TestLoadWARCWritingHTTPClient(t *testing.T) {
+	t.Run("direct", func(t *testing.T) {
+		// you can supply your own proxy via the ENV var.
+		// Good for debugging with e.g. Proxyman, mitmproxy et cetera.
+		runLoad(t, os.Getenv(proxyEnvVar), nil)
+	})
+	t.Run("socks5", func(t *testing.T) {
+		addr, counter := proxy.NewSOCKS5Proxy(t)
+		runLoad(t, "socks5://"+addr, counter)
+	})
+}
+
+// runLoad executes the full load scenario (>1k concurrent requests), asserts responses and the WARC files.
+// optionally uses a proxy, and verifies that all requests were routed through it.
+func runLoad(t *testing.T, proxyURL string, proxyCounter *atomic.Int64) {
+	t.Helper()
+	outDir, spoolDir := t.TempDir(), t.TempDir()
+
+	if proxyURL != "" {
+		t.Logf("routing through proxy: %s", proxyURL)
+	}
+
+	// Spin up an echoserver per stack.
+	urls := make([]string, len(loadStacks))
+	for i, s := range loadStacks {
+		var srv *echoserver.EchoServer
+		if s.ipv6 {
+			srv = echoserver.NewIPv6(t, s.tls, s.h2)
+		} else {
+			srv = echoserver.New(t, s.tls, s.h2)
+		}
+		urls[i] = srv.URL
+	}
+
+	// 1 MiB static file to exercise imperative.Body.Filepath.
+	staticFile := filepath.Join(t.TempDir(), "static.bin")
+	if err := os.WriteFile(staticFile, bytes.Repeat([]byte{0xAB}, 1<<20), 0o644); err != nil {
+		t.Fatalf("write static file: %v", err)
+	}
+
+	imps := buildLoadImperatives(staticFile)
+	t.Logf("load: %d imperatives across %d stacks", len(imps), len(loadStacks))
+
+	// gowarc client with many options turned on.
+	rs := warc.NewRotatorSettings()
+	rs.OutputDirectory = outDir
+	rs.Prefix = "LOAD"
+	rs.WARCSize = 20
+	rs.WARCWriterPoolSize = 4
+	rs.Compression = warc.CompressionGzip
+
+	client, err := warc.NewWARCWritingHTTPClient(warc.HTTPClientSettings{
+		RotatorSettings:       rs,
+		TempDir:               spoolDir,
+		Proxy:                 proxyURL,
+		VerifyCerts:           false,
+		FollowRedirects:       false,
+		FullOnDisk:            false,
+		MaxReadBeforeTruncate: 32 << 20,
+		DialTimeout:           5 * time.Second,
+		ResponseHeaderTimeout: 15 * time.Second,
+		TLSHandshakeTimeout:   5 * time.Second,
+		DedupeOptions:         warc.DedupeOptions{LocalDedupe: true, SizeThreshold: 1024},
+		DigestAlgorithm:       warc.SHA1,
+	})
+	if err != nil {
+		t.Fatalf("NewWARCWritingHTTPClient: %v", err)
+	}
+	var warcErrs atomic.Int64
+	go func() {
+		for e := range client.ErrChan {
+			warcErrs.Add(1)
+			t.Logf("WARC ErrChan: %v (func: %s)", e.Err, e.Func)
+		}
+	}()
+
+	// Concurrent fan-out across all stacks.
+	const workers = 32
+	jobs := make(chan int, len(imps))
+	var wg sync.WaitGroup
+	for w := 0; w < workers; w++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for i := range jobs {
+				imp := imps[i]
+				req, err := imp.BuildRequest(urls[i%len(urls)])
+				if err != nil {
+					t.Errorf("BuildRequest[%d]: %v", i, err)
+					continue
+				}
+				resp, err := client.Do(req)
+				if err != nil {
+					if !imp.Aborts() {
+						t.Errorf("Do[%d]: %v", i, err)
+					}
+					continue
+				}
+				body, err := io.ReadAll(resp.Body)
+				resp.Body.Close()
+				if err != nil {
+					if !imp.Aborts() {
+						t.Errorf("read[%d]: %v", i, err)
+					}
+					continue
+				}
+				if !imp.Aborts() {
+					if err := imperative.AssertResponse(resp, body); err != nil {
+						t.Errorf("AssertResponse[%d]: %v", i, err)
+					}
+				}
+			}
+		}()
+	}
+	for i := range imps {
+		jobs <- i
+	}
+	close(jobs)
+	wg.Wait()
+
+	if err := client.Close(); err != nil {
+		t.Fatalf("client.Close: %v", err)
+	}
+
+	// Validate the WARC records, internally.
+	verrs, tracker := warcvalidator.Check(outDir)
+	for _, e := range verrs {
+		t.Errorf("validation errors: %s", e.Error())
+	}
+
+	// Check the existence of every imperative in our resulting WARC files.
+	for hash, group := range imps.Hash() {
+		// requests that were aborted do not show up in the WARC at all.
+		if group[0].Aborts() {
+			continue
+		}
+		c := tracker[hash]
+		// check that the sum of response + revisits records for that imperative
+		// equals the number of times we sent it.
+		if c.Response+c.Revisit != len(group) {
+			t.Errorf("hash=%s: want %d response+revisit, got %d+%d", hash, len(group), c.Response, c.Revisit)
+		}
+		// check that at least one response record was written
+		if c.Response == 0 {
+			t.Errorf("hash=%s: no response record written", hash)
+		}
+		// check that the number of revisits equals the number of times we sent it, minus one (for the initial response)
+		// WARNING: Since we race all these requests concurrently, it's possible that the local dedupe produces more than
+		// one response record for equal payload digest. Added a safety margin of 20 to account for this.
+		if c.Revisit < len(group)-21 {
+			t.Errorf("hash=%s: want %d revisit records, got %d", hash, len(group)-1, c.Revisit)
+		}
+		// check that the number of request records equals the number of times we sent it.
+		if c.Request != len(group) {
+			t.Errorf("hash=%s: want %d request records, got %d", hash, len(group), c.Request)
+		}
+	}
+	// we don't check the error counter against the number of imperatives that abort here,
+	// because the loop above already checks the existence of every non-aborting imperative in the WARC files.
+	if n := warcErrs.Load(); n > 0 {
+		t.Logf("observed %d ErrChan errors (aborts expected)", n)
+	}
+
+	// If a proxy counter was supplied, check that the number of connections observed by the proxy
+	// equals the number of imperatives that we sent.
+	if proxyCounter != nil {
+		n := proxyCounter.Load()
+		if n < int64(len(imps)) {
+			t.Errorf("proxy counter: want >=%d, got %d", len(imps), n)
+		} else {
+			t.Logf("proxy saw %d connections for %d imperatives", n, len(imps))
+		}
+	}
+}
+
+// Produces >1k imperatives covering every attribute that the Imperative struct supports.
+func buildLoadImperatives(staticFile string) imperative.Imperatives {
+	var (
+		methods      = []string{"GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"}
+		codes        = []int{200, 201, 400, 404, 418, 500, 503}
+		contentTypes = []string{"application/json", "text/plain", "application/octet-stream", "text/html"}
+		paths        = []string{"users", "events", "items", "docs", "stream"}
+		encodings    = []string{imperative.EncodingUTF8, imperative.EncodingBinary}
+		compressions = []string{"", "gzip", "deflate", "zstd"}
+	)
+	r := rand.New(rand.NewPCG(1337, 42))
+
+	var imps imperative.Imperatives
+
+	// 1000 diverse imperatives.
+	for i := 0; i < 1000; i++ {
+		method := methods[r.IntN(len(methods))]
+		reqLen, reqSeed := 0, 0
+		// Only methods that conventionally carry a body – HTTP/2 client transports
+		// reject request bodies on GET/DELETE/OPTIONS.
+		if method == "POST" || method == "PUT" || method == "PATCH" {
+			reqLen = r.IntN(32 << 10)
+			if reqLen > 0 {
+				reqSeed = 1 + r.IntN(1_000_000)
+			}
+		}
+		imps = append(imps, imperative.Imperative{
+			Request: imperative.Request{
+				Method:  method,
+				Path:    "/api/" + paths[r.IntN(len(paths))] + "/" + strconv.Itoa(i),
+				Headers: map[string]string{"X-Test-Run": "load", "X-Idx": strconv.Itoa(i)},
+				Body:    imperative.Body{Length: reqLen, Seed: reqSeed, Encoding: encodings[r.IntN(len(encodings))]},
+			},
+			Response: imperative.Response{
+				StatusCode: codes[r.IntN(len(codes))],
+				Headers: map[string]string{
+					"Content-Type":    contentTypes[r.IntN(len(contentTypes))],
+					"X-Server-Marker": strconv.Itoa(i),
+				},
+				Body: imperative.Body{
+					Length:   1 + r.IntN(128<<10),
+					Seed:     1 + r.IntN(1_000_000),
+					Encoding: encodings[r.IntN(len(encodings))],
+					Compress: compressions[r.IntN(len(compressions))],
+				},
+			},
+			Transport: imperative.Transport{Delay: r.IntN(20)},
+		})
+	}
+
+	// 100 identical requests → 1 response + 99 revisits with local dedupe enabled.
+	// The .Seed() shorthand creates a number between 0 and 1 million, we use 2 million to prevent random collisions.
+	dedupe := imperative.Imperative{
+		Request: imperative.Request{Method: "GET", Path: "/cached/doc"},
+		Response: imperative.Response{
+			StatusCode: 200,
+			Headers:    map[string]string{"Content-Type": "application/octet-stream"},
+			Body:       imperative.Body{Length: 64*1024 + 7, Seed: 2_000_000_001, Encoding: imperative.EncodingBinary},
+		},
+	}
+	imps = append(imps, dedupe.Multiply(100)...)
+
+	// 20 mid-response disconnects – the WARC client must not crash or hang.
+	abort := imperative.Imperative{
+		Request: imperative.Request{Method: "GET", Path: "/flaky"},
+		Response: imperative.Response{
+			StatusCode: 200,
+			Headers:    map[string]string{"Content-Type": "application/octet-stream"},
+			Body:       imperative.Body{Length: 128*1024 + 11, Seed: 2_000_000_002, Encoding: imperative.EncodingBinary},
+		},
+		Transport: imperative.Transport{Abort: true},
+	}
+	imps = append(imps, abort.Multiply(20)...)
+
+	// One Filepath-backed body
+	imps = append(imps, imperative.Imperative{
+		Request:  imperative.Request{Method: "GET", Path: "/static/blob"},
+		Response: imperative.Response{StatusCode: 200, Headers: map[string]string{"Content-Type": "application/octet-stream"}, Body: imperative.Body{Filepath: staticFile}},
+	})
+
+	// randomly shuffle the imperatives
+	return imps.Shuffle(77)
+}

From be5748c08a3e0fd723858728eafd04b2c81b6458 Mon Sep 17 00:00:00 2001
From: AltayAkkus <7849969+AltayAkkus@users.noreply.github.com>
Date: Sat, 18 Apr 2026 20:42:38 +0200
Subject: [PATCH 6/6] e2e tests: uncomment ipv6 stacks

---
 e2e/load_test.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/e2e/load_test.go b/e2e/load_test.go
index 3b33f95..8da3527 100644
--- a/e2e/load_test.go
+++ b/e2e/load_test.go
@@ -26,13 +26,13 @@ var loadStacks = []struct {
 	tls, h2, ipv6 bool
 }{
 	{"plain_H1_IPv4", false, false, false},
-	//{"plain_H1_IPv6", false, false, true},
+	{"plain_H1_IPv6", false, false, true},
 	{"plain_H2_IPv4", false, true, false},
-	//{"plain_H2_IPv6", false, true, true},
+	{"plain_H2_IPv6", false, true, true},
 	{"TLS_H1_IPv4", true, false, false},
-	//{"TLS_H1_IPv6", true, false, true},
+	{"TLS_H1_IPv6", true, false, true},
 	{"TLS_H2_IPv4", true, true, false},
-	//{"TLS_H2_IPv6", true, true, true},
+	{"TLS_H2_IPv6", true, true, true},
 }
 
 // Set this env var to route the "direct" subtest's traffic through a
@@ -263,7 +263,7 @@ func buildLoadImperatives(staticFile string) imperative.Imperatives {
 					Compress: compressions[r.IntN(len(compressions))],
 				},
 			},
-			Transport: imperative.Transport{Delay: r.IntN(20)},
+			Transport: imperative.Transport{Delay: r.IntN(1000)},
 		})
 	}