From e356611192b3c68e1988c67d04d188fa61f9dfa5 Mon Sep 17 00:00:00 2001 From: genematx Date: Thu, 24 Sep 2026 15:34:31 +1200 Subject: [PATCH 01/22] Add OpenTelemetry tracing and an example Jaeger/Collector stack Instrument the server with OpenTelemetry tracing, exported over OTLP and disabled unless OTEL_EXPORTER_OTLP_ENDPOINT is set. Health checks and metrics scrapes can be excluded via OTEL_PYTHON_FASTAPI_EXCLUDED_URLS. Add an OpenTelemetry Collector and Jaeger to the example monitoring stack. The Collector receives traces and forwards them to Jaeger, and also scrapes and re-exposes Tiled's Prometheus metrics. Add a 'Distributed Tracing' user-guide page. --- CHANGELOG.md | 3 + compose.dev.yml | 5 ++ compose.monitoring.yml | 32 ++++++++ compose.yml | 5 ++ docs/source/_toc.yml | 1 + docs/source/user-guide/tracing.md | 78 +++++++++++++++++++ .../otel-collector/otel-collector.yml | 62 +++++++++++++++ monitoring_example/prometheus/prometheus.yml | 4 + pyproject.toml | 6 ++ tiled/server/app.py | 38 +++++++++ 10 files changed, 234 insertions(+) create mode 100644 docs/source/user-guide/tracing.md create mode 100644 monitoring_example/otel-collector/otel-collector.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fbd9fd43..2879ca49c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ Write the date in place of the "Unreleased" in the case a new version is release ### Added +- OpenTelemetry tracing for the server, exported over OTLP. The example monitoring + stack (`compose.monitoring.yml`) now includes an OpenTelemetry Collector and + Jaeger, and the Collector also scrapes and re-exposes Tiled's Prometheus metrics. - Documentation: a user-guide page on validating metadata against custom specs via server configuration. diff --git a/compose.dev.yml b/compose.dev.yml index c35e3cb53..7ed91fcfa 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -14,6 +14,11 @@ services: - TILED_WEBHOOKS_ALLOW_DELIVERY_HOSTS=${TILED_WEBHOOKS_ALLOW_DELIVERY_HOSTS:-} - TILED_WEBHOOKS_ALLOW_HTTP=${TILED_WEBHOOKS_ALLOW_HTTP:-false} - TILED_WEBHOOKS_ALLOW_PRIVATE_ADDRESSES=${TILED_WEBHOOKS_ALLOW_PRIVATE_ADDRESSES:-false} + # Export OpenTelemetry traces to the collector (see compose.monitoring.yml). + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + - OTEL_SERVICE_NAME=tiled + # Don't trace health checks and metrics scrapes (operational chatter). + - OTEL_PYTHON_FASTAPI_EXCLUDED_URLS=healthz,api/v1/metrics volumes: - tiled_data:/storage ports: diff --git a/compose.monitoring.yml b/compose.monitoring.yml index 9f5a31a73..bdf359dd4 100644 --- a/compose.monitoring.yml +++ b/compose.monitoring.yml @@ -33,5 +33,37 @@ services: GF_AUTH_DISABLE_SIGNOUT_MENU: "true" GF_AUTH_DISABLE_LOGIN_FORM: "true" + # OpenTelemetry Collector: receives OTLP telemetry from apps and fans it + # out to backends (traces -> Jaeger). It also scrapes Tiled's Prometheus + # metrics endpoint and re-exposes it for Prometheus. Apps on the 'backend' + # network export to http://otel-collector:4318; apps on the host export to + # http://localhost:4318. + otel-collector: + image: otel/opentelemetry-collector-contrib:0.130.0 + command: ["--config=/etc/otelcol/config.yaml"] + volumes: + - ./monitoring_example/otel-collector/otel-collector.yml:/etc/otelcol/config.yaml + ports: + - 4317:4317 # OTLP gRPC + - 4318:4318 # OTLP HTTP + - 8889:8889 # Prometheus exporter (re-exposed scraped metrics) + depends_on: + - jaeger + networks: + - backend + restart: unless-stopped + + # Jaeger all-in-one: trace storage + query UI. In-memory storage (dev + # only; traces are lost on restart). Natively accepts OTLP. + jaeger: + image: jaegertracing/all-in-one:1.62.0 + environment: + COLLECTOR_OTLP_ENABLED: "true" + ports: + - 16686:16686 # Jaeger web UI + networks: + - backend + restart: unless-stopped + networks: backend: {} diff --git a/compose.yml b/compose.yml index e56dec1eb..84ef417a6 100644 --- a/compose.yml +++ b/compose.yml @@ -7,6 +7,11 @@ services: - TILED_CATALOG_URI=postgresql://tiled:${POSTGRES_PASSWORD}@postgres:5432/tiled_catalog - TILED_CATALOG_WRITABLE_STORAGE=["file:///storage", "postgresql://tiled:${POSTGRES_PASSWORD}@postgres:5432/tiled_storage"] - TILED_STREAMING_CACHE_URI=redis://:${REDIS_PASSWORD}@redis:6379 + # Export OpenTelemetry traces to the collector (see compose.monitoring.yml). + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + - OTEL_SERVICE_NAME=tiled + # Don't trace health checks and metrics scrapes (operational chatter). + - OTEL_PYTHON_FASTAPI_EXCLUDED_URLS=healthz,api/v1/metrics volumes: - tiled_data:/storage ports: diff --git a/docs/source/_toc.yml b/docs/source/_toc.yml index 98b9e5ef9..70f461128 100644 --- a/docs/source/_toc.yml +++ b/docs/source/_toc.yml @@ -42,6 +42,7 @@ subtrees: - file: user-guide/api-keys - file: user-guide/custom-clients - file: user-guide/metrics + - file: user-guide/tracing - file: user-guide/direct-client - file: user-guide/tiled-authn-database - file: user-guide/register diff --git a/docs/source/user-guide/tracing.md b/docs/source/user-guide/tracing.md new file mode 100644 index 000000000..8f0f4be79 --- /dev/null +++ b/docs/source/user-guide/tracing.md @@ -0,0 +1,78 @@ +# Distributed Tracing + +In addition to [Prometheus metrics](./metrics.md), Tiled can emit +[OpenTelemetry](https://opentelemetry.io/) traces. Whereas metrics describe +aggregate behavior across many requests, a *trace* records the timeline of a +single request as a tree of *spans*. This is useful for investigating why a +particular request was slow. + +Traces are exported using the OpenTelemetry Protocol (OTLP) to an +[OpenTelemetry Collector](https://opentelemetry.io/docs/collector/), which +forwards them to a tracing backend such as +[Jaeger](https://www.jaegertracing.io/) for storage and visualization. + +``` +tiled --OTLP--> OpenTelemetry Collector --OTLP--> Jaeger +``` + +## Enabling tracing + +Tracing is **disabled by default**. It is turned on by setting the standard +OpenTelemetry environment variable `OTEL_EXPORTER_OTLP_ENDPOINT` to the address +of an OTLP endpoint (an OpenTelemetry Collector, or a backend that accepts OTLP +directly). Related environment variables: + +| Variable | Purpose | +| --- | --- | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | OTLP endpoint, e.g. `http://otel-collector:4318`. Tracing is off when this is unset. | +| `OTEL_SERVICE_NAME` | Name shown for the service in the tracing backend, e.g. `tiled`. | +| `OTEL_PYTHON_FASTAPI_EXCLUDED_URLS` | Comma-separated URL patterns to exclude from tracing, e.g. `healthz,api/v1/metrics` to skip health checks and metrics scrapes. | + + +## How does it work? + +1. When `OTEL_EXPORTER_OTLP_ENDPOINT` is set, Tiled configures an OpenTelemetry + tracer and instruments the ASGI application, creating one span per incoming + HTTP request. + +2. Spans are exported over OTLP to the OpenTelemetry Collector. + +3. The Collector forwards traces to Jaeger, which stores them and serves the UI + used to search and visualize them. + + +## Try it with the example stack + +Tiled ships example configuration that runs an OpenTelemetry Collector and +Jaeger alongside the server, Prometheus, and Grafana. From the repository root, +start the server together with the monitoring services: + +``` +TILED_SINGLE_USER_API_KEY=secret \ + docker compose -f compose.yml -f compose.monitoring.yml up +``` + +The `compose.yml` file already sets the `OTEL_*` variables above so that the +server exports traces to the bundled Collector. + +Generate some activity using the Tiled Python client: + +```python +from tiled.client import from_uri + +c = from_uri("http://localhost:8000", api_key="secret") +c.create_container('test') +list(c) +``` + +Then open the Jaeger UI at +[http://localhost:16686](http://localhost:16686), select the **tiled** service, +and click **Find Traces**. Click any trace to see its span waterfall. + +```{note} +The bundled Collector also scrapes Tiled's `/api/v1/metrics` endpoint and +re-exposes it on port 8889, in addition to Prometheus scraping it directly. +To disable this, remove the `metrics` pipeline from the Collector +configuration in `monitoring_example/otel-collector/otel-collector.yml`. +See [Prometheus Metrics](./metrics.md). +``` diff --git a/monitoring_example/otel-collector/otel-collector.yml b/monitoring_example/otel-collector/otel-collector.yml new file mode 100644 index 000000000..2363dd725 --- /dev/null +++ b/monitoring_example/otel-collector/otel-collector.yml @@ -0,0 +1,62 @@ +# OpenTelemetry Collector configuration. +# +# Traces: tiled --OTLP--> otel-collector --OTLP--> jaeger +# Metrics: otel-collector <--scrape-- tiled:8000/api/v1/metrics +# otel-collector --expose--> :8889 (Prometheus format) +# +# The collector receives OTLP traces and forwards them to Jaeger. It also +# scrapes Tiled's Prometheus metrics endpoint and re-exposes those metrics in +# Prometheus format on port 8889. + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + + # Scrape Tiled's Prometheus metrics endpoint. This is a standard Prometheus + # scrape config. The API key must match TILED_SINGLE_USER_API_KEY (or an API + # key carrying the "metrics" scope in a multi-user deployment). + prometheus: + config: + scrape_configs: + - job_name: tiled + metrics_path: /api/v1/metrics + scrape_interval: 5s + authorization: + type: Apikey + credentials: secret + static_configs: + - targets: ['tiled:8000'] + +processors: + batch: {} + +exporters: + # Forward traces to Jaeger's OTLP gRPC receiver. + otlp/jaeger: + endpoint: jaeger:4317 + tls: + insecure: true + # Re-expose scraped metrics in Prometheus format for Prometheus to scrape. + prometheus: + endpoint: 0.0.0.0:8889 + # Log received spans to the collector's stdout (useful for debugging). + debug: + verbosity: detailed + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp/jaeger, debug] + metrics: + receivers: [prometheus] + processors: [batch] + exporters: [prometheus] + telemetry: + logs: + level: info diff --git a/monitoring_example/prometheus/prometheus.yml b/monitoring_example/prometheus/prometheus.yml index 17fc476e0..8cc092c10 100644 --- a/monitoring_example/prometheus/prometheus.yml +++ b/monitoring_example/prometheus/prometheus.yml @@ -3,6 +3,10 @@ global: scrape_interval: 5s scrape_configs: + # Scrape Tiled's metrics endpoint directly. The OpenTelemetry Collector + # independently scrapes the same endpoint and brings the metrics into its + # pipeline (see monitoring_example/otel-collector/otel-collector.yml), + # re-exposing them on otel-collector:8889 for forwarding to other backends. - job_name: 'tiled' metrics_path: /api/v1/metrics authorization: # Set Authorization header to 'Apikey secret'. diff --git a/pyproject.toml b/pyproject.toml index 2be742fbc..2e6a04ba4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,6 +87,9 @@ all = [ "numba >=0.59.0", # indirect, pinned to assist uv solve "obstore", "openpyxl", + "opentelemetry-exporter-otlp-proto-http", + "opentelemetry-instrumentation-fastapi", + "opentelemetry-sdk", "packaging", "pandas <3", "pillow", @@ -226,6 +229,9 @@ server = [ "numpy", "obstore", "openpyxl", + "opentelemetry-exporter-otlp-proto-http", + "opentelemetry-instrumentation-fastapi", + "opentelemetry-sdk", "packaging", "pandas", "pillow", diff --git a/tiled/server/app.py b/tiled/server/app.py index 9e7c2d421..5827ebf3c 100644 --- a/tiled/server/app.py +++ b/tiled/server/app.py @@ -1067,9 +1067,47 @@ async def current_principal_logging_filter( generator=lambda: secrets.token_hex(8), ) + _setup_opentelemetry_tracing(app) + return app +def _setup_opentelemetry_tracing(app: FastAPI) -> None: + """Enable OpenTelemetry request tracing when an OTLP endpoint is configured. + + Tracing is activated only when the standard ``OTEL_EXPORTER_OTLP_ENDPOINT`` + environment variable is set, so it is off by default and adds no overhead + unless explicitly enabled. Spans are exported over OTLP/HTTP. + """ + if not os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"): + return + try: + from opentelemetry import trace + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, + ) + from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + except ImportError: + logger.warning( + "OTEL_EXPORTER_OTLP_ENDPOINT is set but the OpenTelemetry packages " + "are not installed; tracing is disabled." + ) + return + + # Configure the global tracer provider once per process. + if not isinstance(trace.get_tracer_provider(), TracerProvider): + resource = Resource.create( + {"service.name": os.getenv("OTEL_SERVICE_NAME", "tiled")} + ) + provider = TracerProvider(resource=resource) + provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) + trace.set_tracer_provider(provider) + FastAPIInstrumentor.instrument_app(app) + + def build_app_from_config(config: Union[Config, dict[str, Any]], scalable=False): """ Convenience function that calls build_app(...) given config as parsed Config instance From 9761f6018e9a3574bb5fac7a51773d6153f7e4ad Mon Sep 17 00:00:00 2001 From: genematx Date: Thu, 24 Sep 2026 17:30:39 +1200 Subject: [PATCH 02/22] Add Grafana Tempo to the monitoring example Send traces to both Jaeger and Grafana Tempo: the OpenTelemetry Collector now fans traces out to a Tempo service in addition to Jaeger, and Tempo is added as a Grafana datasource so traces can be explored in Grafana with TraceQL. Bump Grafana to a version that supports TraceQL, and add the required apiVersion to the datasource provisioning files. Illustrate the telemetry flow in the tracing docs with a diagram. --- CHANGELOG.md | 5 +- compose.monitoring.yml | 23 +++++-- docs/source/user-guide/tracing.md | 69 ++++++++++++++++--- .../provisioning/datasources/prometheus.yml | 1 + .../provisioning/datasources/tempo.yml | 7 ++ .../otel-collector/otel-collector.yml | 7 +- monitoring_example/tempo/tempo.yml | 33 +++++++++ 7 files changed, 127 insertions(+), 18 deletions(-) create mode 100644 monitoring_example/grafana/provisioning/datasources/tempo.yml create mode 100644 monitoring_example/tempo/tempo.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 2879ca49c..68d030d98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,9 @@ Write the date in place of the "Unreleased" in the case a new version is release ### Added - OpenTelemetry tracing for the server, exported over OTLP. The example monitoring - stack (`compose.monitoring.yml`) now includes an OpenTelemetry Collector and - Jaeger, and the Collector also scrapes and re-exposes Tiled's Prometheus metrics. + stack (`compose.monitoring.yml`) now includes an OpenTelemetry Collector, Jaeger, + and Grafana Tempo (traces are sent to both backends), and the Collector also + scrapes and re-exposes Tiled's Prometheus metrics. - Documentation: a user-guide page on validating metadata against custom specs via server configuration. diff --git a/compose.monitoring.yml b/compose.monitoring.yml index bdf359dd4..e07334d79 100644 --- a/compose.monitoring.yml +++ b/compose.monitoring.yml @@ -14,7 +14,7 @@ services: restart: unless-stopped grafana: - image: docker.io/grafana/grafana:8.2.6 + image: docker.io/grafana/grafana:11.3.0 depends_on: - prometheus ports: @@ -34,10 +34,10 @@ services: GF_AUTH_DISABLE_LOGIN_FORM: "true" # OpenTelemetry Collector: receives OTLP telemetry from apps and fans it - # out to backends (traces -> Jaeger). It also scrapes Tiled's Prometheus - # metrics endpoint and re-exposes it for Prometheus. Apps on the 'backend' - # network export to http://otel-collector:4318; apps on the host export to - # http://localhost:4318. + # out to backends (traces -> Jaeger and Tempo). It also scrapes Tiled's + # Prometheus metrics endpoint and re-exposes it for Prometheus. Apps on the + # 'backend' network export to http://otel-collector:4318; apps on the host + # export to http://localhost:4318. otel-collector: image: otel/opentelemetry-collector-contrib:0.130.0 command: ["--config=/etc/otelcol/config.yaml"] @@ -49,6 +49,7 @@ services: - 8889:8889 # Prometheus exporter (re-exposed scraped metrics) depends_on: - jaeger + - tempo networks: - backend restart: unless-stopped @@ -65,5 +66,17 @@ services: - backend restart: unless-stopped + # Grafana Tempo: trace storage queried from Grafana (no UI of its own). + # Stores traces on local ephemeral storage (see monitoring_example/tempo). + # Natively accepts OTLP. Explore traces in Grafana via the Tempo datasource. + tempo: + image: docker.io/grafana/tempo:2.6.1 + command: ["-config.file=/etc/tempo/tempo.yml"] + volumes: + - ./monitoring_example/tempo/tempo.yml:/etc/tempo/tempo.yml + networks: + - backend + restart: unless-stopped + networks: backend: {} diff --git a/docs/source/user-guide/tracing.md b/docs/source/user-guide/tracing.md index 8f0f4be79..8eb9aa6e7 100644 --- a/docs/source/user-guide/tracing.md +++ b/docs/source/user-guide/tracing.md @@ -8,13 +8,55 @@ particular request was slow. Traces are exported using the OpenTelemetry Protocol (OTLP) to an [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/), which -forwards them to a tracing backend such as -[Jaeger](https://www.jaegertracing.io/) for storage and visualization. - -``` -tiled --OTLP--> OpenTelemetry Collector --OTLP--> Jaeger +forwards them to one or more tracing backends for storage and visualization, +such as [Jaeger](https://www.jaegertracing.io/) or +[Grafana Tempo](https://grafana.com/oss/tempo/). + +```{mermaid} +flowchart LR + tiled["Tiled server"] + collector["OpenTelemetry
Collector"] + + subgraph backends["Storage backends"] + direction TB + jaeger["Jaeger"] + tempo["Grafana Tempo"] + prometheus["Prometheus"] + loki["Loki"] + end + + subgraph viz["Visualization"] + direction TB + jaegerui["Jaeger UI"] + grafana["Grafana"] + end + + %% Configured in the example + tiled -->|"traces (OTLP)"| collector + collector -->|OTLP| jaeger + collector -->|OTLP| tempo + tiled -->|"metrics (scrape)"| prometheus + + %% Visualization + jaeger --> jaegerui + tempo --> grafana + prometheus --> grafana + + %% Metrics and logs over OTLP: possible extension, not enabled + tiled -.->|"metrics (OTLP)"| collector + tiled -.->|"logs (OTLP)"| collector + collector -.->|metrics| prometheus + collector -.->|logs| loki + loki -.-> grafana ``` +Solid arrows are what the example configures today: Tiled pushes **traces** over +OTLP to the Collector, which fans them out to Jaeger and Grafana Tempo, while +Prometheus scrapes Tiled's metrics endpoint. Dashed arrows show how the same +Collector could also carry OpenTelemetry's other two signals — **metrics** and +**logs** — over OTLP to backends such as Prometheus and Loki. Those paths are +not currently enabled. + ## Enabling tracing Tracing is **disabled by default**. It is turned on by setting the standard @@ -37,8 +79,9 @@ directly). Related environment variables: 2. Spans are exported over OTLP to the OpenTelemetry Collector. -3. The Collector forwards traces to Jaeger, which stores them and serves the UI - used to search and visualize them. +3. The Collector forwards traces to one or more backends (Jaeger and Grafana + Tempo in the example stack), which store them and make them available to + search and visualize. ## Try it with the example stack @@ -65,9 +108,15 @@ c.create_container('test') list(c) ``` -Then open the Jaeger UI at -[http://localhost:16686](http://localhost:16686), select the **tiled** service, -and click **Find Traces**. Click any trace to see its span waterfall. +The example forwards traces to two backends so you can compare their functionality: + +- **Jaeger:** open [http://localhost:16686](http://localhost:16686), select the + **tiled** service, and click **Find Traces**. Click a trace to see its span + waterfall. +- **Grafana Tempo:** open [http://localhost:3000](http://localhost:3000), go to + **Explore**, select the **Tempo** data source, and search using + [TraceQL](https://grafana.com/docs/tempo/latest/traceql/), for example + `{ resource.service.name = "tiled" }`. ```{note} The bundled Collector also scrapes Tiled's `/api/v1/metrics` endpoint and diff --git a/monitoring_example/grafana/provisioning/datasources/prometheus.yml b/monitoring_example/grafana/provisioning/datasources/prometheus.yml index 0cd616ec1..6509b42b5 100644 --- a/monitoring_example/grafana/provisioning/datasources/prometheus.yml +++ b/monitoring_example/grafana/provisioning/datasources/prometheus.yml @@ -1,3 +1,4 @@ +apiVersion: 1 datasources: - name: Prometheus access: proxy diff --git a/monitoring_example/grafana/provisioning/datasources/tempo.yml b/monitoring_example/grafana/provisioning/datasources/tempo.yml new file mode 100644 index 000000000..65c2d04b0 --- /dev/null +++ b/monitoring_example/grafana/provisioning/datasources/tempo.yml @@ -0,0 +1,7 @@ +apiVersion: 1 +datasources: +- name: Tempo + access: proxy + type: tempo + url: http://tempo:3200 + uid: tempo diff --git a/monitoring_example/otel-collector/otel-collector.yml b/monitoring_example/otel-collector/otel-collector.yml index 2363dd725..cc78bd99c 100644 --- a/monitoring_example/otel-collector/otel-collector.yml +++ b/monitoring_example/otel-collector/otel-collector.yml @@ -40,6 +40,11 @@ exporters: endpoint: jaeger:4317 tls: insecure: true + # Forward the same traces to Grafana Tempo's OTLP gRPC receiver. + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true # Re-expose scraped metrics in Prometheus format for Prometheus to scrape. prometheus: endpoint: 0.0.0.0:8889 @@ -52,7 +57,7 @@ service: traces: receivers: [otlp] processors: [batch] - exporters: [otlp/jaeger, debug] + exporters: [otlp/jaeger, otlp/tempo, debug] metrics: receivers: [prometheus] processors: [batch] diff --git a/monitoring_example/tempo/tempo.yml b/monitoring_example/tempo/tempo.yml new file mode 100644 index 000000000..c887d1f3c --- /dev/null +++ b/monitoring_example/tempo/tempo.yml @@ -0,0 +1,33 @@ +# Grafana Tempo configuration (single binary, for the example stack). +# +# Tempo receives traces over OTLP from the OpenTelemetry Collector and stores +# them on the local filesystem. This example writes to /tmp (ephemeral, lost on +# restart) so it needs no volume or permission setup. Traces are queried from +# Grafana via the Tempo datasource. + +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +ingester: + max_block_duration: 5m + +compactor: + compaction: + block_retention: 1h + +storage: + trace: + backend: local + local: + path: /tmp/tempo/blocks + wal: + path: /tmp/tempo/wal From c87e01f8a9659928be4c13b6a1b27fdb402dc282 Mon Sep 17 00:00:00 2001 From: genematx Date: Thu, 24 Sep 2026 17:47:43 +1200 Subject: [PATCH 03/22] Emit OpenTelemetry spans for internal request phases Wrap record_timing in an OpenTelemetry span so the phases it already times (access control, read, tokenize, pack) appear as child spans in a request's trace, giving a per-request breakdown of where time is spent. The span is a no-op when OpenTelemetry is not installed or no tracer provider is configured. --- CHANGELOG.md | 3 +++ tiled/server/utils.py | 30 ++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68d030d98..07f74cbbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ Write the date in place of the "Unreleased" in the case a new version is release stack (`compose.monitoring.yml`) now includes an OpenTelemetry Collector, Jaeger, and Grafana Tempo (traces are sent to both backends), and the Collector also scrapes and re-exposes Tiled's Prometheus metrics. +- Emit OpenTelemetry spans for internal request phases (access control, read, + tokenize, pack) so they appear as child spans in a request's trace, giving a + per-request breakdown of where time is spent. - Documentation: a user-guide page on validating metadata against custom specs via server configuration. diff --git a/tiled/server/utils.py b/tiled/server/utils.py index 564c44a3e..f6a35629e 100644 --- a/tiled/server/utils.py +++ b/tiled/server/utils.py @@ -17,14 +17,40 @@ API_KEY_QUERY_PARAMETER = "api_key" CSRF_COOKIE_NAME = "tiled_csrf" +try: + from opentelemetry import trace + + _tracer = trace.get_tracer("tiled.server") +except ImportError: + # OpenTelemetry is an optional dependency; tracing is simply disabled. + _tracer = None + +# Human-readable OpenTelemetry span names for the phases timed below. +_SPAN_NAMES = { + "app": "tiled.app", + "acl": "tiled.access_control", + "read": "tiled.read", + "tok": "tiled.tokenize", + "pack": "tiled.pack", +} + @contextlib.contextmanager def record_timing(metrics: dict[str, Any], key: str) -> Generator[None]: """ - Set timings[key] equal to the run time (in milliseconds) of the context body. + Set timings[key] equal to the run time (in seconds) of the context body. + + Also open an OpenTelemetry span around the body so that these phases appear + as child spans in a request's trace. If OpenTelemetry is not installed or no + tracer provider is configured, the span is a no-op. """ + if _tracer is None: + span = contextlib.nullcontext() + else: + span = _tracer.start_as_current_span(_SPAN_NAMES.get(key, f"tiled.{key}")) t0 = time.perf_counter() - yield + with span: + yield metrics[key]["dur"] += time.perf_counter() - t0 # Units: seconds From ead3c81aeb39ee8b92ccb369fdd1fa5ce52cb058 Mon Sep 17 00:00:00 2001 From: genematx Date: Tue, 29 Sep 2026 10:21:47 +1300 Subject: [PATCH 04/22] Only emit request-phase spans within a traced request record_timing opened a phase span unconditionally, so requests that are excluded from tracing (health checks, metrics scrapes) produced orphaned single-span traces that cluttered the trace UI. Only open a phase span when there is an active recording span. --- tiled/server/utils.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tiled/server/utils.py b/tiled/server/utils.py index f6a35629e..12d4bd06a 100644 --- a/tiled/server/utils.py +++ b/tiled/server/utils.py @@ -40,14 +40,16 @@ def record_timing(metrics: dict[str, Any], key: str) -> Generator[None]: """ Set timings[key] equal to the run time (in seconds) of the context body. - Also open an OpenTelemetry span around the body so that these phases appear - as child spans in a request's trace. If OpenTelemetry is not installed or no - tracer provider is configured, the span is a no-op. + When there is an active recording trace span (i.e. this request is being + traced), also open a child OpenTelemetry span around the body so these + phases appear in the request's trace. Outside a traced request (tracing + disabled, or an excluded endpoint such as health checks and metrics + scrapes) no span is created, avoiding orphaned single-span traces. """ - if _tracer is None: - span = contextlib.nullcontext() - else: + if _tracer is not None and trace.get_current_span().is_recording(): span = _tracer.start_as_current_span(_SPAN_NAMES.get(key, f"tiled.{key}")) + else: + span = contextlib.nullcontext() t0 = time.perf_counter() with span: yield From 11407fbf3b65c6e3ef6cc85c760e5cd132efe603 Mon Sep 17 00:00:00 2001 From: genematx Date: Wed, 30 Sep 2026 22:31:47 +1300 Subject: [PATCH 05/22] Opt out of FastAPI's built-in OpenTelemetry auto-configuration FastAPI >=0.142 ships built-in OpenTelemetry support that, when OTEL_EXPORTER_OTLP_ENDPOINT is set, registers its own OTLP export pipeline on the global tracer provider. Combined with the tracing pipeline Tiled configures, this exported every span twice. Pass telemetry={"auto_configure": False} to FastAPI() so Tiled remains the sole exporter. --- CHANGELOG.md | 4 ++++ tiled/server/app.py | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07f74cbbe..3356e9a54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ Write the date in place of the "Unreleased" in the case a new version is release - Emit OpenTelemetry spans for internal request phases (access control, read, tokenize, pack) so they appear as child spans in a request's trace, giving a per-request breakdown of where time is spent. +- Disable FastAPI's built-in OpenTelemetry auto-configuration + (`telemetry={"auto_configure": False}`) so that, on FastAPI >=0.142, it does + not register a second OTLP exporter alongside Tiled's own tracing pipeline and + export every span twice. - Documentation: a user-guide page on validating metadata against custom specs via server configuration. diff --git a/tiled/server/app.py b/tiled/server/app.py index 5827ebf3c..45ffcaf90 100644 --- a/tiled/server/app.py +++ b/tiled/server/app.py @@ -286,7 +286,21 @@ async def lifespan(app: FastAPI): finally: await shutdown_event() - app = FastAPI(lifespan=lifespan, strict_content_type=False) + try: + # FastAPI >=0.142 ships built-in OpenTelemetry support that, when + # `OTEL_EXPORTER_OTLP_ENDPOINT` (or a related variable) is set, + # auto-registers its own OTLP export pipeline on the global tracer + # provider. Tiled configures and manages its own tracing pipeline (see + # `_setup_opentelemetry_tracing`), so FastAPI's auto-configuration + # would register a second exporter and emit every span twice. Opt out. + app = FastAPI( + lifespan=lifespan, + strict_content_type=False, + telemetry={"auto_configure": False}, + ) + except TypeError: + # FastAPI <0.142 has no built-in telemetry and no `telemetry` option. + app = FastAPI(lifespan=lifespan, strict_content_type=False) # Healthcheck for deployment to containerized systems, needs to preempt other responses. # Standardized for Kubernetes, but also used by other systems. From d9574d935a553517c40d8781ee6c915bcfd5c65c Mon Sep 17 00:00:00 2001 From: genematx Date: Wed, 30 Sep 2026 22:55:08 +1300 Subject: [PATCH 06/22] Add in-process tests for OpenTelemetry request tracing Use an in-memory span exporter to verify, without a running collector or backend: a traced request emits the FastAPI server span and child phase spans (single trace, no duplicate span IDs); excluded endpoints emit no spans; tracing stays off when OTEL_EXPORTER_OTLP_ENDPOINT is unset; and FastAPI's built-in telemetry does not register a second export pipeline. --- tests/test_tracing.py | 169 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 tests/test_tracing.py diff --git a/tests/test_tracing.py b/tests/test_tracing.py new file mode 100644 index 000000000..e7c98b9d3 --- /dev/null +++ b/tests/test_tracing.py @@ -0,0 +1,169 @@ +"""In-process tests for the OpenTelemetry request tracing configured by +``tiled.server.app._setup_opentelemetry_tracing``. + +These use an in-memory span exporter, so they need no running OpenTelemetry +Collector, Jaeger, or Tempo. OpenTelemetry's global tracer provider can only be +set once per process, so a single provider is installed for the whole module and +the exporter is cleared between tests. +""" +import os + +import pytest + +pytest.importorskip("opentelemetry.sdk") + +# The FastAPI instrumentation reads OTEL_PYTHON_FASTAPI_EXCLUDED_URLS once, at +# import time, into a module-level default that ``instrument_app`` uses. Set it +# before that module is first imported (which the tracing hook does lazily on +# the first traced ``build_app``). In deployments this variable is likewise set +# in the environment before the process starts. +os.environ["OTEL_PYTHON_FASTAPI_EXCLUDED_URLS"] = "healthz,api/v1/metrics" + +from opentelemetry import trace # noqa: E402 +from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402 + InMemorySpanExporter, +) +from opentelemetry.trace import SpanKind # noqa: E402 + +from tiled.client import Context, from_context # noqa: E402 +from tiled.server.app import build_app_from_config # noqa: E402 + +CONFIG = { + "authentication": {"single_user_api_key": "secret"}, + "trees": [{"path": "/", "tree": "tiled.examples.generated_minimal:tree"}], +} +# A syntactically valid endpoint that is never actually contacted: the tracing +# hook reuses the in-memory provider installed below instead of creating an OTLP +# exporter, so no network traffic occurs. +ENDPOINT = "http://otel-collector.invalid:4318" + +# Global library instrumentation installed by the tracing hook, which must be +# undone so it does not leak into other test modules. +_GLOBAL_INSTRUMENTORS = [ + ("opentelemetry.instrumentation.asyncpg", "AsyncPGInstrumentor"), + ("opentelemetry.instrumentation.redis", "RedisInstrumentor"), + ("opentelemetry.instrumentation.httpx", "HTTPXClientInstrumentor"), +] + + +@pytest.fixture(scope="module") +def span_exporter(): + exporter = InMemorySpanExporter() + current = trace.get_tracer_provider() + if isinstance(current, TracerProvider): + # Another module already installed a real provider; attach to it. + provider = current + else: + provider = TracerProvider() + trace.set_tracer_provider(provider) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + yield exporter + for module, cls in _GLOBAL_INSTRUMENTORS: + try: + mod = __import__(module, fromlist=[cls]) + getattr(mod, cls)().uninstrument() + except Exception: + pass + + +@pytest.fixture(autouse=True) +def _clear_spans(span_exporter): + span_exporter.clear() + yield + span_exporter.clear() + + +def _build_app(monkeypatch, *, endpoint=ENDPOINT): + if endpoint is None: + monkeypatch.delenv("OTEL_EXPORTER_OTLP_ENDPOINT", raising=False) + else: + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint) + return build_app_from_config(CONFIG) + + +def _is_descendant_of(span, ancestor_span_id, by_id): + seen = set() + cur = span + while cur is not None and cur.parent is not None: + parent_id = cur.parent.span_id + if parent_id == ancestor_span_id: + return True + if parent_id in seen: + break + seen.add(parent_id) + cur = by_id.get(parent_id) + return False + + +def test_traced_request_emits_server_and_phase_spans(monkeypatch, span_exporter): + app = _build_app(monkeypatch) + with Context.from_app(app) as context: + # Discard spans emitted while the Context was being set up, so we measure + # exactly one request. + span_exporter.clear() + response = context.http_client.get("/api/v1/metadata/") + assert response.status_code == 200 + spans = span_exporter.get_finished_spans() + + assert spans, "expected spans to be exported for a traced request" + ids = [s.context.span_id for s in spans] + assert len(ids) == len(set(ids)), "spans must not be duplicated" + assert len({s.context.trace_id for s in spans}) == 1, "one request => one trace" + + roots = [s for s in spans if s.parent is None] + assert len(roots) == 1, "expected exactly one root span" + assert roots[0].kind == SpanKind.SERVER, "root should be the FastAPI server span" + + by_id = {s.context.span_id: s for s in spans} + names = {s.name for s in spans} + assert "tiled.app" in names, "expected the per-request phase span 'tiled.app'" + app_span = next(s for s in spans if s.name == "tiled.app") + assert _is_descendant_of(app_span, roots[0].context.span_id, by_id), ( + "phase spans should be children of the request's server span" + ) + + +def test_excluded_endpoint_emits_no_spans(monkeypatch, span_exporter): + app = _build_app(monkeypatch) + with Context.from_app(app) as context: + span_exporter.clear() + response = context.http_client.get("/healthz") + assert response.status_code == 200 + assert not span_exporter.get_finished_spans(), ( + "excluded endpoints must not produce spans (no orphan traces)" + ) + + +def test_tracing_disabled_by_default_emits_no_spans(monkeypatch, span_exporter): + app = _build_app(monkeypatch, endpoint=None) + with Context.from_app(app) as context: + span_exporter.clear() + response = context.http_client.get("/api/v1/metadata/") + assert response.status_code == 200 + assert not span_exporter.get_finished_spans(), ( + "without OTEL_EXPORTER_OTLP_ENDPOINT the app should not be traced" + ) + + +def test_no_duplicate_export_pipeline(monkeypatch, span_exporter): + """Guard against FastAPI's built-in telemetry (>=0.142) registering a second + OTLP export pipeline, which would export every span twice.""" + from starlette.testclient import TestClient + + provider = trace.get_tracer_provider() + processors = provider._active_span_processor._span_processors + before = len(processors) + + app = _build_app(monkeypatch) + # Entering the TestClient runs the ASGI lifespan; FastAPI configures its + # built-in telemetry on ``lifespan.startup``. + with TestClient(app): + pass + + after = len(provider._active_span_processor._span_processors) + assert after == before, ( + "a second span processor was registered on the global provider; " + "FastAPI's built-in OpenTelemetry auto-configuration is not disabled" + ) From caca48fa107d3731a917d602df6c833953387bdf Mon Sep 17 00:00:00 2001 From: genematx Date: Thu, 1 Oct 2026 06:30:18 +1300 Subject: [PATCH 07/22] MNT: lint tests --- tests/test_tracing.py | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/tests/test_tracing.py b/tests/test_tracing.py index e7c98b9d3..d1543d159 100644 --- a/tests/test_tracing.py +++ b/tests/test_tracing.py @@ -1,5 +1,5 @@ """In-process tests for the OpenTelemetry request tracing configured by -``tiled.server.app._setup_opentelemetry_tracing``. +`tiled.server.app._setup_opentelemetry_tracing`. These use an in-memory span exporter, so they need no running OpenTelemetry Collector, Jaeger, or Tempo. OpenTelemetry's global tracer provider can only be @@ -9,27 +9,24 @@ import os import pytest +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import SpanKind + +from tiled.client import Context +from tiled.server.app import build_app_from_config pytest.importorskip("opentelemetry.sdk") # The FastAPI instrumentation reads OTEL_PYTHON_FASTAPI_EXCLUDED_URLS once, at -# import time, into a module-level default that ``instrument_app`` uses. Set it +# import time, into a module-level default that `instrument_app` uses. Set it # before that module is first imported (which the tracing hook does lazily on -# the first traced ``build_app``). In deployments this variable is likewise set +# the first traced `build_app`). In deployments this variable is likewise set # in the environment before the process starts. os.environ["OTEL_PYTHON_FASTAPI_EXCLUDED_URLS"] = "healthz,api/v1/metrics" -from opentelemetry import trace # noqa: E402 -from opentelemetry.sdk.trace import TracerProvider # noqa: E402 -from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 -from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402 - InMemorySpanExporter, -) -from opentelemetry.trace import SpanKind # noqa: E402 - -from tiled.client import Context, from_context # noqa: E402 -from tiled.server.app import build_app_from_config # noqa: E402 - CONFIG = { "authentication": {"single_user_api_key": "secret"}, "trees": [{"path": "/", "tree": "tiled.examples.generated_minimal:tree"}], @@ -120,9 +117,9 @@ def test_traced_request_emits_server_and_phase_spans(monkeypatch, span_exporter) names = {s.name for s in spans} assert "tiled.app" in names, "expected the per-request phase span 'tiled.app'" app_span = next(s for s in spans if s.name == "tiled.app") - assert _is_descendant_of(app_span, roots[0].context.span_id, by_id), ( - "phase spans should be children of the request's server span" - ) + assert _is_descendant_of( + app_span, roots[0].context.span_id, by_id + ), "phase spans should be children of the request's server span" def test_excluded_endpoint_emits_no_spans(monkeypatch, span_exporter): @@ -131,9 +128,9 @@ def test_excluded_endpoint_emits_no_spans(monkeypatch, span_exporter): span_exporter.clear() response = context.http_client.get("/healthz") assert response.status_code == 200 - assert not span_exporter.get_finished_spans(), ( - "excluded endpoints must not produce spans (no orphan traces)" - ) + assert ( + not span_exporter.get_finished_spans() + ), "excluded endpoints must not produce spans (no orphan traces)" def test_tracing_disabled_by_default_emits_no_spans(monkeypatch, span_exporter): @@ -142,9 +139,9 @@ def test_tracing_disabled_by_default_emits_no_spans(monkeypatch, span_exporter): span_exporter.clear() response = context.http_client.get("/api/v1/metadata/") assert response.status_code == 200 - assert not span_exporter.get_finished_spans(), ( - "without OTEL_EXPORTER_OTLP_ENDPOINT the app should not be traced" - ) + assert ( + not span_exporter.get_finished_spans() + ), "without OTEL_EXPORTER_OTLP_ENDPOINT the app should not be traced" def test_no_duplicate_export_pipeline(monkeypatch, span_exporter): From 07222a06cfa932d973ef4aee25c78e93c5af5ce1 Mon Sep 17 00:00:00 2001 From: genematx Date: Thu, 1 Oct 2026 06:46:29 +1300 Subject: [PATCH 08/22] Make disabled-tracing test robust to FastAPI built-in telemetry FastAPI >=0.142 ships its own OpenTelemetry integration that emits request spans on any globally installed tracer provider when the app is not instrumented by opentelemetry-instrumentation-fastapi. The in-memory provider the tests install made test_tracing_disabled_by_default capture those spans and fail on CI. Assert instead that the tracing hook did not instrument the app (its documented off-by-default behavior). Also use single backticks in comments/docstrings. --- tests/test_tracing.py | 17 ++++++++--------- tiled/server/app.py | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tests/test_tracing.py b/tests/test_tracing.py index d1543d159..45e16cf02 100644 --- a/tests/test_tracing.py +++ b/tests/test_tracing.py @@ -133,15 +133,14 @@ def test_excluded_endpoint_emits_no_spans(monkeypatch, span_exporter): ), "excluded endpoints must not produce spans (no orphan traces)" -def test_tracing_disabled_by_default_emits_no_spans(monkeypatch, span_exporter): +def test_tracing_disabled_by_default(monkeypatch): + # Without OTEL_EXPORTER_OTLP_ENDPOINT the tracing hook returns early and does + # not instrument the app, so tracing is off and adds no overhead. (Asserting + # on emitted spans is not reliable here: FastAPI >=0.142 ships its own + # telemetry that emits request spans on any globally installed provider when + # the app is not instrumented by OpenTelemetry.) app = _build_app(monkeypatch, endpoint=None) - with Context.from_app(app) as context: - span_exporter.clear() - response = context.http_client.get("/api/v1/metadata/") - assert response.status_code == 200 - assert ( - not span_exporter.get_finished_spans() - ), "without OTEL_EXPORTER_OTLP_ENDPOINT the app should not be traced" + assert not getattr(app, "_is_instrumented_by_opentelemetry", False) def test_no_duplicate_export_pipeline(monkeypatch, span_exporter): @@ -155,7 +154,7 @@ def test_no_duplicate_export_pipeline(monkeypatch, span_exporter): app = _build_app(monkeypatch) # Entering the TestClient runs the ASGI lifespan; FastAPI configures its - # built-in telemetry on ``lifespan.startup``. + # built-in telemetry on `lifespan.startup`. with TestClient(app): pass diff --git a/tiled/server/app.py b/tiled/server/app.py index 45ffcaf90..496155c8a 100644 --- a/tiled/server/app.py +++ b/tiled/server/app.py @@ -1089,7 +1089,7 @@ async def current_principal_logging_filter( def _setup_opentelemetry_tracing(app: FastAPI) -> None: """Enable OpenTelemetry request tracing when an OTLP endpoint is configured. - Tracing is activated only when the standard ``OTEL_EXPORTER_OTLP_ENDPOINT`` + Tracing is activated only when the standard `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable is set, so it is off by default and adds no overhead unless explicitly enabled. Spans are exported over OTLP/HTTP. """ From 159e49f044f701422106fe6c5587da0ae65c56ac Mon Sep 17 00:00:00 2001 From: genematx Date: Thu, 1 Oct 2026 10:12:57 +1300 Subject: [PATCH 09/22] Make tracing opt-in via the monitoring compose overlay The base compose.yml and compose.dev.yml set OTEL_EXPORTER_OTLP_ENDPOINT pointing at otel-collector, which is only defined in compose.monitoring.yml. Running the base files on their own therefore enabled tracing against an unreachable host, causing continuous export failures. Move the OTEL_* variables into compose.monitoring.yml, next to the collector they target, so tracing is off unless that overlay is used. Also update the tracing user guide to launch the example with compose.dev.yml (which builds the image from this checkout) instead of compose.yml (whose pinned published image may not include tracing yet). --- compose.dev.yml | 5 ----- compose.monitoring.yml | 12 ++++++++++++ compose.yml | 5 ----- docs/source/user-guide/tracing.md | 8 +++++--- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/compose.dev.yml b/compose.dev.yml index 7ed91fcfa..c35e3cb53 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -14,11 +14,6 @@ services: - TILED_WEBHOOKS_ALLOW_DELIVERY_HOSTS=${TILED_WEBHOOKS_ALLOW_DELIVERY_HOSTS:-} - TILED_WEBHOOKS_ALLOW_HTTP=${TILED_WEBHOOKS_ALLOW_HTTP:-false} - TILED_WEBHOOKS_ALLOW_PRIVATE_ADDRESSES=${TILED_WEBHOOKS_ALLOW_PRIVATE_ADDRESSES:-false} - # Export OpenTelemetry traces to the collector (see compose.monitoring.yml). - - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 - - OTEL_SERVICE_NAME=tiled - # Don't trace health checks and metrics scrapes (operational chatter). - - OTEL_PYTHON_FASTAPI_EXCLUDED_URLS=healthz,api/v1/metrics volumes: - tiled_data:/storage ports: diff --git a/compose.monitoring.yml b/compose.monitoring.yml index e07334d79..b31a140b0 100644 --- a/compose.monitoring.yml +++ b/compose.monitoring.yml @@ -1,5 +1,17 @@ --- services: + # Turn on OpenTelemetry tracing for the Tiled server (defined in compose.yml + # or compose.dev.yml) and point it at the collector below. These variables + # live here, alongside the collector, so that running the base compose files + # on their own leaves tracing off instead of exporting to a collector that is + # not running. + tiled: + environment: + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + - OTEL_SERVICE_NAME=tiled + # Don't trace health checks and metrics scrapes (operational chatter). + - OTEL_PYTHON_FASTAPI_EXCLUDED_URLS=healthz,api/v1/metrics + prometheus: image: docker.io/prom/prometheus:v2.42.0 volumes: diff --git a/compose.yml b/compose.yml index 84ef417a6..e56dec1eb 100644 --- a/compose.yml +++ b/compose.yml @@ -7,11 +7,6 @@ services: - TILED_CATALOG_URI=postgresql://tiled:${POSTGRES_PASSWORD}@postgres:5432/tiled_catalog - TILED_CATALOG_WRITABLE_STORAGE=["file:///storage", "postgresql://tiled:${POSTGRES_PASSWORD}@postgres:5432/tiled_storage"] - TILED_STREAMING_CACHE_URI=redis://:${REDIS_PASSWORD}@redis:6379 - # Export OpenTelemetry traces to the collector (see compose.monitoring.yml). - - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 - - OTEL_SERVICE_NAME=tiled - # Don't trace health checks and metrics scrapes (operational chatter). - - OTEL_PYTHON_FASTAPI_EXCLUDED_URLS=healthz,api/v1/metrics volumes: - tiled_data:/storage ports: diff --git a/docs/source/user-guide/tracing.md b/docs/source/user-guide/tracing.md index 8eb9aa6e7..6d1cf3bb1 100644 --- a/docs/source/user-guide/tracing.md +++ b/docs/source/user-guide/tracing.md @@ -92,11 +92,13 @@ start the server together with the monitoring services: ``` TILED_SINGLE_USER_API_KEY=secret \ - docker compose -f compose.yml -f compose.monitoring.yml up + docker compose -f compose.dev.yml -f compose.monitoring.yml up --build ``` -The `compose.yml` file already sets the `OTEL_*` variables above so that the -server exports traces to the bundled Collector. +`compose.dev.yml` builds the Tiled image from this checkout (so it includes the +tracing support), and `compose.monitoring.yml` sets the `OTEL_*` variables above +and runs the Collector, so the server exports traces to it. (The published image +referenced by `compose.yml` may not yet include tracing.) Generate some activity using the Tiled Python client: From def2bdfdffc96b434b4f386e7d761163030c56ab Mon Sep 17 00:00:00 2001 From: genematx Date: Wed, 30 Sep 2026 17:11:16 +1300 Subject: [PATCH 10/22] ENH: instrument asyncpg and redis globally --- pyproject.toml | 4 ++++ tiled/server/app.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 2e6a04ba4..7a217cee8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,7 +88,9 @@ all = [ "obstore", "openpyxl", "opentelemetry-exporter-otlp-proto-http", + "opentelemetry-instrumentation-asyncpg", "opentelemetry-instrumentation-fastapi", + "opentelemetry-instrumentation-redis", "opentelemetry-sdk", "packaging", "pandas <3", @@ -230,7 +232,9 @@ server = [ "obstore", "openpyxl", "opentelemetry-exporter-otlp-proto-http", + "opentelemetry-instrumentation-asyncpg", "opentelemetry-instrumentation-fastapi", + "opentelemetry-instrumentation-redis", "opentelemetry-sdk", "packaging", "pandas", diff --git a/tiled/server/app.py b/tiled/server/app.py index 496155c8a..ed146b59b 100644 --- a/tiled/server/app.py +++ b/tiled/server/app.py @@ -1121,6 +1121,21 @@ def _setup_opentelemetry_tracing(app: FastAPI) -> None: trace.set_tracer_provider(provider) FastAPIInstrumentor.instrument_app(app) + # Emit spans for calls to the PostgreSQL driver (asyncpg) and Redis. These + # patch the libraries globally, so they are no-ops until a request uses them. + try: + from opentelemetry.instrumentation.asyncpg import AsyncPGInstrumentor + except ImportError: + pass + else: + AsyncPGInstrumentor().instrument() + try: + from opentelemetry.instrumentation.redis import RedisInstrumentor + except ImportError: + pass + else: + RedisInstrumentor().instrument() + def build_app_from_config(config: Union[Config, dict[str, Any]], scalable=False): """ From bddf403ac83cc5b0b15ba6b5e98d5293883d84ab Mon Sep 17 00:00:00 2001 From: genematx Date: Fri, 2 Oct 2026 07:39:30 +1300 Subject: [PATCH 11/22] Instrument the ADBC storage database for tracing The storage database is accessed via ADBC, which the asyncpg instrumentation does not cover, so its queries would otherwise be invisible in traces. Wrap the ADBC connection factory with opentelemetry-instrumentation-dbapi (gated on OTEL_EXPORTER_OTLP_ENDPOINT) to emit a span per query, and emit an explicit span around the bulk adbc_ingest write, which bypasses the DBAPI execute path. --- pyproject.toml | 2 ++ tiled/adapters/sql.py | 40 ++++++++++++++++++++++++++++++++++++++-- tiled/storage.py | 30 +++++++++++++++++++++++++++++- 3 files changed, 69 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7a217cee8..e10962e3d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,7 @@ all = [ "openpyxl", "opentelemetry-exporter-otlp-proto-http", "opentelemetry-instrumentation-asyncpg", + "opentelemetry-instrumentation-dbapi", "opentelemetry-instrumentation-fastapi", "opentelemetry-instrumentation-redis", "opentelemetry-sdk", @@ -233,6 +234,7 @@ server = [ "openpyxl", "opentelemetry-exporter-otlp-proto-http", "opentelemetry-instrumentation-asyncpg", + "opentelemetry-instrumentation-dbapi", "opentelemetry-instrumentation-fastapi", "opentelemetry-instrumentation-redis", "opentelemetry-sdk", diff --git a/tiled/adapters/sql.py b/tiled/adapters/sql.py index 1da9f0684..1db28f744 100644 --- a/tiled/adapters/sql.py +++ b/tiled/adapters/sql.py @@ -3,9 +3,10 @@ import copy import hashlib import logging +import os import re from collections.abc import Set -from contextlib import closing +from contextlib import AbstractContextManager, closing, nullcontext from typing import ( TYPE_CHECKING, Any, @@ -20,6 +21,7 @@ Union, cast, ) +from urllib.parse import urlparse from tiled.utils import UnsafeIdentifier @@ -49,6 +51,13 @@ from ..type_aliases import JSON from .array import ArrayAdapter +try: + from opentelemetry import trace as _otel_trace + + _STORAGE_TRACER = _otel_trace.get_tracer("tiled.storage") +except ImportError: # OpenTelemetry is an optional dependency. + _STORAGE_TRACER = None + DIALECTS = Literal["postgresql", "sqlite", "duckdb"] TABLE_NAME_PATTERN = re.compile(r"^[a-z][a-z0-9_]*$") COLUMN_NAME_PATTERN = re.compile(r"^[a-zA-Z_].*$") @@ -503,9 +512,36 @@ def append_partition( with closing(self.storage.connect()) as conn: with conn.cursor() as cursor: - cursor.adbc_ingest(self.table_name, table, mode="append") + with self._adbc_ingest_span(): + cursor.adbc_ingest(self.table_name, table, mode="append") conn.commit() + def _adbc_ingest_span(self) -> AbstractContextManager[Any]: + """OpenTelemetry tracing span for the bulk `adbc_ingest` write. + + `adbc_ingest` bypasses the DBAPI `execute()` path, so the generic + dbapi instrumentation does not trace it; emit a span explicitly with the + same db attributes as the other storage spans. A no-op if OpenTelemetry + is not installed or tracing is not configured. + """ + if _STORAGE_TRACER is None or not os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"): + return nullcontext() + attributes = { + "db.system": self.storage.dialect, + "db.sql.table": self.table_name, + } + db_name = urlparse(self.storage.uri).path.lstrip("/") + if db_name: + attributes["db.name"] = db_name + return cast( + AbstractContextManager[Any], + _STORAGE_TRACER.start_as_current_span( + "adbc_ingest", + kind=_otel_trace.SpanKind.CLIENT, + attributes=attributes, + ), + ) + def _read_full_table_or_partition( self, fields: Optional[List[str]] = None, partition: Optional[int] = None ) -> pyarrow.Table: diff --git a/tiled/storage.py b/tiled/storage.py index c3ede0163..d45ecd957 100644 --- a/tiled/storage.py +++ b/tiled/storage.py @@ -247,7 +247,7 @@ def _adbc_connection(self) -> "adbc_driver_manager.dbapi.Connection": def _connection_pool(self) -> "sqlalchemy.pool.QueuePool": from .server.metrics import monitor_db_pool - creator = self._adbc_connection.adbc_clone + creator = self._instrument_adbc_creator(self._adbc_connection.adbc_clone) if (self.dialect == "duckdb") or (":memory:" in self.uri): pool = sqlalchemy.pool.StaticPool(creator) else: @@ -258,6 +258,34 @@ def _connection_pool(self) -> "sqlalchemy.pool.QueuePool": return pool + def _instrument_adbc_creator(self, creator): + """Wrap the ADBC connection factory to emit OpenTelemetry spans for its queries. + + The storage database is accessed via ADBC, which the asyncpg instrumentation does + not cover, so its queries would otherwise be invisible in traces. A no-op if tracing + is off or the optional dbapi instrumentation is not installed. + """ + if not os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"): + return creator + try: + from opentelemetry.instrumentation.dbapi import instrument_connection + except ImportError: + return creator + + dialect = self.dialect + + def instrumented_creator(): + # adbc_current_catalog is the database name (e.g. "tiled_storage"), + # which populates db.name so the database appears as its own node. + return instrument_connection( + "tiled.storage", + creator(), + dialect, + connection_attributes={"database": "adbc_current_catalog"}, + ) + + return instrumented_creator + def connect(self) -> "adbc_driver_manager.dbapi.Connection": "Get a connection from the pool." return self._connection_pool.connect() From 232f82da859636b71aaa93e960a9fc21b547db73 Mon Sep 17 00:00:00 2001 From: genematx Date: Fri, 2 Oct 2026 07:39:36 +1300 Subject: [PATCH 12/22] Instrument outbound httpx calls and name them in the service graph Trace Tiled's outbound HTTP calls (webhook deliveries, OIDC authentication, external policy servers) with opentelemetry-instrumentation-httpx. A request hook sets peer.service so each dependency becomes its own node in the Tempo service graph: all webhook deliveries (identified by the X-Tiled-Event-ID header) group under a single 'webhooks' node, while other calls are named by host. --- pyproject.toml | 2 ++ tiled/server/app.py | 28 +++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e10962e3d..0a44c9676 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,6 +91,7 @@ all = [ "opentelemetry-instrumentation-asyncpg", "opentelemetry-instrumentation-dbapi", "opentelemetry-instrumentation-fastapi", + "opentelemetry-instrumentation-httpx", "opentelemetry-instrumentation-redis", "opentelemetry-sdk", "packaging", @@ -236,6 +237,7 @@ server = [ "opentelemetry-instrumentation-asyncpg", "opentelemetry-instrumentation-dbapi", "opentelemetry-instrumentation-fastapi", + "opentelemetry-instrumentation-httpx", "opentelemetry-instrumentation-redis", "opentelemetry-sdk", "packaging", diff --git a/tiled/server/app.py b/tiled/server/app.py index ed146b59b..ec7863081 100644 --- a/tiled/server/app.py +++ b/tiled/server/app.py @@ -1121,7 +1121,8 @@ def _setup_opentelemetry_tracing(app: FastAPI) -> None: trace.set_tracer_provider(provider) FastAPIInstrumentor.instrument_app(app) - # Emit spans for calls to the PostgreSQL driver (asyncpg) and Redis. These + # Emit spans for calls to the PostgreSQL driver (asyncpg), Redis, and + # outbound HTTP (httpx: OIDC, webhooks, external policy servers). These # patch the libraries globally, so they are no-ops until a request uses them. try: from opentelemetry.instrumentation.asyncpg import AsyncPGInstrumentor @@ -1135,6 +1136,31 @@ def _setup_opentelemetry_tracing(app: FastAPI) -> None: pass else: RedisInstrumentor().instrument() + try: + from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor + except ImportError: + pass + else: + + def _set_peer_service(span, request): + # Name the downstream dependency so it becomes its own node in the + # Tempo service graph (via peer.service) and is filterable in Jaeger. + # Bundle every webhook delivery under a single 'webhooks' node; name + # other outbound calls (OIDC, external policy servers) by their host. + if span is None or not span.is_recording(): + return + if "x-tiled-event-id" in request.headers: + span.set_attribute("peer.service", "webhooks") + elif request.url.host: + span.set_attribute("peer.service", request.url.host) + + async def _set_peer_service_async(span, request): + _set_peer_service(span, request) + + HTTPXClientInstrumentor().instrument( + request_hook=_set_peer_service, + async_request_hook=_set_peer_service_async, + ) def build_app_from_config(config: Union[Config, dict[str, Any]], scalable=False): From fc72041e21215a153a5e8f5e6aab5ed0e546fb79 Mon Sep 17 00:00:00 2001 From: genematx Date: Fri, 2 Oct 2026 07:39:43 +1300 Subject: [PATCH 13/22] Add an external-service dependency graph to the monitoring example Set peer.service on database and cache client spans in the Collector (the database name for Postgres, db.system otherwise) and drop noisy transaction-control statements. Have Tempo's metrics generator build service graph metrics from the peer attributes and remote-write them to Prometheus, and wire Grafana's Tempo datasource to that Prometheus for the Service Graph. --- compose.monitoring.yml | 5 +++ .../provisioning/datasources/prometheus.yml | 1 + .../provisioning/datasources/tempo.yml | 3 ++ .../otel-collector/otel-collector.yml | 35 ++++++++++++++++--- monitoring_example/tempo/tempo.yml | 27 ++++++++++++++ 5 files changed, 66 insertions(+), 5 deletions(-) diff --git a/compose.monitoring.yml b/compose.monitoring.yml index b31a140b0..19cbda57a 100644 --- a/compose.monitoring.yml +++ b/compose.monitoring.yml @@ -11,6 +11,9 @@ services: - OTEL_SERVICE_NAME=tiled # Don't trace health checks and metrics scrapes (operational chatter). - OTEL_PYTHON_FASTAPI_EXCLUDED_URLS=healthz,api/v1/metrics + # Sample a fraction of traces in production (default: trace everything). + # - OTEL_TRACES_SAMPLER=parentbased_traceidratio + # - OTEL_TRACES_SAMPLER_ARG=0.1 prometheus: image: docker.io/prom/prometheus:v2.42.0 @@ -21,6 +24,8 @@ services: - '--storage.tsdb.path=/prometheus' - '--web.console.libraries=/usr/share/prometheus/console_libraries' - '--web.console.templates=/usr/share/prometheus/consoles' + # Accept remote-written metrics from Tempo's metrics generator (service graph). + - '--web.enable-remote-write-receiver' networks: - backend restart: unless-stopped diff --git a/monitoring_example/grafana/provisioning/datasources/prometheus.yml b/monitoring_example/grafana/provisioning/datasources/prometheus.yml index 6509b42b5..6900ca7ee 100644 --- a/monitoring_example/grafana/provisioning/datasources/prometheus.yml +++ b/monitoring_example/grafana/provisioning/datasources/prometheus.yml @@ -1,6 +1,7 @@ apiVersion: 1 datasources: - name: Prometheus + uid: prometheus access: proxy type: prometheus url: http://prometheus:9090 diff --git a/monitoring_example/grafana/provisioning/datasources/tempo.yml b/monitoring_example/grafana/provisioning/datasources/tempo.yml index 65c2d04b0..a235a4cce 100644 --- a/monitoring_example/grafana/provisioning/datasources/tempo.yml +++ b/monitoring_example/grafana/provisioning/datasources/tempo.yml @@ -5,3 +5,6 @@ datasources: type: tempo url: http://tempo:3200 uid: tempo + jsonData: + serviceMap: + datasourceUid: prometheus diff --git a/monitoring_example/otel-collector/otel-collector.yml b/monitoring_example/otel-collector/otel-collector.yml index cc78bd99c..e8af77611 100644 --- a/monitoring_example/otel-collector/otel-collector.yml +++ b/monitoring_example/otel-collector/otel-collector.yml @@ -1,12 +1,15 @@ # OpenTelemetry Collector configuration. # -# Traces: tiled --OTLP--> otel-collector --OTLP--> jaeger +# Traces: tiled --OTLP--> otel-collector --OTLP--> jaeger and tempo # Metrics: otel-collector <--scrape-- tiled:8000/api/v1/metrics # otel-collector --expose--> :8889 (Prometheus format) # -# The collector receives OTLP traces and forwards them to Jaeger. It also -# scrapes Tiled's Prometheus metrics endpoint and re-exposes those metrics in -# Prometheus format on port 8889. +# The collector receives OTLP traces and forwards the same spans to both Jaeger +# and Grafana Tempo. Database and cache calls stay under the "tiled" service +# (they are client spans emitted by Tiled) and are identified by their +# db.system / peer.service attributes; peer.service also lets Tempo draw the +# service graph. The collector also scrapes Tiled's Prometheus metrics endpoint +# and re-exposes those metrics on port 8889. receivers: otlp: @@ -34,6 +37,28 @@ receivers: processors: batch: {} + # Drop noisy transaction/session-control statements so traces show the + # queries that matter, not every BEGIN/COMMIT/ROLLBACK. + filter/db_noise: + error_mode: ignore + traces: + span: + - 'attributes["db.system"] != nil and IsMatch(name, "^(BEGIN|COMMIT|ROLLBACK|SET|SHOW)")' + + # Set peer.service on database/cache client spans so Tempo's service graph + # names the dependency nodes. Postgres spans use the database name, so the + # catalog, storage, and authn databases appear as separate nodes; other + # datastores (e.g. Redis) fall back to db.system. + transform/peer_service: + error_mode: ignore + trace_statements: + - context: span + statements: + - set(attributes["peer.service"], attributes["db.name"]) + where attributes["db.system"] == "postgresql" and attributes["db.name"] != nil + - set(attributes["peer.service"], attributes["db.system"]) + where attributes["peer.service"] == nil and attributes["db.system"] != nil + exporters: # Forward traces to Jaeger's OTLP gRPC receiver. otlp/jaeger: @@ -56,7 +81,7 @@ service: pipelines: traces: receivers: [otlp] - processors: [batch] + processors: [filter/db_noise, transform/peer_service, batch] exporters: [otlp/jaeger, otlp/tempo, debug] metrics: receivers: [prometheus] diff --git a/monitoring_example/tempo/tempo.yml b/monitoring_example/tempo/tempo.yml index c887d1f3c..7dcdc2c8f 100644 --- a/monitoring_example/tempo/tempo.yml +++ b/monitoring_example/tempo/tempo.yml @@ -31,3 +31,30 @@ storage: path: /tmp/tempo/blocks wal: path: /tmp/tempo/wal + +# Generate service-graph and span metrics from incoming traces and remote-write +# them to Prometheus. Grafana's Tempo "Service Graph" reads these. Client spans +# to Postgres and Redis (which have no server span) become virtual nodes named +# from the peer.service attribute the collector sets: the database name for +# Postgres (e.g. tiled_catalog, tiled_storage) and db.system for others (e.g. +# redis). +metrics_generator: + registry: + external_labels: + source: tempo + storage: + path: /tmp/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + traces_storage: + path: /tmp/tempo/generator/traces + processor: + service_graphs: + # Use peer.service (set by the collector) to name dependency nodes. + peer_attributes: [peer.service] + +overrides: + defaults: + metrics_generator: + processors: [service-graphs, span-metrics] From a51b70dd71bd730fa61b49defac4d3ed35015d0e Mon Sep 17 00:00:00 2001 From: genematx Date: Fri, 2 Oct 2026 07:39:53 +1300 Subject: [PATCH 14/22] Test external-service tracing spans Extend the in-process tracing tests to assert the external-service spans: asyncpg (catalog Postgres), ADBC (SQL storage), Redis (streaming cache), and httpx (webhook delivery). These run against live backends and skip when TILED_TEST_POSTGRESQL_URI / TILED_TEST_REDIS are not configured. --- tests/test_tracing.py | 225 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 214 insertions(+), 11 deletions(-) diff --git a/tests/test_tracing.py b/tests/test_tracing.py index 45e16cf02..7078a046b 100644 --- a/tests/test_tracing.py +++ b/tests/test_tracing.py @@ -1,13 +1,24 @@ -"""In-process tests for the OpenTelemetry request tracing configured by -`tiled.server.app._setup_opentelemetry_tracing`. +"""In-process tests for Tiled's OpenTelemetry tracing. -These use an in-memory span exporter, so they need no running OpenTelemetry -Collector, Jaeger, or Tempo. OpenTelemetry's global tracer provider can only be -set once per process, so a single provider is installed for the whole module and -the exporter is cleared between tests. +Two groups, both capturing spans with an in-memory exporter (no Collector, +Jaeger, or Tempo needed): + +* request-level tracing configured by + `tiled.server.app._setup_opentelemetry_tracing` -- the server span and phase + spans, excluded endpoints, off-by-default, and no duplicate export pipeline; +* external-service instrumentation -- asyncpg (catalog Postgres), ADBC (SQL + storage), Redis (streaming cache), and httpx (webhook delivery) -- exercised + against live backends, which skip when the backend is not configured via + `TILED_TEST_POSTGRESQL_URI` / `TILED_TEST_REDIS`. + +OpenTelemetry's global tracer provider can only be set once per process, so a +single provider is installed for the whole module and the exporter is cleared +between tests. """ import os +import numpy as np +import pyarrow import pytest from opentelemetry import trace from opentelemetry.sdk.trace import TracerProvider @@ -15,10 +26,11 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from opentelemetry.trace import SpanKind -from tiled.client import Context -from tiled.server.app import build_app_from_config - -pytest.importorskip("opentelemetry.sdk") +from tiled.catalog import in_memory +from tiled.client import Context, from_context +from tiled.config import Authentication, WebhooksConfig +from tiled.server.app import build_app, build_app_from_config +from tiled.server.schemas import WebhookRegistrationRequest # The FastAPI instrumentation reads OTEL_PYTHON_FASTAPI_EXCLUDED_URLS once, at # import time, into a module-level default that `instrument_app` uses. Set it @@ -27,14 +39,19 @@ # in the environment before the process starts. os.environ["OTEL_PYTHON_FASTAPI_EXCLUDED_URLS"] = "healthz,api/v1/metrics" +# Minimal in-memory tree used by the request-level tests. CONFIG = { "authentication": {"single_user_api_key": "secret"}, "trees": [{"path": "/", "tree": "tiled.examples.generated_minimal:tree"}], } # A syntactically valid endpoint that is never actually contacted: the tracing # hook reuses the in-memory provider installed below instead of creating an OTLP -# exporter, so no network traffic occurs. +# exporter, so no network traffic occurs. We only declare it to turn tracing on. ENDPOINT = "http://otel-collector.invalid:4318" +API_KEY = "secret" +# respx mocks this, so no real delivery happens (and HTTPS keeps the default +# URL validator happy without allowing http/private targets). +WEBHOOK_URL = "https://webhook.example.com/tiled-events" # Global library instrumentation installed by the tracing hook, which must be # undone so it does not leak into other test modules. @@ -72,6 +89,12 @@ def _clear_spans(span_exporter): span_exporter.clear() +def _enable_tracing(monkeypatch): + # Set before build_app so the tracing hook runs and instruments the + # libraries against the in-memory provider. + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", ENDPOINT) + + def _build_app(monkeypatch, *, endpoint=ENDPOINT): if endpoint is None: monkeypatch.delenv("OTEL_EXPORTER_OTLP_ENDPOINT", raising=False) @@ -94,6 +117,13 @@ def _is_descendant_of(span, ancestor_span_id, by_id): return False +def _spans_where(spans, key, value): + return [s for s in spans if s.attributes.get(key) == value] + + +# --- request-level tracing ------------------------------------------------- + + def test_traced_request_emits_server_and_phase_spans(monkeypatch, span_exporter): app = _build_app(monkeypatch) with Context.from_app(app) as context: @@ -163,3 +193,176 @@ def test_no_duplicate_export_pipeline(monkeypatch, span_exporter): "a second span processor was registered on the global provider; " "FastAPI's built-in OpenTelemetry auto-configuration is not disabled" ) + + +# --- external-service spans (live backends; skip when not configured) ------ + + +def test_catalog_query_emits_asyncpg_spans( + monkeypatch, span_exporter, postgres_uri, tmp_path +): + """Catalog access over asyncpg produces postgresql client spans.""" + _enable_tracing(monkeypatch) + config = { + "authentication": {"single_user_api_key": API_KEY}, + "trees": [ + { + "tree": "catalog", + "path": "/", + "args": { + "uri": postgres_uri, + "writable_storage": [str(tmp_path / "data")], + "init_if_not_exists": True, + }, + } + ], + } + with Context.from_app(build_app_from_config(config)) as context: + client = from_context(context) + client.write_array(np.arange(5), key="arr") + span_exporter.clear() + list(client) # a search -> catalog SELECT over asyncpg + + spans = span_exporter.get_finished_spans() + pg_spans = _spans_where(spans, "db.system", "postgresql") + assert pg_spans, "expected asyncpg (postgresql) spans for the catalog query" + # The catalog database name appears on the span so it is distinguishable in the service graph + assert any(s.attributes.get("db.name") for s in pg_spans) + + +def test_sql_storage_write_emits_adbc_span( + monkeypatch, span_exporter, postgres_uri, sqlite_uri, tmp_path +): + """Writing an appendable table to SQL (Postgres) storage produces both the + manual `adbc_ingest` span (the bulk write bypasses the DBAPI `execute` path) + and the DBAPI-level spans from the instrumented ADBC connection.""" + _enable_tracing(monkeypatch) + config = { + "authentication": {"single_user_api_key": API_KEY}, + "trees": [ + { + "tree": "catalog", + "path": "/", + "args": { + # Catalog on SQLite; storage on Postgres so the write goes through ADBC. + "uri": sqlite_uri, + "writable_storage": [postgres_uri], + "init_if_not_exists": True, + }, + } + ], + } + table = pyarrow.Table.from_pydict({"A": [1, 2, 3], "B": [4, 5, 6]}) + with Context.from_app(build_app_from_config(config)) as context: + client = from_context(context) + span_exporter.clear() + appendable = client.create_appendable_table(schema=table.schema, key="tab") + appendable.append_partition(0, table) + + spans = span_exporter.get_finished_spans() + names = {s.name for s in spans} + assert "adbc_ingest" in names, "expected the manual adbc_ingest span" + ingest = next(s for s in spans if s.name == "adbc_ingest") + assert ingest.kind == SpanKind.CLIENT + assert ingest.attributes.get("db.system") == "postgresql" + + # The ADBC connection factory is wrapped by _instrument_adbc_creator, so the + # DBAPI-level statements (e.g. the CREATE TABLE preceding the ingest) are + # also traced, carrying the storage database name (from adbc_current_catalog). + dbapi_spans = [ + s + for s in spans + if s.attributes.get("db.system") == "postgresql" and s.name != "adbc_ingest" + ] + assert dbapi_spans, "expected DBAPI-instrumented storage query spans" + assert any(s.attributes.get("db.name") for s in dbapi_spans) + + +def test_streaming_emits_redis_spans(monkeypatch, span_exporter, redis_uri, tmp_path): + """Subscribing to a node's stream exercises the Redis streaming cache.""" + _enable_tracing(monkeypatch) + config = { + "authentication": {"single_user_api_key": API_KEY}, + "trees": [ + { + "tree": "catalog", + "path": "/", + "args": { + "uri": "sqlite:///:memory:", + "writable_storage": [str(tmp_path / "data")], + "init_if_not_exists": True, + }, + } + ], + "streaming_cache": { + "uri": redis_uri, + "data_ttl": 600, + "seq_ttl": 600, + "socket_timeout": 600, + "socket_connect_timeout": 10, + }, + } + with Context.from_app(build_app_from_config(config)) as context: + client = from_context(context) + test_client = context.http_client # the underlying starlette TestClient + node = client.write_array(np.arange(10), key="stream_node") + span_exporter.clear() + with test_client.websocket_connect( + "/api/v1/stream/single/stream_node?envelope_format=json", + headers={"Authorization": f"Apikey {API_KEY}"}, + ): + node.write(np.arange(10) + 1) + + spans = span_exporter.get_finished_spans() + redis_spans = _spans_where(spans, "db.system", "redis") + assert redis_spans, "expected Redis spans for the streaming subscription" + + +def test_webhook_delivery_emits_httpx_span(monkeypatch, span_exporter, tmp_path): + """Delivering a webhook goes through httpx, producing an outbound client + span. No external backend is needed; the delivery is mocked with respx.""" + respx = pytest.importorskip("respx") + from unittest.mock import patch + + from httpx import Response + + _enable_tracing(monkeypatch) + tree = in_memory(writable_storage=[f"file://localhost{tmp_path / 'data'}"]) + app = build_app( + tree, + authentication=Authentication(single_user_api_key=API_KEY), + # A non-None webhooks config enables the delivery dispatcher. + server_settings={"webhooks": WebhooksConfig(secret_keys=["test-webhook-key"])}, + ) + + with Context.from_app(app) as context: + client = from_context(context) + # respx mocks the delivery; patching the SSRF check lets us register an example.com target + with respx.mock, patch("tiled.server.webhook_router.check_url_ssrf_safety"): + respx.post(WEBHOOK_URL).mock(return_value=Response(200)) + context.http_client.post( + "/api/v1/webhooks/target/", + json=WebhookRegistrationRequest(url=WEBHOOK_URL).model_dump( + mode="json" + ), + ).raise_for_status() + span_exporter.clear() + client.create_container("triggers_webhook") + + spans = span_exporter.get_finished_spans() + httpx_spans = [ + s + for s in spans + if s.kind == SpanKind.CLIENT + and ( + s.attributes.get("http.method") == "POST" + or s.attributes.get("http.request.method") == "POST" + ) + ] + assert httpx_spans, "expected an outbound httpx client span for the webhook POST" + # The span targets the webhook URL's host. + urls = [ + str(s.attributes.get("http.url") or s.attributes.get("url.full") or "") + for s in httpx_spans + ] + assert any("webhook.example.com" in u for u in urls) From 63b1d85a4d0649ade05f057323192737715e3c15 Mon Sep 17 00:00:00 2001 From: genematx Date: Fri, 2 Oct 2026 07:39:53 +1300 Subject: [PATCH 15/22] Document external-service tracing Describe the asyncpg/ADBC/Redis/httpx spans, note that database tracing covers PostgreSQL (not SQLite), add a sampling section, and explain the Grafana service graph of Tiled's dependencies. --- CHANGELOG.md | 6 ++++ docs/source/user-guide/tracing.md | 46 +++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3356e9a54..0b817fc47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,12 @@ Write the date in place of the "Unreleased" in the case a new version is release (`telemetry={"auto_configure": False}`) so that, on FastAPI >=0.142, it does not register a second OTLP exporter alongside Tiled's own tracing pipeline and export every span twice. +- Emit OpenTelemetry spans for PostgreSQL queries (asyncpg for the catalog and + authentication databases, ADBC for the storage database), Redis commands, and + outbound HTTP calls (httpx: OIDC, webhooks, external policy servers), so + external-service calls appear in traces. The example monitoring stack also + generates a service graph viewable in Grafana, with Tiled's separate Postgres + databases (catalog, storage, authn) and Redis shown as distinct nodes. - Documentation: a user-guide page on validating metadata against custom specs via server configuration. diff --git a/docs/source/user-guide/tracing.md b/docs/source/user-guide/tracing.md index 6d1cf3bb1..a2df4c0af 100644 --- a/docs/source/user-guide/tracing.md +++ b/docs/source/user-guide/tracing.md @@ -57,6 +57,11 @@ Collector could also carry OpenTelemetry's other two signals — **metrics** and **logs** — over OTLP to backends such as Prometheus and Loki. Those paths are not currently enabled. +```{note} +Database-query spans are emitted only for PostgreSQL (via asyncpg and ADBC); +tracing with SQLite-backed catalogs is not supported. +``` + ## Enabling tracing Tracing is **disabled by default**. It is turned on by setting the standard @@ -71,6 +76,30 @@ directly). Related environment variables: | `OTEL_PYTHON_FASTAPI_EXCLUDED_URLS` | Comma-separated URL patterns to exclude from tracing, e.g. `healthz,api/v1/metrics` to skip health checks and metrics scrapes. | +## Sampling + +By default every request is traced in full. That is convenient for trying it out +but can be a lot of data in production, especially since each request emits a +span per database query. Sampling is controlled by the standard OpenTelemetry +environment variables: + +| Variable | Purpose | +| --- | --- | +| `OTEL_TRACES_SAMPLER` | Sampling strategy. Default `parentbased_always_on` (trace everything). Use `parentbased_traceidratio` to keep a fraction. | +| `OTEL_TRACES_SAMPLER_ARG` | Argument for the sampler; for the ratio samplers, the fraction of traces to keep (0.0-1.0). | + +For example, to keep 10% of traces: + +``` +OTEL_TRACES_SAMPLER=parentbased_traceidratio +OTEL_TRACES_SAMPLER_ARG=0.1 +``` + +The `parentbased_*` samplers make the decision once at the start of a trace and +apply it to all of that trace's spans, so a sampled request keeps its database +and cache spans together with the rest of the trace. + + ## How does it work? 1. When `OTEL_EXPORTER_OTLP_ENDPOINT` is set, Tiled configures an OpenTelemetry @@ -120,6 +149,23 @@ The example forwards traces to two backends so you can compare their functionali [TraceQL](https://grafana.com/docs/tempo/latest/traceql/), for example `{ resource.service.name = "tiled" }`. +Each trace also includes spans for the **PostgreSQL** queries (against the +catalog, storage, and authentication databases), **Redis** commands (for the +streaming cache), and any **outbound HTTP** calls Tiled makes while serving the +request (OIDC authentication, webhooks, and external policy servers, via +httpx). These are client spans emitted by Tiled, so they share the `tiled` +service, but they carry a `db.system` attribute (`postgresql` or `redis`) — or, +for HTTP calls, the target host — that distinguishes them from Tiled's own +`tiled.*` spans. The Collector drops transaction-control statements +(`BEGIN`/`COMMIT`/`ROLLBACK`) to keep traces concise and readable. + +Filter spans by the `db.system` attribute (Grafana's span filters, or Jaeger's +find-within-trace box) to highlight the database and cache work. Grafana's +**Service Graph** (Explore → Tempo) also renders Tiled's dependencies as nodes: +each Postgres database (`tiled_catalog`, `tiled_storage`, authn) by `db.name`, +`redis`, and outbound HTTP — webhook deliveries grouped under one `webhooks` +node, other calls (e.g. OIDC) named by host. + ```{note} The bundled Collector also scrapes Tiled's `/api/v1/metrics` endpoint and re-exposes it on port 8889, in addition to Prometheus scraping it directly. From af2018de738fef12a4c8a147dbcb653131c0e3fe Mon Sep 17 00:00:00 2001 From: genematx Date: Wed, 7 Oct 2026 09:38:36 +1300 Subject: [PATCH 16/22] Restore CHANGELOG entry dropped in merge from main The FastAPI auto_configure opt-out entry was lost while resolving a CHANGELOG conflict when merging main. --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44632e379..a8fd54035 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ Write the date in place of the "Unreleased" in the case a new version is release - Emit OpenTelemetry spans for internal request phases (access control, read, tokenize, pack) so they appear as child spans in a request's trace, giving a per-request breakdown of where time is spent. +- Disable FastAPI's built-in OpenTelemetry auto-configuration + (`telemetry={"auto_configure": False}`) so that, on FastAPI >=0.142, it does + not register a second OTLP exporter alongside Tiled's own tracing pipeline and + export every span twice. - Expose the Deployment `strategy` in the helm chart, so that a deployment can use `Recreate` instead of the default `RollingUpdate`. - Add a `DELETE /api/v1/asset/{path}?id=N` endpoint to dissociate a single From ed75483b82a6ac1797cd2d3acabaed567d217355 Mon Sep 17 00:00:00 2001 From: genematx Date: Fri, 9 Oct 2026 09:43:47 +1300 Subject: [PATCH 17/22] STY: refactor app kwargs --- tiled/server/app.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/tiled/server/app.py b/tiled/server/app.py index 496155c8a..17151abd8 100644 --- a/tiled/server/app.py +++ b/tiled/server/app.py @@ -1,6 +1,7 @@ import asyncio import collections import contextvars +import importlib import logging import os import secrets @@ -79,6 +80,7 @@ CSRF_QUERY_PARAMETER = "csrf" MINIMUM_SUPPORTED_PYTHON_CLIENT_VERSION = packaging.version.parse("0.1.0a104") +FASTAPI_VERSION = packaging.version.Version(importlib.metadata.version("fastapi")) logger = logging.getLogger(__name__) logger.setLevel("INFO") @@ -286,21 +288,17 @@ async def lifespan(app: FastAPI): finally: await shutdown_event() - try: - # FastAPI >=0.142 ships built-in OpenTelemetry support that, when - # `OTEL_EXPORTER_OTLP_ENDPOINT` (or a related variable) is set, - # auto-registers its own OTLP export pipeline on the global tracer - # provider. Tiled configures and manages its own tracing pipeline (see - # `_setup_opentelemetry_tracing`), so FastAPI's auto-configuration - # would register a second exporter and emit every span twice. Opt out. - app = FastAPI( - lifespan=lifespan, - strict_content_type=False, - telemetry={"auto_configure": False}, - ) - except TypeError: - # FastAPI <0.142 has no built-in telemetry and no `telemetry` option. - app = FastAPI(lifespan=lifespan, strict_content_type=False) + # FastAPI >=0.142 ships built-in OpenTelemetry support that, when + # `OTEL_EXPORTER_OTLP_ENDPOINT` (or a related variable) is set, + # auto-registers its own OTLP export pipeline on the global tracer + # provider. Tiled configures and manages its own tracing pipeline (see + # `_setup_opentelemetry_tracing`), so FastAPI's auto-configuration + # would register a second exporter and emit every span twice. Opt out. + kwargs = dict(lifespan=lifespan, strict_content_type=False) + if FASTAPI_VERSION >= packaging.version.Version("0.142"): + kwargs["telemetry"] = {"auto_configure": False} + + app = FastAPI(**kwargs) # Healthcheck for deployment to containerized systems, needs to preempt other responses. # Standardized for Kubernetes, but also used by other systems. From d928a48052c4809b4032f0c29747dffcf1920f91 Mon Sep 17 00:00:00 2001 From: genematx Date: Fri, 9 Oct 2026 09:47:54 +1300 Subject: [PATCH 18/22] STY: explicit import checks --- tiled/server/utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tiled/server/utils.py b/tiled/server/utils.py index f18dfb6ac..cceba461f 100644 --- a/tiled/server/utils.py +++ b/tiled/server/utils.py @@ -1,4 +1,5 @@ import contextlib +import importlib.util import time from collections.abc import Generator from typing import Any, Literal, Mapping, Optional, Sequence @@ -18,14 +19,6 @@ API_KEY_QUERY_PARAMETER = "api_key" CSRF_COOKIE_NAME = "tiled_csrf" -try: - from opentelemetry import trace - - _tracer = trace.get_tracer("tiled.server") -except ImportError: - # OpenTelemetry is an optional dependency; tracing is simply disabled. - _tracer = None - # Human-readable OpenTelemetry span names for the phases timed below. _SPAN_NAMES = { "app": "tiled.app", @@ -35,6 +28,13 @@ "pack": "tiled.pack", } +# Enable tracing if OpenTelemetry is installed +_tracer = None +if importlib.util.find_spec("opentelemetry"): + from opentelemetry import trace + + _tracer = trace.get_tracer("tiled.server") + def normalize_root_path(root_path: Optional[str]) -> str: """Coerce a root_path to "" or "/prefix" (no trailing slash).""" From b57fdaaa85b0ae3682d7516d28a32396d8011741 Mon Sep 17 00:00:00 2001 From: genematx Date: Fri, 9 Oct 2026 09:56:16 +1300 Subject: [PATCH 19/22] FIX: robust OpenTelemetry and importlib.metadata imports `opentelemetry` is a namespace package shared by all `opentelemetry-*` distributions, so `find_spec("opentelemetry")` succeeds even when the API package (which provides `opentelemetry.trace`) is not installed, and importing `tiled.server.utils` then fails. Check for `opentelemetry.trace` itself. Import `importlib.metadata` explicitly: `import importlib` does not load the submodule, and `app.py` only worked because another import loaded it. --- tiled/server/app.py | 2 +- tiled/server/utils.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tiled/server/app.py b/tiled/server/app.py index 17151abd8..b058cf304 100644 --- a/tiled/server/app.py +++ b/tiled/server/app.py @@ -1,7 +1,7 @@ import asyncio import collections import contextvars -import importlib +import importlib.metadata import logging import os import secrets diff --git a/tiled/server/utils.py b/tiled/server/utils.py index cceba461f..84900eb5b 100644 --- a/tiled/server/utils.py +++ b/tiled/server/utils.py @@ -28,9 +28,13 @@ "pack": "tiled.pack", } -# Enable tracing if OpenTelemetry is installed +# Enable tracing if the OpenTelemetry API is installed. `opentelemetry` is a +# namespace package shared by all `opentelemetry-*` distributions, so check for +# the `trace` module itself and its parent (first). _tracer = None -if importlib.util.find_spec("opentelemetry"): +if importlib.util.find_spec("opentelemetry") and importlib.util.find_spec( + "opentelemetry.trace" +): from opentelemetry import trace _tracer = trace.get_tracer("tiled.server") From 529d65b41ff7688cca5d52fa3e3419e53712dc87 Mon Sep 17 00:00:00 2001 From: genematx Date: Fri, 9 Oct 2026 10:11:19 +1300 Subject: [PATCH 20/22] FIX: tracing broke DuckDB storage connections The DBAPI instrumentation of the ADBC storage connections reads `adbc_current_catalog` for `db.name`. DuckDB's ADBC driver raises instead of returning a value, and the instrumentation only tolerates a missing attribute, so with tracing on every new DuckDB storage connection failed. Read it once and leave `db.name` unset if the driver cannot provide it. Also explain the name: ADBC uses SQL-standard terms, where a "catalog" is a database, unrelated to Tiled's catalog. Add a test writing to SQLite and DuckDB storage with tracing on. --- tests/test_tracing.py | 37 +++++++++++++++++++++++++++++++++++++ tiled/storage.py | 17 ++++++++++++++--- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/tests/test_tracing.py b/tests/test_tracing.py index 7078a046b..9d88c0ff3 100644 --- a/tests/test_tracing.py +++ b/tests/test_tracing.py @@ -278,6 +278,43 @@ def test_sql_storage_write_emits_adbc_span( assert any(s.attributes.get("db.name") for s in dbapi_spans) +@pytest.mark.parametrize("scheme", ["sqlite", "duckdb"]) +def test_embedded_sql_storage_write_with_tracing( + monkeypatch, span_exporter, tmp_path, scheme +): + """Tracing must not break embedded SQL storage. DuckDB's ADBC driver does not + implement `adbc_current_catalog` (it raises rather than returning nothing), so + the storage spans are emitted without a database name instead.""" + _enable_tracing(monkeypatch) + config = { + "authentication": {"single_user_api_key": API_KEY}, + "trees": [ + { + "tree": "catalog", + "path": "/", + "args": { + "uri": f"sqlite:///{tmp_path / 'catalog.db'}", + "writable_storage": [ + str(tmp_path / "data"), + f"{scheme}:///{tmp_path / f'tables.{scheme}'}", + ], + "init_if_not_exists": True, + }, + } + ], + } + table = pyarrow.Table.from_pydict({"A": [1, 2, 3]}) + with Context.from_app(build_app_from_config(config)) as context: + client = from_context(context) + span_exporter.clear() + appendable = client.create_appendable_table(schema=table.schema, key="tab") + appendable.append_partition(0, table) + assert appendable.read()["A"].tolist() == [1, 2, 3] + + spans = span_exporter.get_finished_spans() + assert _spans_where(spans, "db.system", scheme), f"expected {scheme} storage spans" + + def test_streaming_emits_redis_spans(monkeypatch, span_exporter, redis_uri, tmp_path): """Subscribing to a node's stream exercises the Redis streaming cache.""" _enable_tracing(monkeypatch) diff --git a/tiled/storage.py b/tiled/storage.py index 24b794c56..106e846b8 100644 --- a/tiled/storage.py +++ b/tiled/storage.py @@ -273,15 +273,26 @@ def _instrument_adbc_creator(self, creator): return creator dialect = self.dialect + # Report the database name as `db.name`, so the storage database appears as + # its own node in the service graph. ADBC follows the SQL standard naming, + # where a "catalog" is a database (and a "schema" is a namespace inside it), + # so on Postgres `adbc_current_catalog` is e.g. "tiled_storage". This has + # nothing to do with Tiled's catalog. Not every driver implements it (DuckDB + # raises), and the instrumentation only tolerates a missing attribute, not + # an error, so check once and leave `db.name` unset if it fails. + try: + self._adbc_connection.adbc_current_catalog + except Exception: + connection_attributes = {} + else: + connection_attributes = {"database": "adbc_current_catalog"} def instrumented_creator(): - # adbc_current_catalog is the database name (e.g. "tiled_storage"), - # which populates db.name so the database appears as its own node. return instrument_connection( "tiled.storage", creator(), dialect, - connection_attributes={"database": "adbc_current_catalog"}, + connection_attributes=connection_attributes, ) return instrumented_creator From 00963f9e66664079cdfc152e2ae32a73842cc1fc Mon Sep 17 00:00:00 2001 From: genematx Date: Fri, 9 Oct 2026 10:28:34 +1300 Subject: [PATCH 21/22] TST: run the SQL storage tracing test on SQLite, DuckDB and Postgres Fold the embedded-storage test into the Postgres storage test using the existing `sql_storage_uri` fixture. `db.name` is not checked on DuckDB, whose ADBC driver does not implement `adbc_current_catalog`. --- tests/test_tracing.py | 77 +++++++++++++------------------------------ 1 file changed, 22 insertions(+), 55 deletions(-) diff --git a/tests/test_tracing.py b/tests/test_tracing.py index 9d88c0ff3..d8c8b9f0e 100644 --- a/tests/test_tracing.py +++ b/tests/test_tracing.py @@ -16,6 +16,7 @@ between tests. """ import os +from urllib.parse import urlparse import numpy as np import pyarrow @@ -230,13 +231,15 @@ def test_catalog_query_emits_asyncpg_spans( assert any(s.attributes.get("db.name") for s in pg_spans) -def test_sql_storage_write_emits_adbc_span( - monkeypatch, span_exporter, postgres_uri, sqlite_uri, tmp_path +def test_sql_storage_write_emits_adbc_spans( + monkeypatch, span_exporter, sql_storage_uri, tmp_path ): - """Writing an appendable table to SQL (Postgres) storage produces both the - manual `adbc_ingest` span (the bulk write bypasses the DBAPI `execute` path) - and the DBAPI-level spans from the instrumented ADBC connection.""" + """Writing an appendable table to SQL storage (SQLite, DuckDB, or Postgres) + produces both the manual `adbc_ingest` span (the bulk write bypasses the DBAPI + `execute` path) and the DBAPI-level spans from the instrumented ADBC + connection.""" _enable_tracing(monkeypatch) + dialect = urlparse(sql_storage_uri).scheme config = { "authentication": {"single_user_api_key": API_KEY}, "trees": [ @@ -244,9 +247,8 @@ def test_sql_storage_write_emits_adbc_span( "tree": "catalog", "path": "/", "args": { - # Catalog on SQLite; storage on Postgres so the write goes through ADBC. - "uri": sqlite_uri, - "writable_storage": [postgres_uri], + "uri": f"sqlite:///{tmp_path / 'catalog.db'}", + "writable_storage": [sql_storage_uri], "init_if_not_exists": True, }, } @@ -258,61 +260,26 @@ def test_sql_storage_write_emits_adbc_span( span_exporter.clear() appendable = client.create_appendable_table(schema=table.schema, key="tab") appendable.append_partition(0, table) + # Tracing must not break the storage connections. + assert appendable.read()["A"].tolist() == [1, 2, 3] spans = span_exporter.get_finished_spans() - names = {s.name for s in spans} - assert "adbc_ingest" in names, "expected the manual adbc_ingest span" - ingest = next(s for s in spans if s.name == "adbc_ingest") - assert ingest.kind == SpanKind.CLIENT - assert ingest.attributes.get("db.system") == "postgresql" + ingest = [s for s in spans if s.name == "adbc_ingest"] + assert ingest, "expected the manual adbc_ingest span" + assert ingest[0].kind == SpanKind.CLIENT + assert ingest[0].attributes.get("db.system") == dialect # The ADBC connection factory is wrapped by _instrument_adbc_creator, so the # DBAPI-level statements (e.g. the CREATE TABLE preceding the ingest) are - # also traced, carrying the storage database name (from adbc_current_catalog). + # also traced. dbapi_spans = [ - s - for s in spans - if s.attributes.get("db.system") == "postgresql" and s.name != "adbc_ingest" + s for s in _spans_where(spans, "db.system", dialect) if s.name != "adbc_ingest" ] assert dbapi_spans, "expected DBAPI-instrumented storage query spans" - assert any(s.attributes.get("db.name") for s in dbapi_spans) - - -@pytest.mark.parametrize("scheme", ["sqlite", "duckdb"]) -def test_embedded_sql_storage_write_with_tracing( - monkeypatch, span_exporter, tmp_path, scheme -): - """Tracing must not break embedded SQL storage. DuckDB's ADBC driver does not - implement `adbc_current_catalog` (it raises rather than returning nothing), so - the storage spans are emitted without a database name instead.""" - _enable_tracing(monkeypatch) - config = { - "authentication": {"single_user_api_key": API_KEY}, - "trees": [ - { - "tree": "catalog", - "path": "/", - "args": { - "uri": f"sqlite:///{tmp_path / 'catalog.db'}", - "writable_storage": [ - str(tmp_path / "data"), - f"{scheme}:///{tmp_path / f'tables.{scheme}'}", - ], - "init_if_not_exists": True, - }, - } - ], - } - table = pyarrow.Table.from_pydict({"A": [1, 2, 3]}) - with Context.from_app(build_app_from_config(config)) as context: - client = from_context(context) - span_exporter.clear() - appendable = client.create_appendable_table(schema=table.schema, key="tab") - appendable.append_partition(0, table) - assert appendable.read()["A"].tolist() == [1, 2, 3] - - spans = span_exporter.get_finished_spans() - assert _spans_where(spans, "db.system", scheme), f"expected {scheme} storage spans" + # They carry the database name (`adbc_current_catalog`), except on DuckDB, whose + # ADBC driver does not implement it. + if dialect != "duckdb": + assert any(s.attributes.get("db.name") for s in dbapi_spans) def test_streaming_emits_redis_spans(monkeypatch, span_exporter, redis_uri, tmp_path): From ed2537cfd2787769d3bff78e1a827c6f602c946d Mon Sep 17 00:00:00 2001 From: genematx Date: Fri, 9 Oct 2026 10:41:02 +1300 Subject: [PATCH 22/22] FIX: cap the batch size in the example collector `batch: {}` has no size cap. Jaeger and Tempo reject OTLP/gRPC messages larger than 4 MiB by default and the collector drops such batches, so a burst of traces was partly lost: of 10000 spans sent in a burst, Jaeger received 1808. Cap batches at 1024 spans (send 512), which fits spans averaging under ~4 KiB; with the cap, all spans arrive in both backends. --- monitoring_example/otel-collector/otel-collector.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/monitoring_example/otel-collector/otel-collector.yml b/monitoring_example/otel-collector/otel-collector.yml index e8af77611..1503ef53a 100644 --- a/monitoring_example/otel-collector/otel-collector.yml +++ b/monitoring_example/otel-collector/otel-collector.yml @@ -35,7 +35,13 @@ receivers: - targets: ['tiled:8000'] processors: - batch: {} + # Cap the batch size: Jaeger and Tempo reject OTLP/gRPC messages larger than + # 4 MiB by default, and an oversized batch is dropped, not retried. With no + # cap (`batch: {}`), a burst of traces can exceed that limit. 1024 spans fit + # if spans average under ~4 KiB. + batch: + send_batch_size: 512 + send_batch_max_size: 1024 # Drop noisy transaction/session-control statements so traces show the # queries that matter, not every BEGIN/COMMIT/ROLLBACK.