Add Basic OpenTelemetry Tracing

CHANGELOG.md

@@ -7,6 +7,17 @@ Write the date in place of the "Unreleased" in the case a new version is release
 
 ### Added
 
+- OpenTelemetry tracing for the server, exported over OTLP. The example monitoring
+  stack (`compose.monitoring.yml`) now includes an OpenTelemetry Collector, Jaeger,
+  and Grafana Tempo (traces are sent to both backends), and the Collector also
+  scrapes and re-exposes Tiled's Prometheus metrics.
+- Emit OpenTelemetry spans for internal request phases (access control, read,
+  tokenize, pack) so they appear as child spans in a request's trace, giving a
+  per-request breakdown of where time is spent.
+- Disable FastAPI's built-in OpenTelemetry auto-configuration
+  (`telemetry={"auto_configure": False}`) so that, on FastAPI >=0.142, it does
+  not register a second OTLP exporter alongside Tiled's own tracing pipeline and
+  export every span twice.
 - Expose the Deployment `strategy` in the helm chart, so that a deployment can
   use `Recreate` instead of the default `RollingUpdate`.
 - Add a `DELETE /api/v1/asset/{path}?id=N` endpoint to dissociate a single

compose.monitoring.yml

@@ -1,5 +1,17 @@
 ---
 services:
+  # Turn on OpenTelemetry tracing for the Tiled server (defined in compose.yml
+  # or compose.dev.yml) and point it at the collector below. These variables
+  # live here, alongside the collector, so that running the base compose files
+  # on their own leaves tracing off instead of exporting to a collector that is
+  # not running.
+  tiled:
+    environment:
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
+      - OTEL_SERVICE_NAME=tiled
+      # Don't trace health checks and metrics scrapes (operational chatter).
+      - OTEL_PYTHON_FASTAPI_EXCLUDED_URLS=healthz,api/v1/metrics
+
   prometheus:
     image: docker.io/prom/prometheus:v2.42.0
     volumes:
@@ -14,7 +26,7 @@ services:
     restart: unless-stopped
 
   grafana:
-    image: docker.io/grafana/grafana:8.2.6
+    image: docker.io/grafana/grafana:11.3.0
     depends_on:
       - prometheus
     ports:
@@ -33,5 +45,50 @@ services:
       GF_AUTH_DISABLE_SIGNOUT_MENU: "true"
       GF_AUTH_DISABLE_LOGIN_FORM: "true"
 
+  # OpenTelemetry Collector: receives OTLP telemetry from apps and fans it
+  # out to backends (traces -> Jaeger and Tempo). It also scrapes Tiled's
+  # Prometheus metrics endpoint and re-exposes it for Prometheus. Apps on the
+  # 'backend' network export to http://otel-collector:4318; apps on the host
+  # export to http://localhost:4318.
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:0.130.0
+    command: ["--config=/etc/otelcol/config.yaml"]
+    volumes:
+      - ./monitoring_example/otel-collector/otel-collector.yml:/etc/otelcol/config.yaml
+    ports:
+      - 4317:4317   # OTLP gRPC
+      - 4318:4318   # OTLP HTTP
+      - 8889:8889   # Prometheus exporter (re-exposed scraped metrics)
+    depends_on:
+      - jaeger
+      - tempo
+    networks:
+      - backend
+    restart: unless-stopped
+
+  # Jaeger all-in-one: trace storage + query UI. In-memory storage (dev
+  # only; traces are lost on restart). Natively accepts OTLP.
+  jaeger:
+    image: jaegertracing/all-in-one:1.62.0
+    environment:
+      COLLECTOR_OTLP_ENABLED: "true"
+    ports:
+      - 16686:16686   # Jaeger web UI
+    networks:
+      - backend
+    restart: unless-stopped
+
+  # Grafana Tempo: trace storage queried from Grafana (no UI of its own).
+  # Stores traces on local ephemeral storage (see monitoring_example/tempo).
+  # Natively accepts OTLP. Explore traces in Grafana via the Tempo datasource.
+  tempo:
+    image: docker.io/grafana/tempo:2.6.1
+    command: ["-config.file=/etc/tempo/tempo.yml"]
+    volumes:
+      - ./monitoring_example/tempo/tempo.yml:/etc/tempo/tempo.yml
+    networks:
+      - backend
+    restart: unless-stopped
+
 networks:
   backend: {}

docs/source/_toc.yml

@@ -42,6 +42,7 @@ subtrees:
             - file: user-guide/api-keys
             - file: user-guide/custom-clients
             - file: user-guide/metrics
+            - file: user-guide/tracing
             - file: user-guide/direct-client
             - file: user-guide/tiled-authn-database
             - file: user-guide/register

docs/source/user-guide/tracing.md

@@ -0,0 +1,129 @@
+# Distributed Tracing
+
+In addition to [Prometheus metrics](./metrics.md), Tiled can emit
+[OpenTelemetry](https://opentelemetry.io/) traces. Whereas metrics describe
+aggregate behavior across many requests, a *trace* records the timeline of a
+single request as a tree of *spans*. This is useful for investigating why a
+particular request was slow.
+
+Traces are exported using the OpenTelemetry Protocol (OTLP) to an
+[OpenTelemetry Collector](https://opentelemetry.io/docs/collector/), which
+forwards them to one or more tracing backends for storage and visualization,
+such as [Jaeger](https://www.jaegertracing.io/) or
+[Grafana Tempo](https://grafana.com/oss/tempo/).
+
+```{mermaid}
+flowchart LR
+    tiled["Tiled server"]
+    collector["OpenTelemetry<br/>Collector"]
+
+    subgraph backends["Storage backends"]
+        direction TB
+        jaeger["Jaeger"]
+        tempo["Grafana Tempo"]
+        prometheus["Prometheus"]
+        loki["Loki"]
+    end
+
+    subgraph viz["Visualization"]
+        direction TB
+        jaegerui["Jaeger UI"]
+        grafana["Grafana"]
+    end
+
+    %% Configured in the example
+    tiled -->|"traces (OTLP)"| collector
+    collector -->|OTLP| jaeger
+    collector -->|OTLP| tempo
+    tiled -->|"metrics (scrape)"| prometheus
+
+    %% Visualization
+    jaeger --> jaegerui
+    tempo --> grafana
+    prometheus --> grafana
+
+    %% Metrics and logs over OTLP: possible extension, not enabled
+    tiled -.->|"metrics (OTLP)"| collector
+    tiled -.->|"logs (OTLP)"| collector
+    collector -.->|metrics| prometheus
+    collector -.->|logs| loki
+    loki -.-> grafana
+```
+
+Solid arrows are what the example configures today: Tiled pushes **traces** over
+OTLP to the Collector, which fans them out to Jaeger and Grafana Tempo, while
+Prometheus scrapes Tiled's metrics endpoint. Dashed arrows show how the same
+Collector could also carry OpenTelemetry's other two signals — **metrics** and
+**logs** — over OTLP to backends such as Prometheus and Loki. Those paths are
+not currently enabled.
+
+## Enabling tracing
+
+Tracing is **disabled by default**. It is turned on by setting the standard
+OpenTelemetry environment variable `OTEL_EXPORTER_OTLP_ENDPOINT` to the address
+of an OTLP endpoint (an OpenTelemetry Collector, or a backend that accepts OTLP
+directly). Related environment variables:
+
+| Variable | Purpose |
+| --- | --- |
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | OTLP endpoint, e.g. `http://otel-collector:4318`. Tracing is off when this is unset. |
+| `OTEL_SERVICE_NAME` | Name shown for the service in the tracing backend, e.g. `tiled`. |
+| `OTEL_PYTHON_FASTAPI_EXCLUDED_URLS` | Comma-separated URL patterns to exclude from tracing, e.g. `healthz,api/v1/metrics` to skip health checks and metrics scrapes. |
+
+
+## How does it work?
+
+1. When `OTEL_EXPORTER_OTLP_ENDPOINT` is set, Tiled configures an OpenTelemetry
+   tracer and instruments the ASGI application, creating one span per incoming
+   HTTP request.
+
+2. Spans are exported over OTLP to the OpenTelemetry Collector.
+
+3. The Collector forwards traces to one or more backends (Jaeger and Grafana
+   Tempo in the example stack), which store them and make them available to
+   search and visualize.
+
+
+## Try it with the example stack
+
+Tiled ships example configuration that runs an OpenTelemetry Collector and
+Jaeger alongside the server, Prometheus, and Grafana. From the repository root,
+start the server together with the monitoring services:
+
+```
+TILED_SINGLE_USER_API_KEY=secret \
+  docker compose -f compose.dev.yml -f compose.monitoring.yml up --build
+```
+
+`compose.dev.yml` builds the Tiled image from this checkout (so it includes the
+tracing support), and `compose.monitoring.yml` sets the `OTEL_*` variables above
+and runs the Collector, so the server exports traces to it. (The published image
+referenced by `compose.yml` may not yet include tracing.)
+
+Generate some activity using the Tiled Python client:
+
+```python
+from tiled.client import from_uri
+
+c = from_uri("http://localhost:8000", api_key="secret")
+c.create_container('test')
+list(c)
+```
+
+The example forwards traces to two backends so you can compare their functionality:
+
+- **Jaeger:** open [http://localhost:16686](http://localhost:16686), select the
+  **tiled** service, and click **Find Traces**. Click a trace to see its span
+  waterfall.
+- **Grafana Tempo:** open [http://localhost:3000](http://localhost:3000), go to
+  **Explore**, select the **Tempo** data source, and search using
+  [TraceQL](https://grafana.com/docs/tempo/latest/traceql/), for example
+  `{ resource.service.name = "tiled" }`.
+
+```{note}
+The bundled Collector also scrapes Tiled's `/api/v1/metrics` endpoint and
+re-exposes it on port 8889, in addition to Prometheus scraping it directly.
+To disable this, remove the `metrics` pipeline from the Collector
+configuration in `monitoring_example/otel-collector/otel-collector.yml`.
+See [Prometheus Metrics](./metrics.md).
+```

monitoring_example/grafana/provisioning/datasources/prometheus.yml

@@ -1,3 +1,4 @@
+apiVersion: 1
 datasources:
 - name: Prometheus
   access: proxy

monitoring_example/grafana/provisioning/datasources/tempo.yml

@@ -0,0 +1,7 @@
+apiVersion: 1
+datasources:
+- name: Tempo
+  access: proxy
+  type: tempo
+  url: http://tempo:3200
+  uid: tempo

monitoring_example/otel-collector/otel-collector.yml

@@ -0,0 +1,67 @@
+# OpenTelemetry Collector configuration.
+#
+# Traces:   tiled --OTLP--> otel-collector --OTLP--> jaeger
+# Metrics:  otel-collector <--scrape-- tiled:8000/api/v1/metrics
+#           otel-collector --expose--> :8889 (Prometheus format)
+#
+# The collector receives OTLP traces and forwards them to Jaeger. It also
+# scrapes Tiled's Prometheus metrics endpoint and re-exposes those metrics in
+# Prometheus format on port 8889.
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+  # Scrape Tiled's Prometheus metrics endpoint. This is a standard Prometheus
+  # scrape config. The API key must match TILED_SINGLE_USER_API_KEY (or an API
+  # key carrying the "metrics" scope in a multi-user deployment).
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: tiled
+          metrics_path: /api/v1/metrics
+          scrape_interval: 5s
+          authorization:
+            type: Apikey
+            credentials: secret
+          static_configs:
+            - targets: ['tiled:8000']
+
+processors:
+  batch: {}
+
+exporters:
+  # Forward traces to Jaeger's OTLP gRPC receiver.
+  otlp/jaeger:
+    endpoint: jaeger:4317
+    tls:
+      insecure: true
+  # Forward the same traces to Grafana Tempo's OTLP gRPC receiver.
+  otlp/tempo:
+    endpoint: tempo:4317
+    tls:
+      insecure: true
+  # Re-expose scraped metrics in Prometheus format for Prometheus to scrape.
+  prometheus:
+    endpoint: 0.0.0.0:8889
+  # Log received spans to the collector's stdout (useful for debugging).
+  debug:
+    verbosity: detailed
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp/jaeger, otlp/tempo, debug]
+    metrics:
+      receivers: [prometheus]
+      processors: [batch]
+      exporters: [prometheus]
+  telemetry:
+    logs:
+      level: info

monitoring_example/prometheus/prometheus.yml

@@ -3,6 +3,10 @@ global:
   scrape_interval: 5s
 
 scrape_configs:
+  # Scrape Tiled's metrics endpoint directly. The OpenTelemetry Collector
+  # independently scrapes the same endpoint and brings the metrics into its
+  # pipeline (see monitoring_example/otel-collector/otel-collector.yml),
+  # re-exposing them on otel-collector:8889 for forwarding to other backends.
   - job_name: 'tiled'
     metrics_path: /api/v1/metrics
     authorization:  # Set Authorization header to 'Apikey secret'.

monitoring_example/tempo/tempo.yml

@@ -0,0 +1,33 @@
+# Grafana Tempo configuration (single binary, for the example stack).
+#
+# Tempo receives traces over OTLP from the OpenTelemetry Collector and stores
+# them on the local filesystem. This example writes to /tmp (ephemeral, lost on
+# restart) so it needs no volume or permission setup. Traces are queried from
+# Grafana via the Tempo datasource.
+
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: 0.0.0.0:4317
+        http:
+          endpoint: 0.0.0.0:4318
+
+ingester:
+  max_block_duration: 5m
+
+compactor:
+  compaction:
+    block_retention: 1h
+
+storage:
+  trace:
+    backend: local
+    local:
+      path: /tmp/tempo/blocks
+    wal:
+      path: /tmp/tempo/wal

pyproject.toml

@@ -87,6 +87,9 @@ all = [
     "numba >=0.59.0",  # indirect, pinned to assist uv solve
     "obstore",
     "openpyxl",
+    "opentelemetry-exporter-otlp-proto-http",
+    "opentelemetry-instrumentation-fastapi",
+    "opentelemetry-sdk",
     "packaging",
     "pandas <3",
     "pillow",
@@ -226,6 +229,9 @@ server = [
     "numpy",
     "obstore",
     "openpyxl",
+    "opentelemetry-exporter-otlp-proto-http",
+    "opentelemetry-instrumentation-fastapi",
+    "opentelemetry-sdk",
     "packaging",
     "pandas",
     "pillow",