diff --git a/README.md b/README.md index 443018d..09ed09c 100644 --- a/README.md +++ b/README.md @@ -62,3 +62,4 @@ run with `--metrics` argument, e.g. `--metrics 127.0.0.1:9001`. Metrics can then be retrieved with `curl http://127.0.0.1:9001/metrics`. +In `contrib/grafana ` are two example dashboards for deliveries and failures. diff --git a/contrib/grafana/dashboard-notifications-deliveries.json b/contrib/grafana/dashboard-notifications-deliveries.json new file mode 100644 index 0000000..64d3a30 --- /dev/null +++ b/contrib/grafana/dashboard-notifications-deliveries.json @@ -0,0 +1,2223 @@ +{ + "apiVersion": "dashboard.grafana.app/v2", + "kind": "Dashboard", + "metadata": { + "name": "cdiuqfaf7jk74cCd", + "generation": 2, + "creationTimestamp": "2026-04-08T08:21:12Z", + "labels": {}, + "annotations": {} + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana", + "version": "v0", + "spec": {}, + "labels": { + "grafana.app/export-label": "grafana-1" + } + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "builtIn": true, + "legacyOptions": { + "type": "dashboard" + } + } + } + ], + "cursorSync": "Tooltip", + "description": "Chatmail notifications: successful deliveries, heartbeat, and debounce. See https://github.com/chatmail/notifiers", + "editable": true, + "elements": { + "panel-10": { + "kind": "Panel", + "spec": { + "id": 10, + "title": "Heartbeat registrations (20m)", + "description": "APNS device tokens newly registered for heartbeat via /register, summed per 20-minute window. Bursts typically indicate app update or reinstall waves.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "increase(heartbeat_registrations_total[20m])", + "legendFormat": "registrations", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "rate(heartbeat_registrations_total[$smoothing]) * 1200", + "legendFormat": "registrations trend", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "registrations" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "registrations trend" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 5 + ], + "fill": "dot" + } + }, + { + "id": "custom.lineInterpolation", + "value": "linear" + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + } + ] + } + ] + } + } + } + } + }, + "panel-11": { + "kind": "Panel", + "spec": { + "id": 11, + "title": "Heartbeat notifications", + "description": "Silent APNS wakeup notifications sent on the heartbeat schedule. Left axis: heartbeats delivered per 20-minute window. Right axis: tokens currently scheduled for heartbeat. Small fluctuations are expected from scheduling jitter.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "increase(heartbeat_notifications_total[20m])", + "legendFormat": "delivered", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "rate(heartbeat_notifications_total[$smoothing]) * 1200", + "legendFormat": "delivered trend", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "C", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "heartbeat_tokens", + "legendFormat": "registered tokens", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "avg_over_time(heartbeat_tokens[$smoothing])", + "legendFormat": "registered tokens trend", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "D", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "delivered / 20m", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "delivered" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "delivered trend" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 5 + ], + "fill": "dot" + } + }, + { + "id": "custom.lineInterpolation", + "value": "linear" + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "registered tokens" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.axisLabel", + "value": "registered tokens" + }, + { + "id": "color", + "value": { + "fixedColor": "#5794F2", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "registered tokens trend" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5794F2", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 5 + ], + "fill": "dot" + } + }, + { + "id": "custom.lineInterpolation", + "value": "linear" + }, + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + }, + { + "id": "custom.axisLabel", + "value": "registered tokens" + } + ] + } + ] + } + } + } + } + }, + "panel-12": { + "kind": "Panel", + "spec": { + "id": 12, + "title": "Open file descriptors", + "description": "File descriptors open in the notifier process. A sustained rising trend indicates a leak.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "notifiers_open_files", + "legendFormat": "open files", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "avg_over_time(notifiers_open_files[$smoothing])", + "legendFormat": "open files trend", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + }, + { + "value": 80, + "color": "red" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "open files" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "open files trend" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 5 + ], + "fill": "dot" + } + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + } + ] + } + ] + } + } + } + } + }, + "panel-14": { + "kind": "Panel", + "spec": { + "id": 14, + "title": "Notifications sent", + "description": "Notifications successfully handed off to each push provider (APNS, FCM, UBports, WebPush). Counted after the provider acknowledges the send, not at enqueue time. Raw lines: $__rate_interval average. Dotted trends: $smoothing average.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "expr": "rate(direct_notifications_total[$__rate_interval]) * 60", + "legendFormat": "Apple (APNS)" + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "avg_over_time(sum(rate(direct_notifications_total[$__rate_interval]))[$smoothing:1m]) * 60", + "legendFormat": "Apple (APNS) trend", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "a", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "expr": "rate(fcm_notifications_total[$__rate_interval]) * 60", + "legendFormat": "Android (FCM)" + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "avg_over_time(sum(rate(fcm_notifications_total[$__rate_interval]))[$smoothing:1m]) * 60", + "legendFormat": "Android (FCM) trend", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "b", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "expr": "rate(ubports_notifications_total[$__rate_interval]) * 60", + "legendFormat": "UBports" + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "C", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "avg_over_time(sum(rate(ubports_notifications_total[$__rate_interval]))[$smoothing:1m]) * 60", + "legendFormat": "UBports trend", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "c", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "expr": "rate(webpush_notifications_total[$__rate_interval]) * 60", + "legendFormat": "WebPush" + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "E", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "avg_over_time(sum(rate(webpush_notifications_total[$__rate_interval]))[$smoothing:1m]) * 60", + "legendFormat": "WebPush trend", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "e", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "avg_over_time((sum(rate(direct_notifications_total[$__rate_interval])) + sum(rate(fcm_notifications_total[$__rate_interval])) + sum(rate(ubports_notifications_total[$__rate_interval])) + sum(rate(webpush_notifications_total[$__rate_interval])))[$smoothing:1m]) * 60", + "legendFormat": "Total trend", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "T", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "notifications/min ($__rate_interval avg)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "linearThreshold": 10, + "log": 10, + "type": "symlog" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apple (APNS)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Apple (APNS) trend" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 5 + ], + "fill": "dot" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineInterpolation", + "value": "linear" + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Android (FCM)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Android (FCM) trend" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 5 + ], + "fill": "dot" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineInterpolation", + "value": "linear" + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "UBports" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "UBports trend" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 5 + ], + "fill": "dot" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineInterpolation", + "value": "linear" + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "WebPush" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5794F2", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "WebPush trend" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5794F2", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 5 + ], + "fill": "dot" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineInterpolation", + "value": "linear" + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total trend" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 3 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 5 + ], + "fill": "dot" + } + }, + { + "id": "custom.fillOpacity", + "value": 5 + }, + { + "id": "custom.lineInterpolation", + "value": "linear" + } + ] + } + ] + } + } + } + } + }, + "panel-16": { + "kind": "Panel", + "spec": { + "id": 16, + "title": "Debounced set size", + "description": "Current number of notifications held in the deduplication window. When the same token is registered twice (typically after a reinstall), the gateway suppresses the duplicate within a short window.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "debounced_set_size", + "legendFormat": "debounced", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "avg_over_time(debounced_set_size[$smoothing])", + "legendFormat": "debounced trend", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "debounced" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "debounced trend" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 0, + 5 + ], + "fill": "dot" + } + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + } + ] + } + ] + } + } + } + } + }, + "panel-200": { + "kind": "Panel", + "spec": { + "id": 200, + "title": "Deliveries (last hour)", + "description": "Per-provider delivery rate over the last hour. Linear scale; no smoothing overlay.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "expr": "rate(direct_notifications_total[$__rate_interval]) * 60", + "legendFormat": "Apple (APNS)" + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "expr": "rate(fcm_notifications_total[$__rate_interval]) * 60", + "legendFormat": "Android (FCM)" + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "B", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "expr": "rate(ubports_notifications_total[$__rate_interval]) * 60", + "legendFormat": "UBports" + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "C", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "expr": "rate(webpush_notifications_total[$__rate_interval]) * 60", + "legendFormat": "WebPush" + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "D", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": { + "timeFrom": "now-1h" + } + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "notifications/min", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apple (APNS)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Android (FCM)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "UBports" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "WebPush" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5794F2", + "mode": "fixed" + } + } + ] + } + ] + } + } + } + } + }, + "panel-201": { + "kind": "Panel", + "spec": { + "id": 201, + "title": "Failures (last hour)", + "description": "Real delivery failures over the last hour, per provider. Excludes token cleanup (HTTP 404/410 and APNS dead-token errors). For the full failure breakdown see Notifications: Failures.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "sum by (provider) (rate(notification_failures_total{reason!~\"${cleanup_reasons}\", details!~\"${dead_token_details}\"}[$__rate_interval])) * 60", + "legendFormat": "{{provider}}", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": { + "timeFrom": "now-1h" + } + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "", + "spec": { + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "failures/min", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apple (APNS)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Android (FCM)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "UBports" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF9830", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "WebPush" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5794F2", + "mode": "fixed" + } + } + ] + } + ] + } + } + } + } + }, + "panel-9": { + "kind": "Panel", + "spec": { + "id": 9, + "title": "Notifier process status", + "description": "Notifier process availability.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "up{job=\"notifications\"}", + "instant": false, + "legendFormat": " ", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "state-timeline", + "version": "", + "spec": { + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Down", + "color": "red", + "index": 1 + }, + "1": { + "text": "Up", + "color": "green", + "index": 0 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": null, + "color": "red" + }, + { + "value": 1, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + }, + "custom": { + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + } + }, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-9" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 3, + "width": 24, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-14" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 11, + "width": 12, + "height": 7, + "element": { + "kind": "ElementReference", + "name": "panel-200" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 11, + "width": 12, + "height": 7, + "element": { + "kind": "ElementReference", + "name": "panel-201" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 18, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-11" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 18, + "width": 12, + "height": 8, + "element": { + "kind": "ElementReference", + "name": "panel-10" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 26, + "width": 12, + "height": 6, + "element": { + "kind": "ElementReference", + "name": "panel-16" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 26, + "width": 12, + "height": 6, + "element": { + "kind": "ElementReference", + "name": "panel-12" + } + } + } + ] + } + }, + "links": [ + { + "title": "→ Notifications: Failures", + "type": "dashboards", + "icon": "dashboard", + "tooltip": "", + "url": "", + "tags": [ + "chatmail", + "notifications", + "failures" + ], + "asDropdown": false, + "targetBlank": false, + "includeVars": true, + "keepTime": true + }, + { + "title": "notifiers source", + "type": "link", + "icon": "external link", + "tooltip": "chatmail/notifiers on GitHub", + "url": "https://github.com/chatmail/notifiers", + "tags": [], + "asDropdown": false, + "targetBlank": true, + "includeVars": false, + "keepTime": false + } + ], + "liveNow": false, + "preload": false, + "tags": [ + "chatmail", + "notifications", + "delivery" + ], + "timeSettings": { + "timezone": "browser", + "from": "now-7d", + "to": "now", + "autoRefresh": "5m", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "hideTimepicker": false, + "fiscalYearStartMonth": 0 + }, + "title": "Notifications: Deliveries", + "variables": [ + { + "kind": "CustomVariable", + "spec": { + "name": "smoothing", + "query": "60m,6h,1d,1w", + "current": { + "text": "1d", + "value": "1d" + }, + "options": [ + { + "selected": false, + "text": "60m", + "value": "60m" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": true, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "1w", + "value": "1w" + } + ], + "multi": false, + "includeAll": false, + "label": "Smoothing", + "hide": "dontHide", + "skipUrlSync": false, + "allowCustomValue": true + } + }, + { + "kind": "ConstantVariable", + "spec": { + "name": "cleanup_reasons", + "query": "404|410", + "current": { + "text": "404|410", + "value": "404|410" + }, + "label": "Cleanup reasons", + "hide": "hideVariable", + "skipUrlSync": true + } + }, + { + "kind": "ConstantVariable", + "spec": { + "name": "dead_token_details", + "query": "BadDeviceToken|Unregistered|DeviceTokenNotForTopic", + "current": { + "text": "BadDeviceToken|Unregistered|DeviceTokenNotForTopic", + "value": "BadDeviceToken|Unregistered|DeviceTokenNotForTopic" + }, + "label": "Dead-token APNS details", + "hide": "hideVariable", + "skipUrlSync": true + } + } + ] + } +} diff --git a/contrib/grafana/dashboard-notifications-failures.json b/contrib/grafana/dashboard-notifications-failures.json new file mode 100644 index 0000000..4f471c6 --- /dev/null +++ b/contrib/grafana/dashboard-notifications-failures.json @@ -0,0 +1,1571 @@ +{ + "apiVersion": "dashboard.grafana.app/v2", + "kind": "Dashboard", + "metadata": { + "name": "kslwk1k3bv6pmsC", + "generation": 2, + "creationTimestamp": "2026-04-08T08:02:35Z", + "labels": {}, + "annotations": {} + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "grafana", + "version": "v0", + "spec": {}, + "labels": { + "grafana.app/export-label": "grafana-1" + } + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "builtIn": true + } + } + ], + "cursorSync": "Tooltip", + "description": "Chatmail notifications: delivery failures and token cleanup. See https://github.com/chatmail/notifiers", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "id": 1, + "title": "Notification delivery failures", + "description": "Real delivery failures per hour, grouped by provider, reason, and detail. Excludes token cleanup. The red threshold marks 100 failures/hour.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "sum by (provider, reason, details) (rate(notification_failures_total{provider=~\"$provider\", reason!~\"${cleanup_reasons}\", details!~\"${dead_token_details}\"}[$__rate_interval])) * 3600", + "instant": false, + "legendFormat": "{{provider}} {{reason}} {{details}}", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.0.2", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "transparent" + }, + { + "value": 100, + "color": "red" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "failures/h", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "line" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-10": { + "kind": "Panel", + "spec": { + "id": 10, + "title": "Delivery failures (1h)", + "description": "Real delivery failures in the last hour, excluding token cleanup (HTTP 404/410 and APNS dead-token errors).", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "sum(increase(notification_failures_total{provider=~\"$provider\", reason!~\"${cleanup_reasons}\", details!~\"${dead_token_details}\"}[1h]))", + "instant": true, + "legendFormat": "failures (1h)", + "range": false + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.2", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 100, + "color": "yellow" + }, + { + "value": 500, + "color": "red" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-11": { + "kind": "Panel", + "spec": { + "id": 11, + "title": "Token cleanups (1h)", + "description": "Token invalidations in the last hour. These are not errors: the user uninstalled the app or the token expired, and we should stop sending. APNS 410, FCM 404, plus APNS dead-token error reasons.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "(sum(increase(notification_failures_total{provider=~\"$provider\", reason=~\"${cleanup_reasons}\"}[1h])) or vector(0))\n+\n(sum(increase(notification_failures_total{provider=~\"$provider\", reason!~\"${cleanup_reasons}\", details=~\"${dead_token_details}\"}[1h])) or vector(0))", + "instant": true, + "legendFormat": "cleanups (1h)", + "range": false + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.2", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "blue" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-12": { + "kind": "Panel", + "spec": { + "id": 12, + "title": "Failure ratio by provider (5m)", + "description": "Real failures as a percentage of total send attempts per provider, over a 5-minute window. A small global ratio can mask a single dead provider, so this is split per provider. The query joins failure and success counters via label_replace because the success counters lack a provider label.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "100 * (sum by (provider) (rate(notification_failures_total{provider=~\"$provider\", reason!~\"${cleanup_reasons}\", details!~\"${dead_token_details}\"}[5m])))\n /\n clamp_min(\n sum by (provider) (rate(notification_failures_total{provider=~\"$provider\", reason!~\"${cleanup_reasons}\", details!~\"${dead_token_details}\"}[5m]))\n +\n sum by (provider) (\n label_replace(rate(direct_notifications_total[5m]), \"provider\", \"APNS\", \"\", \"\")\n or\n label_replace(rate(fcm_notifications_total[5m]), \"provider\", \"FCM\", \"\", \"\")\n or\n label_replace(rate(ubports_notifications_total[5m]), \"provider\", \"UBports\", \"\", \"\")\n or\n label_replace(rate(webpush_notifications_total[5m]), \"provider\", \"WebPush\", \"\", \"\")\n ),\n 0.0001\n )", + "instant": false, + "legendFormat": "{{provider}}", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.2", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 10, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 0.5, + "color": "yellow" + }, + { + "value": 2, + "color": "orange" + }, + { + "value": 5, + "color": "red" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-13": { + "kind": "Panel", + "spec": { + "id": 13, + "title": "Config errors (1h)", + "description": "Provider configuration failures in the last hour: APNS no_certificate, FCM no_api_key / api_key_fetch, WebPush no_vapid_key. Any non-zero value means an entire provider channel is down.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "(sum(increase(notification_failures_total{reason=~\"no_certificate|no_api_key|api_key_fetch|no_vapid_key\"}[1h]))) or vector(0)", + "instant": true, + "legendFormat": "config errors (1h)", + "range": false + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.2", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + }, + { + "value": 1, + "color": "red" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-15": { + "kind": "Panel", + "spec": { + "id": 15, + "title": "OpenPGP decryption failures (1d)", + "description": "Trend of OpenPGP token decryption failures, in 1-day buckets. A sustained increase may indicate a key rotation issue or misconfiguration.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "increase(openpgp_decryption_failures_total[1d])", + "instant": false, + "legendFormat": "decryption failures", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.0.2", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "id": 2, + "title": "Token invalidations: cleanup, not failures", + "description": "Routine token invalidations per hour, grouped by provider, reason, and detail. Not real delivery failures: they indicate uninstalls or expired tokens. Useful for spotting uninstall waves.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "sum by (provider, reason, details) (rate(notification_failures_total{provider=~\"$provider\", reason=~\"${cleanup_reasons}\"}[$__rate_interval])) * 3600", + "instant": false, + "legendFormat": "{{provider}} {{reason}} {{details}}", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + }, + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "sum by (provider, reason, details) (rate(notification_failures_total{provider=~\"$provider\", reason!~\"${cleanup_reasons}\", details=~\"${dead_token_details}\"}[$__rate_interval])) * 3600", + "instant": false, + "legendFormat": "{{provider}} {{reason}} {{details}}", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "B", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.0.2", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "blue" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "cleanups/h", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 60, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-20": { + "kind": "Panel", + "spec": { + "id": 20, + "title": "APNS: failures by reason and detail", + "description": "Real APNS failures over the selected time range, broken down by reason and APNS error detail. Excludes token cleanup and dead-token errors.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "sum by (reason, details) (increase(notification_failures_total{provider=\"APNS\", reason!~\"${cleanup_reasons}\", details!~\"${dead_token_details}\"}[$__range]))", + "instant": false, + "legendFormat": "{{reason}} {{details}}", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "piechart", + "version": "13.0.2", + "spec": { + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-21": { + "kind": "Panel", + "spec": { + "id": 21, + "title": "FCM: failures by reason", + "description": "Real FCM failures over the selected time range, broken down by reason. Excludes token cleanup. FCM does not populate the details label.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "sum by (reason) (increase(notification_failures_total{provider=\"FCM\", reason!~\"${cleanup_reasons}\"}[$__range]))", + "instant": false, + "legendFormat": "{{reason}}", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "piechart", + "version": "13.0.2", + "spec": { + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "sort": "desc", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "id": 3, + "title": "Failures by provider (7d trend)", + "description": "Last 7 days of real delivery failures per provider, in hourly buckets. This panel uses a fixed 7-day range regardless of the dashboard time picker.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "sum by (provider) (increase(notification_failures_total{provider=~\"$provider\", reason!~\"${cleanup_reasons}\", details!~\"${dead_token_details}\"}[1h]))", + "instant": false, + "legendFormat": "{{provider}}", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": { + "timeFrom": "now-7d" + } + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "timeseries", + "version": "13.0.2", + "spec": { + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "green" + } + ] + }, + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "failures/h", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + } + } + } + } + }, + "panel-30": { + "kind": "Panel", + "spec": { + "id": 30, + "title": "OpenPGP decrypt fails", + "description": "OpenPGP token decryption failures over the selected time range. Device tokens are encrypted with OpenPGP for privacy; on decryption failure the gateway returns HTTP 410 Gone so the email server removes the token.", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "increase(openpgp_decryption_failures_total[$__range])", + "instant": true, + "legendFormat": "openpgp decrypt fails", + "range": false + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "stat", + "version": "13.0.2", + "spec": { + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "purple" + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + } + } + } + } + }, + "panel-9": { + "kind": "Panel", + "spec": { + "id": 9, + "title": "Notifier process status", + "description": "Notifier process availability", + "links": [], + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "editorMode": "code", + "expr": "up{job=\"notifications\"}", + "instant": false, + "legendFormat": " ", + "range": true + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "refId": "A", + "hidden": false + } + } + ], + "transformations": [], + "queryOptions": {} + } + }, + "vizConfig": { + "kind": "VizConfig", + "group": "state-timeline", + "version": "13.0.2", + "spec": { + "options": { + "alignValue": "left", + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Down", + "color": "red", + "index": 1 + }, + "1": { + "text": "Up", + "color": "green", + "index": 0 + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "value": 0, + "color": "red" + }, + { + "value": 1, + "color": "green" + } + ] + }, + "color": { + "mode": "thresholds" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + } + }, + "overrides": [] + } + } + } + } + } + }, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 0, + "width": 24, + "height": 3, + "element": { + "kind": "ElementReference", + "name": "panel-9" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 3, + "width": 3, + "height": 6, + "element": { + "kind": "ElementReference", + "name": "panel-10" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 3, + "y": 3, + "width": 3, + "height": 6, + "element": { + "kind": "ElementReference", + "name": "panel-12" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 6, + "y": 3, + "width": 3, + "height": 6, + "element": { + "kind": "ElementReference", + "name": "panel-13" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 9, + "y": 3, + "width": 3, + "height": 6, + "element": { + "kind": "ElementReference", + "name": "panel-11" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 3, + "width": 4, + "height": 6, + "element": { + "kind": "ElementReference", + "name": "panel-21" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 16, + "y": 3, + "width": 4, + "height": 6, + "element": { + "kind": "ElementReference", + "name": "panel-20" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 20, + "y": 3, + "width": 4, + "height": 6, + "element": { + "kind": "ElementReference", + "name": "panel-30" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 9, + "width": 12, + "height": 13, + "element": { + "kind": "ElementReference", + "name": "panel-1" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 9, + "width": 12, + "height": 13, + "element": { + "kind": "ElementReference", + "name": "panel-2" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 0, + "y": 22, + "width": 12, + "height": 10, + "element": { + "kind": "ElementReference", + "name": "panel-3" + } + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "x": 12, + "y": 22, + "width": 12, + "height": 10, + "element": { + "kind": "ElementReference", + "name": "panel-15" + } + } + } + ] + } + }, + "links": [ + { + "title": "→ Notifications: Deliveries", + "type": "dashboards", + "icon": "dashboard", + "tooltip": "", + "url": "", + "tags": [ + "chatmail", + "notifications", + "delivery" + ], + "asDropdown": false, + "targetBlank": false, + "includeVars": true, + "keepTime": true + }, + { + "title": "notifiers source", + "type": "link", + "icon": "external link", + "tooltip": "chatmail/notifiers on GitHub", + "url": "https://github.com/chatmail/notifiers", + "tags": [], + "asDropdown": false, + "targetBlank": true, + "includeVars": false, + "keepTime": false + } + ], + "liveNow": false, + "preload": false, + "tags": [ + "chatmail", + "notifications", + "failures" + ], + "timeSettings": { + "timezone": "browser", + "from": "now-24h", + "to": "now", + "autoRefresh": "5m", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "hideTimepicker": false, + "fiscalYearStartMonth": 0 + }, + "title": "Notifications: Failures", + "variables": [ + { + "kind": "QueryVariable", + "spec": { + "name": "provider", + "current": { + "text": "", + "value": "" + }, + "label": "Provider", + "hide": "dontHide", + "refresh": "onTimeRangeChanged", + "skipUrlSync": false, + "query": { + "kind": "DataQuery", + "group": "prometheus", + "version": "v0", + "spec": { + "qryType": 1, + "query": "label_values(notification_failures_total, provider)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "labels": { + "grafana.app/export-label": "prometheus-1" + } + }, + "regex": "", + "regexApplyTo": "value", + "sort": "alphabeticalAsc", + "definition": "label_values(notification_failures_total, provider)", + "options": [], + "multi": true, + "includeAll": true, + "allowCustomValue": true + } + }, + { + "kind": "ConstantVariable", + "spec": { + "name": "cleanup_reasons", + "query": "404|410", + "current": { + "text": "404|410", + "value": "404|410" + }, + "label": "Cleanup reasons", + "hide": "hideVariable", + "skipUrlSync": true + } + }, + { + "kind": "ConstantVariable", + "spec": { + "name": "dead_token_details", + "query": "BadDeviceToken|Unregistered|DeviceTokenNotForTopic", + "current": { + "text": "BadDeviceToken|Unregistered|DeviceTokenNotForTopic", + "value": "BadDeviceToken|Unregistered|DeviceTokenNotForTopic" + }, + "label": "Dead-token APNS details", + "hide": "hideVariable", + "skipUrlSync": true + } + } + ] + } +}