{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "links": [], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 20, "panels": [], "title": "🚨 Critical Health Signals (Executive View)", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "ALERT D: Detects if the Gateway Node is down or network is severed. Must be 1.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "0": { "color": "red", "index": 0, "text": "DOWN" }, "1": { "color": "green", "index": 1, "text": "UP" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 0, "y": 1 }, "id": 2, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "up{job=\"$job\", host_name=~\"$host\"}", "legendFormat": "{{host_name}}", "range": true, "refId": "A" } ], "title": "Gateway Uptime (Heartbeat)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "ALERT E: Tracks if the Gateway is rejecting data at the input because it is out of RAM (OOM Protection).", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 0.1 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 4, "y": 1 }, "id": 4, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (host_name) (rate(otelcol_receiver_refused_metric_points_total{job=\"$job\", host_name=~\"$host\"}[5m]))", "legendFormat": "{{host_name}}", "range": true, "refId": "A" } ], "title": "Input Rejected (Rate)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "ALERT A: The rate of data points dropped because the destination rejected them or timed out.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 30, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 16, "x": 8, "y": 1 }, "id": 6, "options": { "legend": { "calcs": [ "max" ], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (host_name, exporter) (rate(otelcol_exporter_send_failed_metric_points_total{job=\"$job\", host_name=~\"$host\", exporter=~\"$exporter_regex\"}[5m]))", "legendFormat": "{{host_name}} - {{exporter}}", "range": true, "refId": "A" } ], "title": "Data Loss Rate (Send Failures)", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 }, "id": 21, "panels": [], "title": "⚠️ Queue Saturation & Throughput (Performance)", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "ALERT B: If > 80%, the Gateway cannot flush data fast enough.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "#EAB839", "value": 50 }, { "color": "red", "value": 80 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "id": 8, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "max by (exporter) (otelcol_exporter_queue_size{job=\"$job\", host_name=~\"$host\", exporter=~\"$exporter_regex\"} / otelcol_exporter_queue_capacity{job=\"$job\", host_name=~\"$host\", exporter=~\"$exporter_regex\"}) * 100", "legendFormat": "{{exporter}}", "range": true, "refId": "A" } ], "title": "Queue Saturation (%)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "ALERT C: Traffic Check. Should be positive.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "pps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "id": 10, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (exporter) (rate(otelcol_exporter_sent_metric_points_total{job=\"$job\", host_name=~\"$host\", exporter=~\"$exporter_regex\"}[5m]))", "legendFormat": "{{exporter}}", "range": true, "refId": "A" } ], "title": "Throughput (Successfully Sent)", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 16 }, "id": 22, "panels": [], "title": "💻 Resource Utilization (Capacity Planning)", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Shows how many CPU cores the Gateway process is using.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Cores", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 17 }, "id": 12, "options": { "legend": { "calcs": [ "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (host_name) (rate(otelcol_process_cpu_seconds_total{job=\"$job\", host_name=~\"$host\"}[5m]))", "legendFormat": "{{host_name}}", "range": true, "refId": "A" } ], "title": "CPU Usage (Cores)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Physical RAM used. Alert if > 85% of VM limit.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 17 }, "id": 14, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "10.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "max by (host_name) (otelcol_process_memory_rss_bytes{job=\"$job\", host_name=~\"$host\"})", "legendFormat": "{{host_name}}", "range": true, "refId": "A" } ], "title": "RAM Usage (RSS)", "type": "timeseries" } ], "refresh": "1m", "schemaVersion": 38, "style": "dark", "tags": [ "otel", "gateway", "monitoring" ], "templating": { "list": [ { "current": { "selected": false, "text": "Thanos", "value": "Thanos" }, "hide": 0, "includeAll": false, "label": "Datasource", "multi": false, "name": "datasource", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "current": { "selected": false, "text": "cdp-opentelemetry-collector", "value": "cdp-opentelemetry-collector" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(up, job)", "hide": 0, "includeAll": false, "label": "Collector Job", "multi": false, "name": "job", "options": [], "query": { "query": "label_values(up, job)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": { "selected": true, "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(up{job=\"$job\"}, host_name)", "hide": 0, "includeAll": true, "label": "Gateway Host", "multi": true, "name": "host", "options": [], "query": { "query": "label_values(up{job=\"$job\"}, host_name)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": { "selected": false, "text": "prometheusremotewrite/.*", "value": "prometheusremotewrite/.*" }, "hide": 2, "includeAll": false, "label": "Exporter Regex", "multi": false, "name": "exporter_regex", "options": [ { "selected": true, "text": "prometheusremotewrite/.*", "value": "prometheusremotewrite/.*" } ], "query": "prometheusremotewrite/.*", "skipUrlSync": false, "type": "custom" } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "OpenTelemetry Gateway Health", "uid": "otel-gateway-health", "version": 1, "weekStart": "" }