{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__elements": {}, "__requires": [ { "type": "panel", "id": "bargauge", "name": "Bar gauge", "version": "" }, { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "9.5.2" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "datasource", "uid": "grafana" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, "links": [], "liveNow": false, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Show the total amount of messages that needs to be read by selected Kafka consumer group (last measurement)", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "noValue": "N/A", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1000 }, { "color": "red", "value": 10000 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 }, "id": 11, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "9.5.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(clamp_min(kafka_consumergroup_lag{k8sCluster=\"$cluster\", consumergroup=~\"$consumer_group\", namespace=\"$namespace\"}, 0))", "instant": true, "interval": "", "legendFormat": "", "refId": "Noprod" } ], "title": "Instant Total Messages Behind", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Shows the number of messages per second read by selected Kafka consumer group (last measurement)", "fieldConfig": { "defaults": { "color": { "fixedColor": "blue", "mode": "continuous-BlPu" }, "decimals": 0, "mappings": [], "max": 30000, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null }, { "color": "purple", "value": 10000 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 0 }, "id": 6, "interval": "5m", "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "9.5.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(deriv(kafka_consumergroup_current_offset{k8sCluster=\"$cluster\", consumergroup=~\"$consumer_group\", namespace=\"$namespace\"}[$__interval]))", "instant": true, "interval": "", "legendFormat": "{{consumergroup}}", "refId": "Noprod" } ], "title": "Instant consumption rate [msg/s]", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-BlPu" }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "#EAB839", "value": 10000 }, { "color": "red", "value": 100000 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 0 }, "id": 20, "interval": "4m", "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "delta" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "9.5.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(kafka_topic_partition_current_offset{k8sCluster=~\"$cluster\", topic=~\"$consumergroup_subscribed_topics\", namespace=~\"$namespace\"})", "instant": false, "interval": "", "legendFormat": "", "refId": "Noprod" } ], "title": "Published messages", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-BlPu" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "orange", "value": 1000 }, { "color": "yellow", "value": 10000 }, { "color": "green", "value": 100000 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 0 }, "id": 22, "interval": "4m", "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "delta" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "9.5.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(kafka_consumergroup_current_offset{k8sCluster=\"$cluster\", consumergroup=~\"$consumer_group\", namespace=\"$namespace\"})", "instant": false, "interval": "", "legendFormat": "", "refId": "Noprod" } ], "title": "Consumed messages", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "msgs", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 70, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "contact-center-mirror.development" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "mia.pass.realtime-updater" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, "id": 13, "interval": "5m", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.0.3", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(kafka_consumergroup_lag{k8sCluster=\"$cluster\", consumergroup=~\"$consumer_group\", namespace=\"$namespace\"}) by (consumergroup)", "format": "time_series", "instant": false, "interval": "", "legendFormat": "{{consumergroup}}", "refId": "Noprod" } ], "title": "Total Messages Behind", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Shows the number of messages per second read by selected Kafka consumer group", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "msg/s", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 70, "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "contact-center-mirror.development" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "contact-center-mirror.preprod" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "mia.pass.realtime-updater" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "mia.pass.test.realtime-updater" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "mia.realtime.importer.preproduction" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "mia.realtime.importer.test" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "padauto-importer-cdc-preprod-2020-06-24" }, "properties": [ { "id": "color", "value": { "fixedColor": "#50b8da", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "padauto-importer-cdc-test-2020-05-27" }, "properties": [ { "id": "color", "value": { "fixedColor": "#50b8da", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "request-pusher-dev-20200806" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "request-pusher-preprod-20200420" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "single-view-indexer" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 }, "id": 10, "interval": "5m", "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.0.3", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "round(sum(deriv(kafka_consumergroup_current_offset{k8sCluster=\"$cluster\", consumergroup=~\"$consumer_group\", namespace=\"$namespace\"}[$__interval])) by (consumergroup))", "hide": false, "interval": "", "legendFormat": "{{consumergroup}}", "refId": "Noprod" } ], "title": "Consumption Rate", "type": "timeseries" }, { "alert": { "alertRuleTags": { "severity": "warn" }, "conditions": [ { "evaluator": { "params": [ 1 ], "type": "lt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "24h", "now" ] }, "reducer": { "params": [], "type": "sum" }, "type": "query" } ], "executionErrorState": "alerting", "for": "0m", "frequency": "1h", "handler": 1, "message": "No messages arrived in the last 24h for REPLACE_WITH_PROJECTION following topics in namespace REPLACE_WITH_NAMESPACE", "name": "NoMessageIn24HoursREPLACE_WITH_PROJECTION", "noDataState": "alerting", "notifications": [] }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 15 }, "id": 26, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(delta(kafka_topic_partition_current_offset{k8sCluster=~\"$cluster\", topic=~\"$consumergroup_subscribed_topics\"}[24h])) by (topic)", "interval": "", "legendFormat": "{{topic}}", "refId": "A" } ], "thresholds": [ { "colorMode": "critical", "op": "lt", "value": 1, "visible": true } ], "title": "Message on 24h - by topic", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 100, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "{deployment=\"adapter\"}" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 23 }, "id": 24, "interval": "5m", "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.0.3", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum by(deployment)(kube_deployment_status_replicas_available{k8sCluster=\"$cluster\", namespace=~\"$namespace\", deployment=~\".*-realtime-updater$\"})\n# considering a specific consumer group, count the number of topics partitions (1 consumer-group for 1 partition) which offset has changed in the last interval", "instant": false, "interval": "", "legendFormat": "{{deployment}}", "refId": "A" } ], "title": "Total Consumers Deployed", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Show for each topic of selected Kafka consumer group the amount of messages that needs to be read (last measurement)", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "min": 0, "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "#EAB839", "value": 10000 }, { "color": "red", "value": 100000 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 24, "x": 0, "y": 31 }, "id": 2, "options": { "displayMode": "gradient", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "showUnfilled": true, "text": {}, "valueMode": "color" }, "pluginVersion": "9.5.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(clamp_min(kafka_consumergroup_lag{consumergroup=~\"$consumer_group\", namespace=\"$namespace\"}, 0)) by (consumergroup, topic)", "hide": false, "instant": true, "interval": "", "legendFormat": "{{topic}}", "refId": "Noprod" } ], "title": "Instant Consumer Lag", "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "#EAB839", "value": 50 }, { "color": "dark-orange", "value": 100 }, { "color": "dark-red", "value": 1000 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 40 }, "id": 18, "interval": "5m", "options": { "displayMode": "gradient", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": { "calcs": [ "delta" ], "fields": "", "values": false }, "showUnfilled": true, "text": {}, "valueMode": "color" }, "pluginVersion": "9.5.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "sum(kafka_topic_partition_current_offset{k8sCluster=\"$cluster\", topic=~\"$consumergroup_subscribed_topics\"}) by (topic)", "hide": false, "interval": "", "legendFormat": "{{topic}}", "refId": "A" } ], "title": "Published messages per topic", "type": "bargauge" } ], "refresh": "1m", "schemaVersion": 38, "style": "dark", "tags": [ "kafka" ], "templating": { "list": [ { "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(kube_pod_info, k8sCluster)", "hide": 0, "includeAll": false, "multi": false, "name": "cluster", "options": [], "query": { "query": "label_values(kube_pod_info, k8sCluster)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(kube_pod_info{k8sCluster=\"$cluster\"}, namespace)", "hide": 0, "includeAll": false, "multi": false, "name": "namespace", "options": [], "query": { "query": "label_values(kube_pod_info{k8sCluster=\"$cluster\"}, namespace)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "allValue": ".*", "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(kafka_consumergroup_lag{k8sCluster=\"$cluster\", namespace=~\"$namespace\"}, consumergroup)", "hide": 0, "includeAll": true, "multi": true, "name": "consumer_group", "options": [], "query": { "query": "label_values(kafka_consumergroup_lag{k8sCluster=\"$cluster\", namespace=~\"$namespace\"}, consumergroup)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(kafka_consumergroup_current_offset{k8sCluster=\"$cluster\", consumergroup=~\"$consumer_group\"}, topic)", "hide": 0, "includeAll": true, "multi": true, "name": "consumergroup_subscribed_topics", "options": [], "query": { "query": "label_values(kafka_consumergroup_current_offset{k8sCluster=\"$cluster\", consumergroup=~\"$consumer_group\"}, topic)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Kafka Messages Dashboard", "uid": "3397b707-4324-48f5-bf48-05949e3fecb0", "version": 2, "weekStart": "" }