{ "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "gnetId": null, "graphTooltip": 0, "id": 5, "iteration": 1643354997783, "links": [], "panels": [ { "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": { "align": null, "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, "x": 7, "y": 0 }, "id": 347, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "7.3.4", "targets": [ { "alias": "", "bucketAggs": [ { "fake": true, "field": "labels.role.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "hide": false, "metrics": [ { "field": "coun", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "count" } ], "query": "uuid.keyword: $uuid AND metricName: \"nodeRoles\"", "refId": "A", "timeField": "timestamp" }, { "alias": "ready state", "bucketAggs": [ { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": 0, "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"nodeStatus\"", "refId": "B", "timeField": "timestamp" } ], "timeFrom": null, "timeShift": null, "title": "Node count", "type": "stat" }, { "datasource": "$Datasource", "description": "", "fieldConfig": { "defaults": { "custom": {}, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 2, "w": 5, "x": 12, "y": 0 }, "id": 369, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "/^labels\\.version$/", "values": false }, "textMode": "value" }, "pluginVersion": "7.3.4", "targets": [ { "alias": "", "bucketAggs": [], "metrics": [ { "field": "versi", "id": "1", "inlineScript": null, "meta": {}, "settings": { "size": 500 }, "type": "raw_data" } ], "query": "uuid.keyword: $uuid AND metricName: \"clusterVersion\"", "refId": "A", "timeField": "timestamp" } ], "timeFrom": null, "timeShift": null, "title": "OpenShift version", "type": "stat" }, { "datasource": "$Datasource", "description": "", "fieldConfig": { "defaults": { "custom": { "align": null, "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 2, "w": 5, "x": 17, "y": 0 }, "id": 418, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "/^labels\\.kubeversion$/", "values": false }, "textMode": "value" }, "pluginVersion": "7.3.4", "targets": [ { "alias": "", "bucketAggs": [], "hide": true, "metrics": [ { "field": "versi", "id": "1", "inlineScript": null, "meta": {}, "settings": { "size": 500 }, "type": "raw_data" } ], "query": "uuid.keyword: $uuid AND metricName: kubevirtVersion", "refId": "A", "timeField": "timestamp" }, { "bucketAggs": [], "metrics": [ { "field": "select field", "id": "1", "meta": {}, "settings": { "size": 500 }, "type": "raw_data" } ], "query": "metricName: kubevirtVersion", "refId": "B", "timeField": "timestamp" } ], "timeFrom": null, "timeShift": null, "title": "CNV (Kubevirt) version", "type": "stat" }, { "datasource": "$Datasource", "description": "", "fieldConfig": { "defaults": { "custom": {}, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 2, "w": 2, "x": 22, "y": 0 }, "id": 371, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "/^labels\\.cluster_version$/", "values": false }, "textMode": "value" }, "pluginVersion": "7.3.4", "targets": [ { "alias": "", "bucketAggs": [], "metrics": [ { "field": "versi", "id": "1", "inlineScript": null, "meta": {}, "settings": { "size": 500 }, "type": "raw_data" } ], "query": "uuid.keyword: $uuid AND metricName: \"etcdVersion\"", "refId": "A", "timeField": "timestamp" } ], "timeFrom": null, "timeShift": null, "title": "Etcd version", "type": "stat" }, { "datasource": "$Datasource", "description": "", "fieldConfig": { "defaults": { "custom": {}, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 2, "w": 12, "x": 12, "y": 2 }, "id": 32, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "vertical", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "7.3.4", "targets": [ { "alias": "Namespaces", "bucketAggs": [ { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"namespaceCount\"", "refId": "A", "timeField": "timestamp" }, { "alias": "Services", "bucketAggs": [ { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"serviceCount\"", "refId": "B", "timeField": "timestamp" }, { "alias": "Configmap", "bucketAggs": [ { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"configmapCount\"", "refId": "C", "timeField": "timestamp" }, { "alias": "Secret", "bucketAggs": [ { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"secretCount\"", "refId": "D", "timeField": "timestamp" } ], "timeFrom": null, "timeShift": null, "title": "", "type": "stat" }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, "id": 500, "panels": [], "title": "CNV (Kubevirt) VM/VMI", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "count(container_cpu_usage_seconds_total{pod=~\".*virt-launcher.*\", container=\"compute\", node!~\".*master.*\", service=\"kubelet\"}) by (node) > 0", "fieldConfig": { "defaults": { "custom": { "align": null, "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "fill": 9, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 }, "hiddenSeries": false, "id": 519, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": false, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { "bucketAggs": [ { "fake": true, "field": "labels.phase.keyword", "id": "3", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "hide": false, "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: vmiStatusCount", "refId": "C", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "VMIPod/Node Count and Balance ", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(rate(kubevirt_vmi_phase_transition_time_from_creation_seconds_count{}[5m])) by (phase) > 0", "fieldConfig": { "defaults": { "custom": { "align": null, "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 }, "hiddenSeries": false, "id": 520, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.phase.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.phase.keyword", "id": "3", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: vmiStatusRate", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "VM Phase Transition Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "count(container_cpu_usage_seconds_total{pod=~\".*virt-launcher.*\", container=\"compute\", node!~\".*master.*\", service=\"kubelet\"}) by (node) > 0", "fieldConfig": { "defaults": { "custom": { "align": null, "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "fill": 9, "fillGradient": 0, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 13 }, "hiddenSeries": false, "id": 522, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": false, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { "bucketAggs": [ { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "hide": false, "metrics": [ { "field": "value", "id": "1", "meta": { "avg": true, "count": false, "max": true, "min": true, "std_deviation": true, "std_deviation_bounds_lower": false, "std_deviation_bounds_upper": false, "sum": true }, "settings": {}, "type": "extended_stats" } ], "query": "uuid: $uuid AND metricName.keyword: vmiNodeCount AND NOT labels.node.keyword: master*", "refId": "C", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "VMIPod/Node Count and Balance ", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "If the VMI was created by a VM object, the VM creation time is the baseline timestamp, otherwise the VMI creation time is used as the baseline timestamp.", "fieldConfig": { "defaults": { "custom": { "align": null, "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 13 }, "hiddenSeries": false, "id": 521, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 0.5, "points": true, "renderer": "flot", "seriesOverrides": [ { "alias": "VMReady", "color": "#96D98D" }, { "alias": "VMIReady", "color": "#FA6400" }, { "alias": "VMIScheduled", "color": "#FFB357" }, { "alias": "PodReady", "color": "#8F3BB8" }, { "alias": "PodContainersReady", "color": "#1F60C4" }, { "alias": "PodInitialized", "color": "#3274D9" }, { "alias": "VMIScheduling", "color": "#FFCB7D" }, { "alias": "PodCreated", "color": "#8AB8FF" }, { "alias": "PodScheduled", "color": "#C0D8FF" }, { "alias": "VMIPending", "color": "#FF7383" }, { "alias": "VMICreated", "color": "#FFA6B0" }, { "alias": "VMIRunning", "color": "#FF780A" } ], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { "alias": "VMReady", "bucketAggs": [ { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": 0, "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "vmReadyLatency", "id": "1", "meta": {}, "settings": {}, "type": "max" } ], "query": "uuid: $uuid AND metricName.keyword: vmiLatencyMeasurement AND vmReadyLatency > 0", "refId": "A", "timeField": "timestamp" }, { "alias": "VMIReady", "bucketAggs": [ { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": 0, "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "vmiReadyLatency", "id": "1", "meta": {}, "settings": {}, "type": "max" } ], "refId": "B", "timeField": "timestamp" }, { "alias": "VMIScheduled", "bucketAggs": [ { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": 0, "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "vmiScheduledLatency", "id": "1", "meta": {}, "settings": {}, "type": "max" } ], "refId": "C", "timeField": "timestamp" }, { "alias": "VMIScheduling", "bucketAggs": [ { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": 0, "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "vmiSchedulingLatency", "id": "1", "meta": {}, "settings": {}, "type": "max" } ], "refId": "D", "timeField": "timestamp" }, { "alias": "PodReady", "bucketAggs": [ { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": 0, "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "podReadyLatency", "id": "1", "meta": {}, "settings": {}, "type": "max" } ], "refId": "E", "timeField": "timestamp" }, { "alias": "PodContainersReady", "bucketAggs": [ { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": 0, "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "podContainersReadyLatency", "id": "1", "meta": {}, "settings": {}, "type": "max" } ], "refId": "F", "timeField": "timestamp" }, { "alias": "PodInitialized", "bucketAggs": [ { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": 0, "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "podInitializedLatency", "id": "1", "meta": {}, "settings": {}, "type": "max" } ], "refId": "G", "timeField": "timestamp" }, { "alias": "PodScheduled", "bucketAggs": [ { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": 0, "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "podScheduledLatency", "id": "1", "meta": {}, "settings": {}, "type": "max" } ], "refId": "H", "timeField": "timestamp" }, { "alias": "PodCreated", "bucketAggs": [ { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": 0, "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "podCreatedLatency", "id": "1", "meta": {}, "settings": {}, "type": "max" } ], "refId": "I", "timeField": "timestamp" }, { "alias": "VMIPending", "bucketAggs": [ { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": 0, "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "vmiPendingLatency", "id": "1", "meta": {}, "settings": {}, "type": "max" } ], "refId": "J", "timeField": "timestamp" }, { "alias": "VMICreated", "bucketAggs": [ { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": 0, "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "vmiCreatedLatency", "id": "1", "meta": {}, "settings": {}, "type": "max" } ], "refId": "K", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "VMs Creation Latency Breakdown", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "If the VMI was created by a VM object, the VM creation time is the baseline timestamp, otherwise the VMI creation time is used as the baseline timestamp.", "fieldConfig": { "defaults": { "custom": { "align": null, "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 23 }, "hiddenSeries": false, "id": 486, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 0.5, "points": true, "renderer": "flot", "seriesOverrides": [ { "alias": "VMReady", "color": "#96D98D" }, { "alias": "VMIReady", "color": "#FA6400" }, { "alias": "VMIScheduled", "color": "#FFB357" }, { "alias": "PodReady", "color": "#8F3BB8" }, { "alias": "PodContainersReady", "color": "#1F60C4" }, { "alias": "PodInitialized", "color": "#3274D9" }, { "alias": "VMIScheduling", "color": "#FFCB7D" }, { "alias": "PodCreated", "color": "#8AB8FF" }, { "alias": "PodScheduled", "color": "#C0D8FF" }, { "alias": "VMIPending", "color": "#FF7383" }, { "alias": "VMICreated", "color": "#FFA6B0" }, { "alias": "VMIRunning", "color": "#FF780A" } ], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { "alias": "{{term quantileName.keyword}}", "bucketAggs": [ { "fake": true, "field": "quantileName.keyword", "id": "14", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "13", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "5m", "min_doc_count": "0", "offset": "", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "P99", "id": "13", "meta": {}, "settings": {}, "type": "max" } ], "query": "uuid: $uuid AND metricName.keyword: vmiLatencyQuantilesMeasurement", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "VM Creation P99 Latency Breakdown", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "If the VMI was created by a VM object, the VM creation time is the baseline timestamp, otherwise the VMI creation time is used as the baseline timestamp.", "fieldConfig": { "defaults": { "custom": { "align": null, "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 23 }, "hiddenSeries": false, "id": 485, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 0.5, "points": true, "renderer": "flot", "seriesOverrides": [ { "alias": "VMReady", "color": "#96D98D" }, { "alias": "VMIReady", "color": "#FA6400" }, { "alias": "VMIScheduled", "color": "#FFB357" }, { "alias": "PodReady", "color": "#8F3BB8" }, { "alias": "PodContainersReady", "color": "#1F60C4" }, { "alias": "PodInitialized", "color": "#3274D9" }, { "alias": "VMIScheduling", "color": "#FFCB7D" }, { "alias": "PodCreated", "color": "#8AB8FF" }, { "alias": "PodScheduled", "color": "#C0D8FF" }, { "alias": "VMIPending", "color": "#FF7383" }, { "alias": "VMICreated", "color": "#FFA6B0" }, { "alias": "VMIRunning", "color": "#FF780A" } ], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { "alias": "{{term quantileName.keyword}}", "bucketAggs": [ { "fake": true, "field": "quantileName.keyword", "id": "14", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "13", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "5m", "min_doc_count": "0", "offset": "", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "P50", "id": "13", "meta": {}, "settings": {}, "type": "max" } ], "query": "uuid: $uuid AND metricName.keyword: vmiLatencyQuantilesMeasurement", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "VM Creation P50 Latency Breakdown", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 33 }, "id": 464, "panels": [], "title": "CNV (Kubevirt) API", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{group=\"kubevirt.io\", verb=~\"LIST|GET\"}[2m])) by (verb,resource,subresource,instance,le)) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 34 }, "hiddenSeries": false, "id": 454, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.resource.keyword}}: {{labels.verb.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.verb.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "0" }, "type": "terms" }, { "fake": true, "field": "labels.resource.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "max" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtAPI99thReadRequestLatency", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s APIServer P99 Response Latency for Read Requests", "tooltip": { "shared": true, "sort": 1, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{group=\"kubevirt.io\", verb=~\"POST|PUT|PATCH|DELETE\"}[2m])) by (verb,resource,subresource,instance,le)) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 34 }, "hiddenSeries": false, "id": 455, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.resource.keyword}}: {{labels.verb.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.verb.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "0" }, "type": "terms" }, { "fake": true, "field": "labels.resource.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtAPI99thWriteRequestLatency", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s APIServer P99 Response Latency for Write Requests", "tooltip": { "shared": true, "sort": 1, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(increase(apiserver_request_total{group=~\".*kubevirt.*\"}[5m])) by (verb,group,resource,subresource,code) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 42 }, "hiddenSeries": false, "id": 495, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.resource.keyword}}: {{labels.verb.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.verb.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "0" }, "type": "terms" }, { "fake": true, "field": "labels.resource.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtAPIRequestCount", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s APIServer Response Count", "tooltip": { "shared": true, "sort": 1, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(irate(apiserver_request_total{group=~\".*kubevirt.*\"}[2m])) by (verb,group,resource,subresource,code) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 42 }, "hiddenSeries": false, "id": 496, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.resource.keyword}}: {{labels.verb.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.verb.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "0" }, "type": "terms" }, { "fake": true, "field": "labels.resource.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtAPIRequestRate", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s APIServer Response Rate", "tooltip": { "shared": true, "sort": 1, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{url=~\".*kubevirt.io.*\", verb=~\"LIST|GET\"}[2m])) by (verb,le,url)) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 9, "w": 13, "x": 0, "y": 50 }, "hiddenSeries": false, "id": 456, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.verb.keyword}}: {{labels.url.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.verb.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.url.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtRestClient99thReadRequestLatency", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "P99 kubevirt Rest Client Read Request Latency", "tooltip": { "shared": true, "sort": 1, "value_type": "individual" }, "transformations": [ { "id": "organize", "options": {} } ], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(irate(rest_client_requests_total{pod=~\"virt-controller.*|virt-handler.*|virt-operator.*|virt-api.*|vm.*|hco.*|kubevirt.*\"}[2m])) by (code, container, job, verb, method) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 9, "w": 11, "x": 13, "y": 50 }, "hiddenSeries": false, "id": 458, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.container.keyword}}: {{labels.method.keyword}} - {{labels.code.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.container.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.method.keyword", "id": "6", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "fake": true, "field": "labels.code.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "sum" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtRestClientRequestRate AND ( labels.method.keyword:\"LIST\" OR labels.method.keyword:\"GET\" OR labels.method.keyword: \"WATCH\")", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "kubevirt Rest Client Read Request Rate", "tooltip": { "shared": true, "sort": 1, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{url=~\".*kubevirt.io.*\", verb=~\"POST|PUT|PATCH|DELETE\"}[2m])) by (verb,le,url)) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 9, "w": 13, "x": 0, "y": 59 }, "hiddenSeries": false, "id": 460, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.verb.keyword}}: {{labels.url.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.verb.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.url.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtRestClient99thWriteRequestLatency", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "P99 kubevirt Rest Client Write Request Latency", "tooltip": { "shared": true, "sort": 1, "value_type": "individual" }, "transformations": [], "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(irate(rest_client_requests_total{pod=~\"virt-controller.*|virt-handler.*|virt-operator.*|virt-api.*|vm.*|hco.*|kubevirt.*\"}[2m])) by (code, container, job, verb, method) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 9, "w": 11, "x": 13, "y": 59 }, "hiddenSeries": false, "id": 514, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.container.keyword}}: {{labels.method.keyword}} - {{labels.code.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.container.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.method.keyword", "id": "6", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "fake": true, "field": "labels.code.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "sum" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtRestClientRequestRate AND ( labels.method.keyword:\"POST\" OR labels.method.keyword:\"PUT\" OR labels.method.keyword: \"PATCH\" OR labels.method.keyword: \"DELETE\")", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "kubevirt Rest Client Write Request Rate", "tooltip": { "shared": true, "sort": 1, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(increase(rest_client_requests_total{pod=~\"virt-controller.*|virt-handler.*|virt-operator.*|virt-api.*|vm.*|hco.*|kubevirt.*\"}[5m])) by (code, container, job, verb, method) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 68 }, "hiddenSeries": false, "id": 515, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.container.keyword}}: {{labels.method.keyword}} - {{labels.code.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.container.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.method.keyword", "id": "6", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "fake": true, "field": "labels.code.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "sum" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtRestClientRequestCount AND ( labels.method.keyword:\"LIST\" OR labels.method.keyword:\"GET\" OR labels.method.keyword: \"WATCH\")", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "kubevirt Rest Client Read Request Count", "tooltip": { "shared": true, "sort": 1, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(increase(rest_client_requests_total{pod=~\"virt-controller.*|virt-handler.*|virt-operator.*|virt-api.*|vm.*|hco.*|kubevirt.*\"}[5m])) by (code, container, job, verb, method) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 68 }, "hiddenSeries": false, "id": 516, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.container.keyword}}: {{labels.method.keyword}} - {{labels.code.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.container.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.method.keyword", "id": "6", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "fake": true, "field": "labels.code.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "sum" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtRestClientRequestCount AND ( labels.method.keyword:\"POST\" OR labels.method.keyword:\"PUT\" OR labels.method.keyword: \"PATCH\" OR labels.method.keyword: \"DELETE\")", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "kubevirt Rest Client Write Request Count", "tooltip": { "shared": true, "sort": 1, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.95, sum(rate(rest_client_rate_limiter_duration_seconds_bucket{container!=\"\"}[5m])) by (container, service, le)) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 77 }, "hiddenSeries": false, "id": 459, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.container.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.container.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtRateLimiter95thDuration", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "kubevirt Rate Limiter P95 Duration", "tooltip": { "shared": true, "sort": 1, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 85 }, "id": 462, "panels": [], "title": "CNV (Kubevirt) WorkQueue", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(rate(kubevirt_workqueue_adds_total{job=~\".*kubevirt.*\"}[1m])) by (instance, name) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 86 }, "hiddenSeries": false, "id": 467, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.name.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.name.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtWorkQueueAddRate", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "kubevirt Workqueue Add Rate", "tooltip": { "shared": true, "sort": 1, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(rate(kubevirt_workqueue_adds_total{job=~\".*kubevirt.*\"}[1m])) by (instance, name) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 86 }, "hiddenSeries": false, "id": 473, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.name.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.name.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtWorkQueueRetryRate", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "kubevirt Workqueue Retry Rate", "tooltip": { "shared": true, "sort": 1, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, sum(rate(kubevirt_workqueue_queue_duration_seconds_bucket{job=~\".*kubevirt.*\"}[1m])) by (instance, name, le)) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 94 }, "hiddenSeries": false, "id": 468, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.name.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.name.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtWorkQueue99thQueueDuration", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "kubevirt Workqueue P99 Wait Duration", "tooltip": { "shared": true, "sort": 1, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, sum(rate(kubevirt_workqueue_work_duration_seconds_bucket{job=~\".*kubevirt.*\"}[1m])) by (instance, name, le)) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 94 }, "hiddenSeries": false, "id": 470, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.name.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.name.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtWorkQueue99thWorkDuration", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "kubevirt Workqueue P99 Work Duration", "tooltip": { "shared": true, "sort": 1, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "kubevirt_workqueue_unfinished_work_seconds{} > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 102 }, "hiddenSeries": false, "id": 471, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.name.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.name.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtWorkQueueUnfinishedWork", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "kubevirt Workqueue Unfinished Work", "tooltip": { "shared": true, "sort": 1, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "kubevirt_workqueue_longest_running_processor_seconds{} > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 102 }, "hiddenSeries": false, "id": 472, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.name.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.name.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtWorkQueueLongestRunningProcessor", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "kubevirt Workqueue Longest Running Processor", "tooltip": { "shared": true, "sort": 1, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(rate(kubevirt_workqueue_adds_total{job=~\".*kubevirt.*\"}[1m])) by (instance, name) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 110 }, "hiddenSeries": false, "id": 469, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.name.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.name.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtWorkQueueDepth", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "kubevirt Workqueue Depth", "tooltip": { "shared": true, "sort": 1, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 118 }, "id": 440, "panels": [], "title": "CNV (Kubevirt) Resource Usage", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(irate(process_cpu_seconds_total{container!=\"\", job=~\".*kubevirt.*\"}[5m]) * 100) by (container, instance) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 119 }, "hiddenSeries": false, "id": 474, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "bucketAggs": [ { "fake": true, "field": "labels.container.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "20" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtCPU", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "kubevirt Process CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percent", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 119 }, "hiddenSeries": false, "id": 119, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "bucketAggs": [ { "fake": true, "field": "labels.container.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "20" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubevirtMemory", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "kubevirt Process RSS Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 127 }, "id": 438, "panels": [], "title": "Etcd", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[2m])) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 128 }, "hiddenSeries": false, "id": 441, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "", "bucketAggs": [ { "fake": true, "field": "labels.pod.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: 99thEtcdDiskWalFsyncDurationSeconds", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "etcd P99 disk WAL fsync latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[2m])) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 128 }, "hiddenSeries": false, "id": 447, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "", "bucketAggs": [ { "fake": true, "field": "labels.pod.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: 99thEtcdDiskWalFsyncDurationSeconds", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "etcd P99 disk backend commit latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 136 }, "hiddenSeries": false, "id": 448, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "", "bucketAggs": [ { "fake": true, "field": "labels.pod.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: 99thEtcdRoundTripTimeSeconds", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "etcd P99 network peer roundtrip time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99,sum(rate(etcd_request_duration_seconds_bucket[2m])) by (le,operation,apiserver)) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 136 }, "hiddenSeries": false, "id": 449, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.apiserver.keyword}}: {{labels.operation.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.operation.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.apiserver.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: P99APIEtcdRequestLatency", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Etcd P99 Request Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "etcd_mvcc_db_total_size_in_bytes > 0\netcd_mvcc_db_total_size_in_use_in_bytes > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 144 }, "hiddenSeries": false, "id": 450, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/.*Logical.*/", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "Physical size: {{labels.pod.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.pod.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: etcdDBPhysicalSizeBytes", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" }, { "alias": "Logical size: {{labels.pod.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.pod.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: etcdDBLogicalSizeBytes", "refId": "B", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Etcd DB size", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "decbytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "decbytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(rate(etcd_object_counts{}[5m])) by (resource) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 144 }, "hiddenSeries": false, "id": 451, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{term labels.resource.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.resource.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "15" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: etcdObjectCount", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Etcd object count", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {}, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 152 }, "hiddenSeries": false, "id": 314, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "Etcd leader changes", "bucketAggs": [ { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: etcdLeaderChangesRate", "refId": "A", "timeField": "timestamp" }, { "alias": "Current leader: {{term labels.pod.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.pod.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: etcdServerIsLeader", "refId": "B", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Etcd leader", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(rate(etcd_object_counts{}[5m])) by (resource) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 152 }, "hiddenSeries": false, "id": 490, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{term labels.resource.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.resource.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "15" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: etcdObjectRate", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Etcd object count rate", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "cps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 160 }, "id": 436, "panels": [], "title": "Golang metrics", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "go_memstats_heap_alloc_bytes{job=~\"apiserver|api|etcd|kubevirt.*\"} > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 161 }, "hiddenSeries": false, "id": 443, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.container.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.container.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"goHeapAllocBytes\"", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Go Heap Alloc Bytes", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "decbytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "go_memstats_heap_inuse_bytes{job=~\"apiserver|api|etcd|kubevirt.*\"} > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 161 }, "hiddenSeries": false, "id": 444, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.container.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.container.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"goHeapInuseBytes\"", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Go Heap Inuse Bytes", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "decbytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(rate(go_gc_duration_seconds_sum{job=~\"apiserver|api|etcd|kubevirt.*\"}[5m])) by (container, instance) / sum(rate(go_gc_duration_seconds_count{job=~\"apiserver|api|etcd|kubevirt.*\"}[5m])) by (container, instance) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 169 }, "hiddenSeries": false, "id": 445, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.container.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.container.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"goGCDurationRate\"", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Go GC Duration Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "go_goroutines{job=~\"apiserver|api|etcd|kubevirt.*\"} > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 169 }, "hiddenSeries": false, "id": 446, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.container.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.container.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"goRoutinesCount\"", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Go Routines Count", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 177 }, "id": 420, "panels": [], "title": "K8s API", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{}[5m])) by (le, pod))", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 178 }, "hiddenSeries": false, "id": 497, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.pod.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.pod.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "sum" } ], "query": "uuid: $uuid AND metricName.keyword: 99thSchedulingDuration", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Scheduling P99 Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{}[5m])) by (le, pod))", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 178 }, "hiddenSeries": false, "id": 498, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.pod.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.pod.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "sum" } ], "query": "uuid: $uuid AND metricName.keyword: 99thSchedulingVolumeDuration", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Scheduling Volume P99 Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(irate(workqueue_adds_total{}[2m])) by (service, name) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 186 }, "hiddenSeries": false, "id": 501, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.service.keyword}}: {{labels.name.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.name.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "15" }, "type": "terms" }, { "fake": true, "field": "labels.service.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "sum" } ], "query": "uuid: $uuid AND metricName.keyword: workQueueAddRate", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s WorkQueue Add Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{}[5m])) by (service, name, le)) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 194 }, "hiddenSeries": false, "id": 503, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.service.keyword}}: {{labels.name.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.name.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "15" }, "type": "terms" }, { "fake": true, "field": "labels.service.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "sum" } ], "query": "uuid: $uuid AND metricName.keyword: workQueue99thQueueDuration", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s WorkQueue P99 Wait Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(irate(workqueue_adds_total{}[2m])) by (service, name) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 194 }, "hiddenSeries": false, "id": 502, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.service.keyword}}: {{labels.name.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.name.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "15" }, "type": "terms" }, { "fake": true, "field": "labels.service.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "sum" } ], "query": "uuid: $uuid AND metricName.keyword: workQueueDepth", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s WorkQueue Depth", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(irate(workqueue_retries_total{}[5m])) by (service, name) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 202 }, "hiddenSeries": false, "id": 507, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.service.keyword}}: {{labels.name.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.name.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "15" }, "type": "terms" }, { "fake": true, "field": "labels.service.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "sum" } ], "query": "uuid: $uuid AND metricName.keyword: workQueueRetryRate", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s WorkQueue Retry Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, sum(rate(workqueue_work_duration_seconds_bucket{}[1m])) by (service, name, le)) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 202 }, "hiddenSeries": false, "id": 504, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.service.keyword}}: {{labels.name.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.name.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "15" }, "type": "terms" }, { "fake": true, "field": "labels.service.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "sum" } ], "query": "uuid: $uuid AND metricName.keyword: workQueue99thWorkDuration", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s WorkQueue P99 Work Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(workqueue_unfinished_work_seconds) by (service, name) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 210 }, "hiddenSeries": false, "id": 505, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.service.keyword}}: {{labels.name.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.name.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "15" }, "type": "terms" }, { "fake": true, "field": "labels.service.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "sum" } ], "query": "uuid: $uuid AND metricName.keyword: workQueueUnfinishedWork", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s WorkQueue Unfinished Work", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(workqueue_longest_running_processor_seconds{}) by (service, name) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 210 }, "hiddenSeries": false, "id": 506, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.service.keyword}}: {{labels.name.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.name.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "15" }, "type": "terms" }, { "fake": true, "field": "labels.service.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "sum" } ], "query": "uuid: $uuid AND metricName.keyword: workQueueLongestRunningProcessor", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s WorkQueue Longest Running Processor", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{group=\"kubevirt\", verb=~\"LIST|GET\"}[2m])) by (verb,resource,subresource,instance,le)) > 0", "fieldConfig": { "defaults": { "custom": { "align": null, "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 218 }, "hiddenSeries": false, "id": 424, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.resource.keyword}}: {{labels.verb.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.resource.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.verb.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "hide": false, "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "max" } ], "query": "uuid: $uuid AND metricName.keyword: API99thReadRequestLatency", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s APIServer P99 Read Request Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(irate(apiserver_request_total{}[2m])) by (verb,resource,subresource,apiserver,component,code) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 218 }, "hiddenSeries": false, "id": 427, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.resource.keyword}}: {{labels.verb.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.resource.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.verb.keyword", "id": "3", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: APIRequestRate AND ( labels.verb.keyword:\"LIST\" OR labels.verb.keyword:\"GET\" OR labels.verb.keyword: \"WATCH\")", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s APIServer Read Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{apiserver=\"kube-apiserver\", verb=~\"POST|PUT|PATCH|DELETE\", subresource!=\"log\"}[2m])) by (verb,resource,subresource,instance,le)) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 226 }, "hiddenSeries": false, "id": 426, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.resource.keyword}}: {{labels.verb.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.resource.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.verb.keyword", "id": "3", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: API99thWriteRequestLatency", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s APIServer P99 Write Request Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(irate(apiserver_request_total{}[2m])) by (verb,resource,subresource,apiserver,component,code) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 226 }, "hiddenSeries": false, "id": 517, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.resource.keyword}}: {{labels.verb.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.resource.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.verb.keyword", "id": "3", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: APIRequestRate AND ( labels.verb.keyword:\"POST\" OR labels.verb.keyword:\"PUT\" OR labels.verb.keyword: \"PATCH\" OR labels.verb.keyword: \"DELETE\")", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s APIServer Write Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(increase(apiserver_request_total{}[5m])) by (verb,resource,subresource,apiserver,component,code) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 234 }, "hiddenSeries": false, "id": 428, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.resource.keyword}}: {{labels.verb.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.resource.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.verb.keyword", "id": "3", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: APIRequestCount AND ( labels.verb.keyword:\"LIST\" OR labels.verb.keyword:\"GET\" OR labels.verb.keyword: \"WATCH\")", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s APIServer Read Request Count", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(increase(apiserver_request_total{}[5m])) by (verb,resource,subresource,apiserver,component,code) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 234 }, "hiddenSeries": false, "id": 518, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.resource.keyword}}: {{labels.verb.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.resource.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.verb.keyword", "id": "3", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: APIRequestCount AND ( labels.verb.keyword:\"POST\" OR labels.verb.keyword:\"PUT\" OR labels.verb.keyword: \"PATCH\" OR labels.verb.keyword: \"DELETE\")", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s APIServer Write Request Count", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(apiserver_current_inflight_requests{}) by (request_kind) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 242 }, "hiddenSeries": false, "id": 429, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "", "bucketAggs": [ { "fake": true, "field": "labels.request_kind.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: APIInflightRequests", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s APIServer Inflight Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(rate(apiserver_flowcontrol_dispatched_requests_total[2m])) by (flow_schema,priority_level) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 242 }, "hiddenSeries": false, "id": 452, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.flow_schema.keyword}}: {{labels.priority_level.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.priority_level.keyword", "id": "5", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.flow_schema.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: flowControlRate", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s APIServer Flow Control Dispatched Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "reqps", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_wait_duration_seconds_bucket{}[5m])) by (le, flow_schema, priority_level)) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 251 }, "hiddenSeries": false, "id": 489, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.flow_schema.keyword}}: {{labels.priority_level.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.priority_level.keyword", "id": "5", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.flow_schema.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: flowControl99thRequestWait", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s APIServer P99 Flow Control Request Wait Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, sum(rate(apiserver_flowcontrol_request_execution_seconds_bucket{}[5m])) by (le, flow_schema, priority_level)) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 251 }, "hiddenSeries": false, "id": 488, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.flow_schema.keyword}}: {{labels.priority_level.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.priority_level.keyword", "id": "5", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.flow_schema.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: flowControl99thRequestExecution", "queryType": "randomWalk", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "k8s APIServer P99 Flow Control Request Execution Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": "$Datasource", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 260 }, "id": 417, "panels": [], "title": "Monitoring stack", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 261 }, "hiddenSeries": false, "id": 413, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.pod.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.pod.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.container.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: containerCPU-Prometheus", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Prometheus CPU usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percent", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 261 }, "hiddenSeries": false, "id": 415, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.pod.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.pod.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.container.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: containerMemory-Prometheus", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Prometheus RSS Memory usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 269 }, "hiddenSeries": false, "id": 508, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.pod.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.pod.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: prometheus-tsdb-storage", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Prometheus TSDB storage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": "$Datasource", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 277 }, "id": 21, "panels": [], "title": "Cluster status", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 278 }, "hiddenSeries": false, "id": 31, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "", "bucketAggs": [ { "fake": true, "field": "labels.condition.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"nodeStatus\"", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Node status summary", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 278 }, "hiddenSeries": false, "id": 30, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "", "bucketAggs": [ { "fake": true, "field": "labels.phase.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"podStatusCount\"", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Pod status summary", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": "$Datasource", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 286 }, "id": 121, "panels": [], "title": "Cluster Kubelet & CRI-O", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 287 }, "hiddenSeries": false, "id": 116, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "bucketAggs": [ { "fake": true, "field": "labels.node.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND (metricName.keyword: top3KubeletCPU OR metricName.keyword: kubeletCPU)", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Top 5 Kubelet process CPU usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percent", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 287 }, "hiddenSeries": false, "id": 475, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "bucketAggs": [ { "fake": true, "field": "labels.node.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND (metricName.keyword: top3KubeletMemory OR metricName.keyword: kubeletMemory)", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Top 5 Kubelet process RSS memory usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_bucket{}[5m])) by (instance, le)) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 295 }, "hiddenSeries": false, "id": 509, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.instance.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubelet99thPodStartDuration", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Kubelet P99 Pod Start Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(rate(kubelet_pod_start_duration_seconds_count{}[5m])) by (node) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 295 }, "hiddenSeries": false, "id": 510, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.node.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.node.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubeletPodStartRate", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Kubelet P99 Pod Start Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{}[5m])) by (node, le, operation_type)) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 303 }, "hiddenSeries": false, "id": 511, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.node.keyword}}: {{labels.operation_type.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.operation_type.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.node.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubelet99thPodWorkerDuration", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Kubelet P99 Pod Worker Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(rate(storage_operation_errors_total{metrics_path=\"/metrics\"}[5m])) by (instance, node, operation_name, volume_plugin) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 303 }, "hiddenSeries": false, "id": 431, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.node.keyword}}: {{labels.volume_plugin.keyword}} - {{labels.operation_name.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.node.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "fake": true, "field": "labels.volume_plugin.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "fake": true, "field": "labels.operation_name.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND (metricName.keyword: kubeletStorageOperationErrorsCount OR metricName.keyword: kubeletStorageOperationErrorsRate)", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Top Kubelet Storage Operation Error Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(rate(storage_operation_duration_seconds_count{metrics_path=\"/metrics\"}[5m])) by (instance, operation_name, volume_plugin) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 311 }, "hiddenSeries": false, "id": 430, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.node.keyword}}: {{labels.volume_plugin.keyword}} - {{labels.operation_name.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.node.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "fake": true, "field": "labels.volume_plugin.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "fake": true, "field": "labels.operation_name.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND (metricName.keyword: kubeletStorageOperationCount OR metricName.keyword: kubeletStorageOperationRate)", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Top Kubelet Storage Operations Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{metrics_path=\"/metrics\"}[5m])) by (le, node, operation_name, volume_plugin)) > 0.01", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 311 }, "hiddenSeries": false, "id": 432, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.node.keyword}}: {{labels.volume_plugin.keyword}} - {{labels.operation_name.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.node.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "fake": true, "field": "labels.volume_plugin.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "fake": true, "field": "labels.operation_name.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName.keyword: kubeletStorageOperation99thDurationSeconds", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Top Kubelet Storage Operation Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(irate(process_cpu_seconds_total{service=\"kubelet\",job=\"crio\"}[2m]) * 100) by (node) and on (node) kube_node_role{role=\"worker\"} > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 321 }, "hiddenSeries": false, "id": 117, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "bucketAggs": [ { "fake": true, "field": "labels.node.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND (metricName.keyword: top3CrioCPU OR metricName.keyword: crioCPU)", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Top 5 CRI-O process CPU usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percent", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(process_resident_memory_bytes{service=\"kubelet\",job=\"crio\"}) by (node) and on (node) kube_node_role{role=\"worker\"} > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 321 }, "hiddenSeries": false, "id": 118, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "bucketAggs": [ { "fake": true, "field": "labels.node.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND (metricName.keyword: top3CrioMemory OR metricName.keyword: crioMemory)", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Top 5 CRI-O process RSS memory usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 11, "w": 12, "x": 0, "y": 329 }, "hiddenSeries": false, "id": 491, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.container.keyword}} - {{labels.job.keyword}} : {{labels.instance.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.container.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "fake": true, "field": "labels.instance.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "fake": true, "field": "labels.job.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "sum" } ], "query": "uuid: $uuid AND metricName.keyword: processCPU", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Top 5 process CPU usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percent", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 11, "w": 12, "x": 12, "y": 329 }, "hiddenSeries": false, "id": 493, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.container.keyword}} - {{labels.job.keyword}} : {{labels.instance.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.container.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "fake": true, "field": "labels.instance.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "fake": true, "field": "labels.job.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "sum" } ], "query": "uuid: $uuid AND metricName.keyword: processMemory", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Top 5 process RSS memory usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "decbytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": "$Datasource", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 340 }, "id": 11, "panels": [], "repeat": null, "title": "Master: $master", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 341 }, "hiddenSeries": false, "id": 108, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.node.keyword}}: {{labels.pod.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.node.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.pod.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "hide": false, "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"podCPU\" AND labels.node.keyword:$master", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Top CPU usage, master: $master", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percent", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": { "align": null, "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 341 }, "hiddenSeries": false, "id": 110, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.node.keyword}}: {{labels.pod.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.pod.keyword", "id": "5", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.node.keyword", "id": "3", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "hide": false, "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"podMemory\" AND labels.node.keyword:$master", "refId": "B", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Top 5 memory RSS, master: $master", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": { "align": null, "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 350 }, "hiddenSeries": false, "id": 2, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/.*user.*/", "color": "#37872D" }, { "alias": "/.*system.*/", "color": "#1F60C4" }, { "alias": "/.*iowait.*/", "color": "#F2495C" }, { "alias": "/.*iqr.*/", "color": "#E0B400" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.instance.keyword}}: {{labels.mode.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.mode.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": "_value*100", "meta": {}, "settings": { "script": { "inline": "_value*100" } }, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"nodeCPU\" AND labels.instance.keyword:$master AND NOT labels.mode.keyword: \"idle\"", "refId": "A", "timeField": "timestamp" }, { "bucketAggs": [], "hide": true, "metrics": [ { "field": "select field", "id": "1", "meta": {}, "settings": { "size": 500 }, "type": "raw_data" } ], "query": "uuid.keyword: $uuid AND metricName: \"nodeCPU\"", "refId": "B", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Node CPU, master: $master", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percent", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 350 }, "hiddenSeries": false, "id": 9, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/.*available.*/", "color": "#73BF69" }, { "alias": "/.*buffer.*/", "color": "#5794F2" }, { "alias": "/.*active.*/", "color": "#FF9830" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.instance.keyword}}: available", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "hide": true, "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"nodeMemoryAvailable\" AND labels.instance.keyword: $master", "refId": "A", "timeField": "timestamp" }, { "alias": "{{labels.instance.keyword}}: cached+buffers", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"nodeMemoryCached+nodeMemoryBuffers\" AND labels.instance.keyword: $master", "refId": "B", "timeField": "timestamp" }, { "alias": "{{labels.instance.keyword}}: active", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"nodeMemoryActive\" AND labels.instance.keyword: $master", "refId": "C", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Node Memory, master: $master", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 358 }, "hiddenSeries": false, "id": 476, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/rxNetworkBytes.*/", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "txNetworkBytes: {{labels.instance.keyword}} - {{labels.device.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.device.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName: \"txNetworkBytes\" AND labels.instance.keyword: $master", "refId": "A", "timeField": "timestamp" }, { "alias": "rxNetworkBytes {{labels.instance.keyword}} - {{labels.device.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.device.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName: \"rxNetworkBytes\" AND labels.instance.keyword: $master", "refId": "B", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Network bytes, master: $master", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(rate(node_disk_written_bytes_total{}[2m])) by (device, instance) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 358 }, "hiddenSeries": false, "id": 35, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/Read.+/", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "Written bytes: {{labels.instance.keyword}} - {{labels.device.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.device.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName: \"nodeDiskWrittenBytes\" AND labels.instance.keyword: $master", "refId": "A", "timeField": "timestamp" }, { "alias": "Read bytes: {{labels.instance.keyword}} - {{labels.device.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.device.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName: \"nodeDiskReadBytes\" AND labels.instance.keyword: $master", "refId": "B", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Disk Throughput, master: $master", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 367 }, "hiddenSeries": false, "id": 6, "legend": { "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/rxDroppedPackets.*/", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "txDroppedPackets: {{labels.instance.keyword}} {{labels.device.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.device.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "hide": false, "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName: \"txDroppedPackets\" AND labels.instance.keyword: $master", "refId": "A", "timeField": "timestamp" }, { "alias": "rxDroppedPackets: {{labels.instance.keyword}} {{labels.device.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.device.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "hide": false, "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName: \"rxDroppedPackets\" AND labels.instance.keyword: $master", "refId": "B", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Dropped packets, master: $master", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "sum(rate(node_disk_reads_completed_total{}[2m])) by (device, instance) > 0\nsum(rate(node_disk_writes_completed_total{}[2m])) by (device, instance) > 0", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 367 }, "hiddenSeries": false, "id": 434, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/Read.+/", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "Write IOPS: {{labels.instance.keyword}} - {{labels.device.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.device.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName: \"nodeDiskWriteOpsSec\" AND labels.instance.keyword: $master", "refId": "A", "timeField": "timestamp" }, { "alias": "Read IOPS: {{labels.instance.keyword}} - {{labels.device.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.device.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName: \"nodeDiskReadOpsSec\" AND labels.instance.keyword: $master", "refId": "B", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Disk IOPS, master: $master", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "description": "", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 375 }, "hiddenSeries": false, "id": 513, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.node.keyword}}: {{labels.container.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.container.keyword", "id": "5", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.node.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "hide": false, "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: containerDiskUsage AND labels.node.keyword:$master", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Top 5 Container FS Write Bytes, master: $master", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 375 }, "hiddenSeries": false, "id": 512, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.node.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.node.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "hide": false, "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: containerFsUsage AND labels.node.keyword:$master", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Top 5 Node FS Usage, master: $master", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "datasource": "$Datasource", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 384 }, "id": 37, "panels": [], "repeat": null, "title": "Worker: $worker", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": { "align": null, "filterable": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 385 }, "hiddenSeries": false, "id": 39, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.node.keyword}}: {{labels.pod.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.node.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "fake": true, "field": "labels.namespace.keyword", "id": "5", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.pod.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 1 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"podCPU\" AND labels.node.keyword:$worker AND labels.namespace.keyword: $namespace", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Top Pod CPU usage, worker: $worker", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percent", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 385 }, "hiddenSeries": false, "id": 85, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.node.keyword}}: {{labels.pod.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.node.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "fake": true, "field": "labels.pod.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "5" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"podMemory\" AND labels.node.keyword: $worker AND labels.namespace.keyword: $namespace", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Top Pod Memory RSS, worker: $worker", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 394 }, "hiddenSeries": false, "id": 68, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/.*user.*/", "color": "#73BF69" }, { "alias": "/.*system.*/", "color": "#5794F2" }, { "alias": "/.*iowait.*/", "color": "#F2495C" }, { "alias": "/.*irq.*/", "color": "#E0B400" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.instance.keyword}} {{labels.mode.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "5", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "fake": true, "field": "labels.mode.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "1", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "auto", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": "_value*100", "meta": {}, "settings": { "script": { "inline": "_value*100" } }, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"nodeCPU\" AND labels.instance.keyword: $worker AND NOT labels.mode.keyword: \"idle\"", "refId": "A", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Node CPU, worker: $worker", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percent", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 3, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 394 }, "hiddenSeries": false, "id": 41, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/.*available.*/", "color": "#73BF69" }, { "alias": "/.*buffer.*/", "color": "#5794F2" }, { "alias": "/.*active.*/", "color": "#FF9830" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "{{labels.instance.keyword}}: available", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "3", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "hide": true, "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"nodeMemoryAvailable\" AND labels.instance.keyword: $worker", "refId": "A", "timeField": "timestamp" }, { "alias": "{{labels.instance.keyword}}: cached+buffers", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "3", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"nodeMemoryCached+nodeMemoryBuffers\" AND labels.instance.keyword: $worker", "refId": "B", "timeField": "timestamp" }, { "alias": "{{labels.instance.keyword}}: active", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "3", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid.keyword: $uuid AND metricName: \"nodeMemoryActive\" AND labels.instance.keyword: $worker", "refId": "C", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Node Memory: worker: $worker", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 11, "w": 12, "x": 0, "y": 402 }, "hiddenSeries": false, "id": 43, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/rxNetworkBytes.*/", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "txNetworkBytes: {{labels.instance.keyword}} - {{labels.device.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "fake": true, "field": "labels.device.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": 0, "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName: \"txNetworkBytes\" AND labels.instance.keyword: $worker", "refId": "A", "timeField": "timestamp" }, { "alias": "rxNetworkBytes {{labels.instance.keyword}} - {{labels.device.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "fake": true, "field": "labels.device.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": 0, "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName: \"rxNetworkBytes\" AND labels.instance.keyword: $worker", "refId": "B", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Network bytes, worker: $worker", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 11, "w": 12, "x": 12, "y": 402 }, "hiddenSeries": false, "id": 67, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/Read.+/", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "Written bytes: {{labels.instance.keyword}} - {{labels.device.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.device.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName: \"nodeDiskWrittenBytes\" AND labels.instance.keyword: $worker", "refId": "A", "timeField": "timestamp" }, { "alias": "Read bytes {{labels.instance.keyword}} - {{labels.device.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.device.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName: \"nodeDiskReadBytes\" AND labels.instance.keyword: $worker", "refId": "B", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Disk Throughput, worker: $worker", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 11, "w": 12, "x": 0, "y": 413 }, "hiddenSeries": false, "id": 63, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/rxDroppedPackets.*/", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "txDroppedPackets - {{labels.instance.keyword}} {{labels.device.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.device.keyword", "id": "3", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": 0, "trimEdges": 0 }, "type": "date_histogram" } ], "hide": false, "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName: txDroppedPackets AND labels.instance.keyword: $worker", "refId": "A", "timeField": "timestamp" }, { "alias": "rxDroppedPackets - {{labels.instance.keyword}} {{labels.device.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "4", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.device.keyword", "id": "3", "settings": { "min_doc_count": 0, "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": 0, "trimEdges": 0 }, "type": "date_histogram" } ], "hide": false, "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName: rxDroppedPackets AND labels.instance.keyword: $worker", "refId": "B", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Dropped packets, worker: $worker", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$Datasource", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 11, "w": 12, "x": 12, "y": 413 }, "hiddenSeries": false, "id": 433, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.4", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/Read.+/", "transform": "negative-Y" } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "alias": "Write IOPS: {{labels.instance.keyword}} - {{labels.device.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.device.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName: \"nodeDiskWriteOpsSec\" AND labels.instance.keyword: $worker", "refId": "A", "timeField": "timestamp" }, { "alias": "Read IOPS: {{labels.instance.keyword}} - {{labels.device.keyword}}", "bucketAggs": [ { "fake": true, "field": "labels.instance.keyword", "id": "4", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "10" }, "type": "terms" }, { "fake": true, "field": "labels.device.keyword", "id": "3", "settings": { "min_doc_count": "1", "order": "desc", "orderBy": "_term", "size": "0" }, "type": "terms" }, { "field": "timestamp", "id": "2", "settings": { "interval": "30s", "min_doc_count": "1", "trimEdges": 0 }, "type": "date_histogram" } ], "metrics": [ { "field": "value", "id": "1", "inlineScript": null, "meta": {}, "settings": {}, "type": "avg" } ], "query": "uuid: $uuid AND metricName: \"nodeDiskReadOpsSec\" AND labels.instance.keyword: $worker", "refId": "B", "timeField": "timestamp" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Disk IOPS, worker: $worker", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "refresh": false, "schemaVersion": 26, "style": "dark", "tags": [], "templating": { "list": [ { "current": { "selected": false, "text": "Elasticsearch-perfscale-burst", "value": "Elasticsearch-perfscale-burst" }, "error": null, "hide": 0, "includeAll": false, "label": "Datasource", "multi": false, "name": "Datasource", "options": [], "query": "elasticsearch", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": "*", "current": { "selected": false, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": "$Datasource", "definition": "{\"find\": \"terms\", \"field\": \"uuid.keyword\", \"query\": \"metricName.keyword: jobSummary\"}", "error": null, "hide": 0, "includeAll": true, "label": "Job UUID", "multi": true, "name": "uuid", "options": [], "query": "{\"find\": \"terms\", \"field\": \"uuid.keyword\", \"query\": \"metricName.keyword: jobSummary\"}", "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": "*", "current": { "selected": false, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": "$Datasource", "definition": "{ \"find\" : \"terms\", \"field\": \"labels.node.keyword\", \"query\": \"metricName.keyword: nodeRoles AND labels.role.keyword: master AND uuid.keyword: $uuid\"}", "error": null, "hide": 0, "includeAll": true, "label": "Master nodes", "multi": true, "name": "master", "options": [], "query": "{ \"find\" : \"terms\", \"field\": \"labels.node.keyword\", \"query\": \"metricName.keyword: nodeRoles AND labels.role.keyword: master AND uuid.keyword: $uuid\"}", "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": "*", "current": { "selected": false, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": "$Datasource", "definition": "{ \"find\" : \"terms\", \"field\": \"labels.node.keyword\", \"query\": \"metricName.keyword: nodeRoles AND labels.role.keyword: worker AND uuid.keyword: $uuid\"}", "error": null, "hide": 0, "includeAll": true, "label": "Worker nodes", "multi": true, "name": "worker", "options": [], "query": "{ \"find\" : \"terms\", \"field\": \"labels.node.keyword\", \"query\": \"metricName.keyword: nodeRoles AND labels.role.keyword: worker AND uuid.keyword: $uuid\"}", "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": "*", "current": { "selected": false, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": "$Datasource", "definition": "{ \"find\" : \"terms\", \"field\": \"labels.namespace.keyword\", \"query\": \"labels.namespace.keyword: /openshift-.*/ AND uuid.keyword: $uuid\"}", "error": null, "hide": 0, "includeAll": true, "label": "Namespace", "multi": true, "name": "namespace", "options": [], "query": "{ \"find\" : \"terms\", \"field\": \"labels.namespace.keyword\", \"query\": \"labels.namespace.keyword: /openshift-.*/ AND uuid.keyword: $uuid\"}", "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "2022-01-15T01:30:00.000Z", "to": "2022-01-15T06:00:59.000Z" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ] }, "timezone": "utc", "title": "Kube-burner CNV report", "uid": "hIBq_IBM_KNvMz", "version": 1 }