From f5944566da93c0b778652ea6222e461f1e1543f5 Mon Sep 17 00:00:00 2001 From: rjshrjndrn Date: Thu, 30 Jun 2022 15:50:41 +0200 Subject: [PATCH] chore(dashboard): Adding clickhouse dashboard Signed-off-by: rjshrjndrn --- .../manifests/observability-values.yaml | 5 + ...ltinity-clickhouse-operator-dashboard.json | 4320 +++++++++++++++++ ...altinity-clickhouse-queries-dashboard.json | 1188 +++++ 3 files changed, 5513 insertions(+) create mode 100644 ee/scripts/helmcharts/manifests/observability/dashboards/clickhouse/altinity-clickhouse-operator-dashboard.json create mode 100644 ee/scripts/helmcharts/manifests/observability/dashboards/clickhouse/altinity-clickhouse-queries-dashboard.json diff --git a/ee/scripts/helmcharts/manifests/observability-values.yaml b/ee/scripts/helmcharts/manifests/observability-values.yaml index 5d2bcbf99..c49593480 100644 --- a/ee/scripts/helmcharts/manifests/observability-values.yaml +++ b/ee/scripts/helmcharts/manifests/observability-values.yaml @@ -27,6 +27,10 @@ kube-prometheus-stack: editable: true type: loki url: http://loki.observability:3100 + - name: k8s-db-openreplay-clickhouse + editable: true + type: vertamedia-clickhouse-datasource + url: http://clickhouse-openreplay-clickhouse.db:8123 plugins: - grafana-piechart-panel - vertamedia-clickhouse-datasource @@ -50,6 +54,7 @@ kube-prometheus-stack: storageSpec: volumeClaimTemplate: spec: + storageClassName: local-path accessModes: ["ReadWriteOnce"] resources: requests: diff --git a/ee/scripts/helmcharts/manifests/observability/dashboards/clickhouse/altinity-clickhouse-operator-dashboard.json b/ee/scripts/helmcharts/manifests/observability/dashboards/clickhouse/altinity-clickhouse-operator-dashboard.json new file mode 100644 index 000000000..9ebfc8804 --- /dev/null +++ b/ee/scripts/helmcharts/manifests/observability/dashboards/clickhouse/altinity-clickhouse-operator-dashboard.json @@ -0,0 +1,4320 @@ +{ + "annotations": { + "list": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "enable": true, + "expr": "ALERTS{app=~\"clickhouse-operator|zookeeper\"}", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "prometheus alerts", + "showIn": 0, + "step": "30s", + "tagKeys": "chi,pod,hostname,exported_namespace,namespace", + "tags": [], + "textFormat": "{{alertstate}}", + "titleFormat": "{{alertname}}", + "type": "tags" + }, + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Alitinity Clickhouse Operatoe metrics exported by Monitoring Agent", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 882, + "graphTooltip": 1, + "id": 33, + "iteration": 1656599112302, + "links": [], + "liveNow": false, + "panels": [ + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "filterNull": false, + "fontSize": "100%", + "gridPos": { + "h": 4, + "w": 10, + "x": 0, + "y": 0 + }, + "id": 15, + "links": [], + "scroll": true, + "showHeader": true, + "sort": { + "col": 2, + "desc": false + }, + "styles": [ + { + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "align": "auto", + "colorMode": "value", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 1, + "pattern": "/.*/", + "thresholds": [ + "3600", + "86400" + ], + "type": "number", + "unit": "s" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_Uptime{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "{{hostname}}", + "metric": "chi_clickhouse_metric_Uptime", + "refId": "A", + "step": 60 + } + ], + "title": "Uptime", + "transform": "timeseries_aggregations", + "type": "table-old" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Clickhouse operator metrics-exporter fails when grab metrics from clickhouse-server\n\nPlease look pods status\n\nkubectl get pods --all-namespaces | grep clickhouse", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 10, + "y": 0 + }, + "id": 47, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "metric_fetch_errors", + "url": "https://github.com/Altinity/clickhouse-operator/search?q=metric_fetch_errors" + } + ], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(chi_clickhouse_metric_fetch_errors{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\",fetch_type=\"system.metrics\"})", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "title": "Failed Pods", + "type": "stat" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "For example, version 11.22.33 is translated to 11022033", + "filterNull": false, + "fontSize": "100%", + "gridPos": { + "h": 4, + "w": 11, + "x": 13, + "y": 0 + }, + "hideTimeOverride": false, + "id": 17, + "links": [], + "scroll": false, + "showHeader": true, + "sort": { + "col": 3, + "desc": true + }, + "styles": [ + { + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 0, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "none" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_VersionInteger{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "{{hostname}}", + "metric": "chi_clickhouse_metric_VersionInteger", + "refId": "A", + "step": 60 + } + ], + "title": "Version", + "transform": "timeseries_aggregations", + "type": "table-old" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Check Zookeeper connection, Disk Free space and network interconnection between replicas ASAP", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 10, + "y": 2 + }, + "id": 6, + "links": [ + { + "targetBlank": true, + "title": "Restore After Failures", + "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/#recovery-after-failures" + }, + { + "targetBlank": true, + "title": "Restore After Data Loss", + "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/#recovery-after-complete-data-loss" + } + ], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.0.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(chi_clickhouse_metric_ReadonlyReplica{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"})", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "title": "ReadOnly replicas", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Show DNS errors and distributed server-server connections failures", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 4 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + { + "targetBlank": true, + "title": "Manage Distributed tables", + "url": "https://clickhouse.tech/docs/en/sql-reference/statements/system/#query-language-system-distributed" + }, + { + "targetBlank": true, + "title": "DNSError", + "url": "https://github.com/ClickHouse/ClickHouse/search?q=DNSError" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_NetworkErrors{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "NetworkErrors {{hostname}}", + "metric": "chi_clickhouse_event_NetworkErrors", + "refId": "A", + "step": 120 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_DistributedConnectionFailAtAll{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "DistributedConnectionFailAtAll {{hostname}}", + "metric": "chi_clickhouse_event_DistributedConnectionFailAtAll", + "refId": "B", + "step": 120 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_DistributedConnectionFailTry{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "DistributedConnectionFailTry {{hostname}}", + "metric": "chi_clickhouse_event_DistributedConnectionFailTry", + "refId": "C", + "step": 120 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_DNSError{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "DNSErrors {{hostname}}", + "metric": "chi_clickhouse_event_NetworkErrors", + "refId": "D", + "step": 120 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "DNS and Distributed Connection Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Show readonly and partial shutdown replicas, zookeeer exceptions, zookeeer sessions, zookeeper init requests", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 4 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + { + "targetBlank": true, + "title": "Recommened Zookeeper Settings", + "url": "https://clickhouse.tech/docs/en/operations/tips/#zookeeper" + }, + { + "targetBlank": true, + "title": "system.zookeeper", + "url": "https://clickhouse.tech/docs/en/operations/system-tables/#system-zookeeper" + }, + { + "targetBlank": true, + "title": "Replication details", + "url": "https://www.slideshare.net/Altinity/introduction-to-the-mysteries-of-clickhouse-replication-by-robert-hodges-and-altinity-engineering-team" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_ReadonlyReplica{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "hide": false, + "intervalFactor": 2, + "legendFormat": "ReadonlyReplica {{hostname}}", + "metric": "chi_clickhouse_metric_ReadonlyReplica", + "refId": "D", + "step": 120 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_ReplicaPartialShutdown{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "ReplicaPartialShutdown {{hostname}}", + "metric": "chi_clickhouse_event_ReplicaPartialShutdown", + "refId": "E", + "step": 120 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_ZooKeeperUserExceptions{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "ZooKeeperUserExceptions {{hostname}}", + "metric": "chi_clickhouse_event_ZooKeeperUserExceptions", + "refId": "B", + "step": 120 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_ZooKeeperInit{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "ZooKeeperInit {{hostname}}", + "metric": "chi_clickhouse_event_ZooKeeperInit", + "refId": "A", + "step": 120 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_metric_ZooKeeperSession{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "ZooKeeperSession {{hostname}}", + "metric": "chi_clickhouse_metric_ZooKeeperSession", + "refId": "C", + "step": 120 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_ZooKeeperHardwareExceptions{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "ZooKeeperHardwareExceptions {{hostname}}", + "metric": "chi_clickhouse_event_ZooKeeperUserExceptions", + "refId": "F", + "step": 120 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Replication and ZooKeeper Exceptions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "delayed query\nNumber of INSERT queries that are throttled due to high number of active data parts for partition in a *MergeTree table.\n\ndelayed blocks\nNumber of times the INSERT of a block to a *MergeTree table was throttled due to high number of active data parts for partition. \n\nrejected blocks\nNumber of times the INSERT of a block to a MergeTree table was rejected with 'Too many parts' exception due to high number of active data parts for partition.\n\n\nplease look\nparts_to_delay_insert\nparts_to_throw_insert\n\nin system.merge_tree_settings table", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 4 + }, + "hiddenSeries": false, + "id": 5, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "system.parts_log", + "url": "https://clickhouse.tech/docs/en/operations/system-tables/#system_tables-part-log" + }, + { + "targetBlank": true, + "title": "system.merge_tree_settings", + "url": "https://clickhouse.tech/docs/en/operations/system-tables/#system-merge_tree_settings" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_DelayedInserts{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "delayed queries {{hostname}}", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_DelayedInserts{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "delayed blocks {{hostname}}", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_RejectedInserts{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "rejected blocks {{hostname}}", + "refId": "C", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_DistributedFilesToInsert{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "delayed distributed files {{ hostname }}", + "refId": "D" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Delayed/Rejected Inserts", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of SELECT queries started to be interpreted and maybe executed. Does not include queries that are failed to parse, that are rejected due to AST size limits; rejected due to quota limits or limits on number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 11 + }, + "hiddenSeries": false, + "id": 1, + "isNew": true, + "legend": { + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "max_concurent_queries", + "url": "https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#max-concurrent-queries" + }, + { + "targetBlank": true, + "title": "max_execution_time", + "url": "https://clickhouse.tech/docs/en/operations/settings/query-complexity/#max-execution-time" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_SelectQuery{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "select {{hostname}}", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_Query{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": true, + "intervalFactor": 2, + "legendFormat": "total {{hostname}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Select Queries", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Show how much bytes read and decompress via compressed buffer on each server in cluster", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 11 + }, + "hiddenSeries": false, + "id": 8, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "I/O buffers architecture", + "url": "https://clickhouse.tech/docs/en/development/architecture/#io" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^uncompressed.+/", + "color": "#73BF69" + }, + { + "alias": "/^(file descriptor|os).+/", + "color": "#F2495C" + }, + { + "alias": "/^compressed.+/", + "color": "#FADE2A" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_CompressedReadBufferBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": false, + "intervalFactor": 2, + "legendFormat": "uncompressed {{hostname}}", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_ReadCompressedBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "compressed {{hostname}}", + "refId": "C", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_ReadBufferFromFileDescriptorReadBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "file descriptor {{hostname}}", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_OSReadBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "os {{hostname}}", + "refId": "D", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Read Bytes", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 10, + "min": "0", + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total amount of memory (bytes) allocated in currently executing queries. \n\nNote that some memory allocations may not be accounted.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 11 + }, + "hiddenSeries": false, + "id": 13, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "max_memory_usage", + "url": "https://clickhouse.tech/docs/en/operations/settings/query-complexity/#settings_max_memory_usage" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_MemoryTracking{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "{{hostname}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory for Queries", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of INSERT queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 30, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "max_memory_usage", + "url": "https://clickhouse.tech/docs/en/operations/settings/query-complexity/#settings_max_memory_usage" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + {}, + {} + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_InsertQuery{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Insert queries {{hostname}}", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Insert Queries", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "reqps", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "## Tracks amount of inserted data.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 18 + }, + "hiddenSeries": false, + "id": 37, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "max_memory_usage", + "url": "https://clickhouse.tech/docs/en/operations/settings/query-complexity/#settings_max_memory_usage" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + {}, + {} + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_InsertedBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Insert bytes {{hostname}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Bytes Inserted", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "reqps", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "## Tracks rows of inserted data.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 18 + }, + "hiddenSeries": false, + "id": 32, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "max_memory_usage", + "url": "https://clickhouse.tech/docs/en/operations/settings/query-complexity/#settings_max_memory_usage" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + {}, + {} + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_InsertedRows{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Insert rows {{hostname}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Rows Inserted", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "reqps", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Show how intensive data exchange between replicas in parts", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 3, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "How replication works", + "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/" + } + ], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^max.+/", + "color": "#F2495C" + }, + { + "alias": "/^check.+/", + "color": "#FF9830" + }, + { + "alias": "/^fetch.+/", + "color": "#B877D9" + }, + { + "alias": "/^send.+/", + "color": "#5794F2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_ReplicasMaxQueueSize{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "max queue size {{hostname}}", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_ReplicatedChecks{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "check {{hostname}}", + "refId": "A", + "step": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_ReplicatedFetch{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "fetch {{hostname}}", + "refId": "B", + "step": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_ReplicatedSend{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "send {{hostname}}", + "refId": "C", + "step": 20 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Replication Queue Jobs", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Show seconds when replicated servers can be delayed relative to current time, when you insert directly in *ReplicatedMegreTree table on one server clickhouse need time to replicate new parts of data to another servers in same shard in background", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 25 + }, + "hiddenSeries": false, + "id": 7, + "isNew": true, + "legend": { + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "Replication architecture", + "url": "https://clickhouse.tech/docs/en/development/architecture/#replication" + }, + { + "targetBlank": true, + "title": "ReplicatedMergeTree engine", + "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/" + }, + { + "targetBlank": true, + "title": "max_replica_delay_for_distributed_queries", + "url": "https://clickhouse.tech/docs/en/operations/settings/settings/#settings-max_replica_delay_for_distributed_queries" + } + ], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^absolute.+/", + "color": "#F2495C" + }, + { + "alias": "/^relative.+/", + "color": "#FADE2A" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_ReplicasMaxAbsoluteDelay{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "absolute {{hostname}}", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_ReplicasMaxRelativeDelay{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "relative {{hostname}}", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Max Replica Delay", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of requests to ZooKeeper transactions per seconds.", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 25 + }, + "hiddenSeries": false, + "id": 34, + "isNew": true, + "legend": { + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "Replication architecture", + "url": "https://clickhouse.tech/docs/en/development/architecture/#replication" + } + ], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_ZooKeeperTransactions{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "transactions {{ hostname }}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_ZooKeeperRequest{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "hide": true, + "legendFormat": "{{ hostname }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Zookeeper Transactions", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Show how intensive background merge processes", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 2, + "isNew": true, + "legend": { + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "START/STOP Merges", + "url": "https://clickhouse.tech/docs/en/sql-reference/statements/system/#query_language-system-stop-merges" + }, + { + "targetBlank": true, + "title": "MegreTree Engine description", + "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/" + } + ], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_Merge{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "merges {{hostname}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Merges", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Show how intensive background merge processes", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 32 + }, + "hiddenSeries": false, + "id": 36, + "isNew": true, + "legend": { + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "START/STOP Merges", + "url": "https://clickhouse.tech/docs/en/sql-reference/statements/system/#query_language-system-stop-merges" + }, + { + "targetBlank": true, + "title": "MegreTree Engine description", + "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/" + } + ], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_MergedRows{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "rows {{hostname}}", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Merged Rows", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Show how intensive background merge processes", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 32 + }, + "hiddenSeries": false, + "id": 49, + "isNew": true, + "legend": { + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "START/STOP Merges", + "url": "https://clickhouse.tech/docs/en/sql-reference/statements/system/#query_language-system-stop-merges" + }, + { + "targetBlank": true, + "title": "MegreTree Engine description", + "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/" + } + ], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_MergedUncompressedBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "intervalFactor": 2, + "legendFormat": "bytes {{hostname}}", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Merged Uncompressed Bytes", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "decimals": 0, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 39 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + { + "targetBlank": true, + "title": "system.parts", + "url": "https://clickhouse.tech/docs/en/operations/system-tables/#system_tables-parts" + }, + { + "targetBlank": true, + "title": "parts_to_delay_insert", + "url": "https://github.com/ClickHouse/ClickHouse/search?q=parts_to_delay_insert" + } + ], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by(hostname) (chi_clickhouse_table_parts{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\",active=\"1\"})", + "legendFormat": "Parts {{hostname}}", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Active Parts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "decimals": 0, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 39 + }, + "hiddenSeries": false, + "id": 50, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + { + "targetBlank": true, + "title": "system.parts", + "url": "https://clickhouse.tech/docs/en/operations/system-tables/#system_tables-parts" + }, + { + "targetBlank": true, + "title": "parts_to_delay_insert", + "url": "https://github.com/ClickHouse/ClickHouse/search?q=parts_to_delay_insert" + } + ], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by(hostname) (chi_clickhouse_table_parts{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\",active=\"0\"})", + "legendFormat": "Parts {{hostname}}", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Inactive Parts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Each logical partition defined over `PARTITION BY` contains few physical data \"parts\" ", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 39 + }, + "hiddenSeries": false, + "id": 4, + "isNew": true, + "legend": { + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "Custom Partitioning Key", + "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key/" + }, + { + "targetBlank": true, + "title": "system.parts", + "url": "https://clickhouse.tech/docs/en/operations/system-tables/#system_tables-parts" + }, + { + "targetBlank": true, + "title": "system.part_log", + "url": "https://clickhouse.tech/docs/en/operations/system-tables/#system_tables-part-log" + } + ], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_MaxPartCountForPartition{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "{{hostname}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Max Part count for Partition", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Memory size allocated for clickhouse-server process\nAvailable for ClickHouse 20.4+\n\nVIRT \nThe total amount of virtual memory used by the task. It includes all code, data and shared libraries plus pages that have been swapped out.\n\nVIRT = SWAP + RES\n\n\nSWAP -- Swapped size (kb)\nThe swapped out portion of a task's total virtual memory image.\n\nRES -- Resident size (kb)\nThe non-swapped physical memory a task has used.\nRES = CODE + USED DATA.\n\nCODE -- Code size (kb)\nThe amount of physical memory devoted to executable code, also known as the 'text resident set' size or TRS\n\nDATA -- Data+Stack size (kb)\nThe amount of physical memory allocated to other than executable code, also known as the 'data resident set' size or DRS.\n\nSHR -- Shared Mem size (kb)\nThe amount of shared memory used by a task. It simply reflects memory that could be potentially shared with other processes.", + "fill": 1, + "fillGradient": 2, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 46 + }, + "hiddenSeries": false, + "id": 46, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + { + "targetBlank": true, + "title": "Describe Linux Process Memory types", + "url": "https://elinux.org/Runtime_Memory_Measurement" + } + ], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/VIRT.+/", + "color": "#73BF69" + }, + { + "alias": "/DATA.+/", + "color": "#C4162A" + }, + { + "alias": "/CODE.+/", + "color": "#FF9830" + }, + { + "alias": "/RES.+/", + "color": "#FADE2A" + }, + { + "alias": "/SHR.+/", + "color": "#5794F2" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_MemoryCode{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "CODE {{ hostname }}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_MemoryResident{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "RES {{ hostname }}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_MemoryShared{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "SHR {{ hostname }}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_MemoryDataAndStack{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "DATA {{ hostname }}", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_MemoryVirtual{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "VIRT {{ hostname }}", + "refId": "E" + } + ], + "thresholds": [], + "timeRegions": [], + "title": " clickhouse-server Process Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Memory size allocated for primary keys", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 46 + }, + "hiddenSeries": false, + "id": 45, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + { + "targetBlank": true, + "title": "How to choose right primary key", + "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/#selecting-the-primary-key" + } + ], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_MemoryPrimaryKeyBytesAllocated{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "{{ hostname }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Primary Keys Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Memory size allocated for dictionaries", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 46 + }, + "hiddenSeries": false, + "id": 43, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + { + "targetBlank": true, + "title": "system.dictionaries", + "url": "https://clickhouse.tech/docs/en/operations/system-tables/#system_tables-dictionaries" + }, + { + "targetBlank": true, + "title": "CREATE DICTIONARY", + "url": "https://clickhouse.tech/docs/en/sql-reference/statements/create/#create-dictionary-query" + } + ], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_MemoryDictionaryBytesAllocated{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "{{ hostname }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Dictionary Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "shows how much space available in the kubernetes pod\n\nbe careful with multiple volumes configuration, kubernetes volume claims and S3 as storage backend", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 53 + }, + "hiddenSeries": false, + "id": 39, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + { + "targetBlank": true, + "title": "system.disks", + "url": "https://clickhouse.tech/docs/en/operations/system-tables/disks/" + }, + { + "targetBlank": true, + "title": "Multiple Disk Volumes", + "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/#table_engine-mergetree-multiple-volumes" + } + ], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_DiskFreeBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"} / chi_clickhouse_metric_DiskTotalBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "{{ disk }} {{hostname}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Disk Space Free", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total data size for all ClickHouse *MergeTree tables\n\n", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 53 + }, + "hiddenSeries": false, + "id": 41, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + { + "targetBlank": true, + "title": "system.parts", + "url": "https://clickhouse.tech/docs/en/operations/system-tables/#system_tables-parts" + } + ], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_DiskDataBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "legendFormat": "{{ hostname }}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Clickhouse Data size on Disk", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Show different types of connections for each server", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 53 + }, + "hiddenSeries": false, + "id": 48, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "max_connections", + "url": "https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#max-connections" + }, + { + "targetBlank": true, + "title": "max_distributed_connections", + "url": "https://clickhouse.tech/docs/en/operations/settings/settings/#max-distributed-connections" + }, + { + "targetBlank": true, + "title": "MySQL Protocol", + "url": "https://clickhouse.tech/docs/en/interfaces/mysql/" + }, + { + "targetBlank": true, + "title": "HTTP Protocol", + "url": "https://clickhouse.tech/docs/en/interfaces/http/" + }, + { + "targetBlank": true, + "title": "Native Protocol", + "url": "https://clickhouse.tech/docs/en/interfaces/tcp/" + } + ], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_TCPConnection{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "tcp {{hostname}}", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_HTTPConnection{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "http {{hostname}}", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_InterserverConnection{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "interserver {{hostname}}", + "refId": "C", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_MySQLConnection{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "mysql {{hostname}}", + "refId": "D", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Connections", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "BackgroundPoolTask\t\n---\nNumber of active tasks in BackgroundProcessingPool (merges, mutations, fetches, or replication queue bookkeeping)\n\n\nBackgroundMovePoolTask\n---\nNumber of active tasks in BackgroundProcessingPool for moves\n\n\nBackgroundSchedulePoolTask\t\n---\nA number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc.", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 60 + }, + "hiddenSeries": false, + "id": 9, + "isNew": true, + "legend": { + "avg": false, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "FETCH PARTITION", + "url": "https://clickhouse.tech/docs/en/sql-reference/statements/alter/partition/#alter_fetch-partition" + }, + { + "targetBlank": true, + "title": "Mutations of data", + "url": "https://clickhouse.tech/docs/en/sql-reference/statements/alter/#mutations" + }, + { + "targetBlank": true, + "title": "Data TTL", + "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/#table_engine-mergetree-ttl" + }, + { + "targetBlank": true, + "title": "MOVE PARTITION", + "url": "https://clickhouse.tech/docs/en/sql-reference/statements/alter/partition/#alter_move-partition" + } + ], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_BackgroundPoolTask{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "merge, mutate, fetch {{hostname}}", + "refId": "A", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_BackgroundSchedulePoolTask{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "clean, alter, replica re-init {{hostname}}", + "refId": "B", + "step": 10 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "chi_clickhouse_metric_BackgroundMovePoolTask{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}", + "intervalFactor": 2, + "legendFormat": "moves {{hostname}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Background Tasks", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of active mutations (ALTER DELETE/ALTER UPDATE) and parts to mutate", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 60 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ + { + "targetBlank": true, + "title": "Mutations", + "url": "https://clickhouse.tech/docs/en/sql-reference/statements/alter/#mutations" + }, + { + "targetBlank": true, + "title": "system.mutations", + "url": "https://clickhouse.tech/docs/en/operations/system-tables/mutations/" + }, + { + "targetBlank": true, + "title": "KILL MUTATION", + "url": "https://clickhouse.tech/docs/en/sql-reference/statements/kill/#kill-mutation" + } + ], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (hostname) (chi_clickhouse_table_mutations{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"})", + "legendFormat": "mutations {{hostname}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (hostname) (chi_clickhouse_table_mutations_parts_to_do{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"})", + "legendFormat": "parts_to_do {{hostname}}", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Mutations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Show which percent of mark files (.mrk) read from memory instead of disk", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 60 + }, + "hiddenSeries": false, + "id": 11, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "mark_cache_size", + "url": "https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-mark-cache-size" + }, + { + "targetBlank": true, + "title": "MergeTree architecture", + "url": "https://clickhouse.tech/docs/en/development/architecture/#merge-tree" + } + ], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_MarkCacheHits{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) / (irate(chi_clickhouse_event_MarkCacheHits{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) + irate(chi_clickhouse_event_MarkCacheMisses{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{hostname}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Marks Cache Hit Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The time which CPU spent on various types of activity ", + "fill": 1, + "fillGradient": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 67 + }, + "hiddenSeries": false, + "id": 51, + "interval": "", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^Disk Read.+/", + "color": "#FF9830" + }, + { + "alias": "/^Disk Write.+/", + "color": "#E0B400" + }, + { + "alias": "/^Real Time.+/", + "color": "#73BF69" + }, + { + "alias": "/^User Time.+/", + "color": "#FFF899" + }, + { + "alias": "/^System Time.+/", + "color": "#F2495C" + }, + { + "alias": "/^OS IO Wait.+/", + "color": "#C4162A" + }, + { + "alias": "/^OS CPU Wait.+/", + "color": "rgb(95, 29, 29)" + }, + { + "alias": "/^OS CPU Virtual.+/", + "color": "#B877D9" + }, + { + "alias": "/^Network Receive.+/", + "color": "#C0D8FF" + }, + { + "alias": "/^Network Send.+/", + "color": "#8AB8FF" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_DiskReadElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": true, + "legendFormat": "Disk Read syscall {{hostname}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_DiskWriteElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": true, + "legendFormat": "Disk Write syscall {{hostname}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_NetworkReceiveElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": true, + "legendFormat": "Network Receive {{hostname}}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_NetworkSendElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "hide": true, + "legendFormat": "Network Send {{hostname}}", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_RealTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Real Time {{hostname}}", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_UserTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "User Time {{hostname}}", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_SystemTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "System Time {{hostname}}", + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_OSIOWaitMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "OS IO Wait {{hostname}}", + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_OSCPUWaitMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "OS CPU Wait {{hostname}}", + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(chi_clickhouse_event_OSCPUVirtualTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "OS CPU Virtual {{hostname}}", + "refId": "J" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU Time per second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "µs", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "breakPoint": "50%", + "combine": { + "label": "Others", + "threshold": "0.01" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The time which CPU spent on various types of activity total for the selected period", + "fontSize": "80%", + "format": "µs", + "gridPos": { + "h": 7, + "w": 16, + "x": 8, + "y": 67 + }, + "id": 52, + "interval": "1m", + "legend": { + "header": "", + "percentage": true, + "show": true, + "sort": "total", + "sortDesc": true, + "values": true + }, + "legendType": "Right side", + "links": [], + "nullPointMode": "connected", + "pieType": "pie", + "strokeWidth": "", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_DiskReadElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Disk Read syscall {{hostname}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_DiskWriteElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Disk Write syscall {{hostname}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_NetworkReceiveElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Network Receive {{hostname}}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_NetworkSendElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Network Send {{hostname}}", + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_RealTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Real Time {{hostname}}", + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_UserTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "User Time {{hostname}}", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_SystemTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "System Time {{hostname}}", + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_OSIOWaitMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "OS IO Wait {{hostname}}", + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_OSCPUWaitMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "OS CPU Wait {{hostname}}", + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_OSCPUVirtualTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "OS CPU Virtual {{hostname}}", + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_ThrottlerSleepMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Throttler Sleep {{hostname}}", + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_DelayedInsertsMilliseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000", + "legendFormat": "Delayed Insert {{hostname}}", + "refId": "L" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_ZooKeeperWaitMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Zookeeper Wait {{hostname}}", + "refId": "M" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_CompileExpressionsMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Compile Expressions {{hostname}}", + "refId": "N" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_MergesTimeMilliseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000", + "legendFormat": "Merges {{hostname}}", + "refId": "O" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_RWLockReadersWaitMilliseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000", + "legendFormat": "RWLock Reader Wait {{hostname}}", + "refId": "P" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_RWLockWritersWaitMilliseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000", + "legendFormat": "RWLock Writer Wait {{hostname}}", + "refId": "Q" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_SelectQueryTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Select Query {{hostname}}", + "refId": "R" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_InsertQueryTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "Insert Query {{hostname}}", + "refId": "S" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_S3ReadMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "S3 Read {{hostname}}", + "refId": "T" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "increase(chi_clickhouse_event_S3WriteMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])", + "legendFormat": "S3 Write {{hostname}}", + "refId": "U" + } + ], + "title": "CPU Time total", + "type": "grafana-piechart-panel", + "valueName": "total" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 74 + }, + "hiddenSeries": false, + "id": 24, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [ + { + "targetBlank": true, + "title": "Howto show detail statsistic on grafana for golang process", + "url": "https://grafana.com/grafana/dashboards/6671" + } + ], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(go_memstats_alloc_bytes_total{app=\"clickhouse-operator\",namespace=~\"$exported_namespace|kube-system\"}[1m])", + "hide": false, + "legendFormat": "{{ namespace }} GO malloc bytes / sec", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "process_resident_memory_bytes{app=\"clickhouse-operator\",namespace=~\"$exported_namespace|kube-system\"}", + "legendFormat": "{{ namespace }} RSS Memory", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Monitoring Agent", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "1m", + "schemaVersion": 36, + "style": "dark", + "tags": [ + "Altinity", + "clickhouse", + "operator" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, exported_namespace)", + "hide": 0, + "includeAll": true, + "label": "K8S Namespace", + "multi": true, + "name": "exported_namespace", + "options": [], + "query": { + "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, exported_namespace)", + "refId": "Prometheus-exported_namespace-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\", exported_namespace=~\"$exported_namespace\"}, chi)", + "hide": 0, + "includeAll": true, + "label": "K8S Clickhouse Installation", + "multi": true, + "name": "chi", + "options": [], + "query": { + "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\", exported_namespace=~\"$exported_namespace\"}, chi)", + "refId": "Prometheus-chi-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\",exported_namespace=~\"$exported_namespace\",chi=~\"$chi\"}, hostname)", + "hide": 0, + "includeAll": true, + "label": "Server", + "multi": true, + "name": "hostname", + "options": [], + "query": { + "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\",exported_namespace=~\"$exported_namespace\",chi=~\"$chi\"}, hostname)", + "refId": "Prometheus-hostname-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Altinity ClickHouse Operator Dashboard", + "uid": "clickhouse-operator-dashboard", + "version": 1, + "weekStart": "" +} diff --git a/ee/scripts/helmcharts/manifests/observability/dashboards/clickhouse/altinity-clickhouse-queries-dashboard.json b/ee/scripts/helmcharts/manifests/observability/dashboards/clickhouse/altinity-clickhouse-queries-dashboard.json new file mode 100644 index 000000000..de9346844 --- /dev/null +++ b/ee/scripts/helmcharts/manifests/observability/dashboards/clickhouse/altinity-clickhouse-queries-dashboard.json @@ -0,0 +1,1188 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Helps to visualize most frequent, slowest, failed queries.\r\nShows queries rate per second, table with last queries", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 2515, + "graphTooltip": 1, + "id": 16, + "iteration": 1656598972006, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "uid": "$db" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 22, + "panels": [], + "targets": [ + { + "datasource": { + "uid": "$db" + }, + "refId": "A" + } + ], + "title": "Top charts", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$db" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 20, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.0.1", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "database": "", + "datasource": { + "uid": "$db" + }, + "dateColDataType": "event_date", + "dateLoading": false, + "dateTimeColDataType": "event_time", + "dateTimeType": "DATETIME", + "datetimeLoading": false, + "expr": "", + "format": "time_series", + "formattedQuery": "SELECT * FROM ($rateColumns(\n substring(query, 1, 45) AS query,\n count() c)\nFROM $table\nWHERE\n cityHash64(query) global in (\n SELECT cityHash64(substring(query, 1, 45)) AS h\n FROM $table\n WHERE\n $timeFilter\n AND type in ($type)\n AND initial_user in ($user)\n AND('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))\n GROUP BY h\n ORDER BY count() desc\n LIMIT $top)\n AND type in ($type)\n AND initial_user in ($user)\n AND('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))) SETTINGS skip_unavailable_shards=1", + "intervalFactor": 2, + "query": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((query, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n substring(query, 1, 45) AS query,\r\n count() c\r\n FROM $table\r\n WHERE $timeFilter\r\n AND cityHash64(query) GLOBAL IN (\r\n SELECT cityHash64(substring(query, 1, 45)) AS h\r\n FROM $table\r\n WHERE $timeFilter\r\n AND type IN ($type)\r\n AND initial_user IN ($user)\r\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT $top\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n AND type IN ($type)\r\n AND initial_user IN ($user)\r\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", + "rawQuery": "SELECT t, arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr) FROM ( SELECT t, groupArray((query, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t, substring(query, 1, 45) AS query, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586192379) AND event_time >= toDateTime(1586192379) AND cityHash64(query) GLOBAL IN ( SELECT cityHash64(substring(query, 1, 45)) AS h FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586192379) AND event_time >= toDateTime(1586192379) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY h ORDER BY count() DESC LIMIT 5 SETTINGS skip_unavailable_shards=1 ) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t, query ORDER BY t ) GROUP BY t ORDER BY t ) SETTINGS skip_unavailable_shards=1", + "refId": "A", + "resultFormat": "time_series", + "round": "0s", + "table": "cluster('all-sharded',system.query_log)", + "tableLoading": false + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Top $top request's rate by type: $type; user: $user; query type: $query_type", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "uid": "$db" + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 17, + "links": [], + "options": { + "content": "1 - successful start of query execution\n\n2 - successful end of query execution\n\n3 - exception before start of query execution\n\n4 - exception while query execution", + "mode": "markdown" + }, + "pluginVersion": "9.0.1", + "targets": [ + { + "datasource": { + "uid": "$db" + }, + "refId": "A" + } + ], + "title": "Types", + "transparent": true, + "type": "text" + }, + { + "columns": [], + "datasource": { + "uid": "$db" + }, + "fontSize": "100%", + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 8 + }, + "height": "400px", + "id": 18, + "links": [], + "scroll": true, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "t", + "type": "hidden" + }, + { + "alias": "duration", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "duration", + "thresholds": [], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "count", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "database": "", + "datasource": { + "uid": "$db" + }, + "dateColDataType": "event_date", + "dateLoading": false, + "dateTimeColDataType": "event_time", + "dateTimeType": "DATETIME", + "datetimeLoading": false, + "expr": "", + "format": "table", + "formattedQuery": "SELECT
rand() as t,
substring(query, 1, 70) AS query,
avg(query_duration_ms) duration,
count() count
FROM $table
WHERE
$timeFilter
AND type in ($type)
and initial_user in ($user)
and('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))
GROUP BY query
ORDER BY duration desc
LIMIT $top", + "intervalFactor": 2, + "query": "SELECT\n rand() as t,\n substring(query, 1, 70) AS query,\n avg(query_duration_ms) duration,\n count() count\nFROM $table\nWHERE\n $timeFilter\n AND type in ($type)\n and initial_user in ($user)\n and('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))\nGROUP BY query\nORDER BY duration desc\nLIMIT $top", + "rawQuery": "SELECT t, arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr) FROM ( SELECT t, groupArray((query, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t, substring(query, 1, 45) AS query, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234275) AND event_time >= toDateTime(1586234275) AND cityHash64(query) GLOBAL IN ( SELECT cityHash64(substring(query, 1, 45)) AS h FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234275) AND event_time >= toDateTime(1586234275) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY h ORDER BY count() DESC LIMIT 5 SETTINGS skip_unavailable_shards=1 ) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t, query ORDER BY t ) GROUP BY t ORDER BY t ) SETTINGS skip_unavailable_shards=1", + "refId": "A", + "resultFormat": "time_series", + "round": "0s", + "table": "cluster('all-sharded',system.query_log)", + "tableLoading": false + } + ], + "title": "Top slow queries by type: $type; user: $user; query type: $query_type", + "transform": "table", + "type": "table-old" + }, + { + "columns": [], + "datasource": { + "uid": "$db" + }, + "fontSize": "100%", + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 8 + }, + "height": "400px", + "id": 19, + "links": [], + "scroll": true, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "t", + "type": "hidden" + }, + { + "alias": "usage", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "usage", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "count", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "database": "", + "datasource": { + "uid": "$db" + }, + "dateColDataType": "event_date", + "dateLoading": false, + "dateTimeColDataType": "event_time", + "dateTimeType": "DATETIME", + "datetimeLoading": false, + "expr": "", + "format": "table", + "formattedQuery": "SELECT
rand() as t,
substring(query, 1, 70) AS query,
avg(memory_usage) usage,
count() count
FROM $table
WHERE
$timeFilter
AND type in ($type)
and initial_user in ($user)
and('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))
GROUP BY query
ORDER BY usage desc
LIMIT $top", + "intervalFactor": 2, + "query": "SELECT\n rand() as t,\n substring(query, 1, 70) AS query,\n avg(memory_usage) usage,\n count() count\nFROM $table\nWHERE\n $timeFilter\n AND type IN ($type)\n AND initial_user IN ($user)\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\nGROUP BY query\nORDER BY usage DESC\nLIMIT $top", + "rawQuery": "SELECT t, arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr) FROM ( SELECT t, groupArray((query, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t, substring(query, 1, 45) AS query, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234300) AND event_time >= toDateTime(1586234300) AND cityHash64(query) GLOBAL IN ( SELECT cityHash64(substring(query, 1, 45)) AS h FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234300) AND event_time >= toDateTime(1586234300) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY h ORDER BY count() DESC LIMIT 5 SETTINGS skip_unavailable_shards=1 ) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t, query ORDER BY t ) GROUP BY t ORDER BY t ) SETTINGS skip_unavailable_shards=1", + "refId": "A", + "resultFormat": "time_series", + "round": "0s", + "table": "cluster('all-sharded',system.query_log)", + "tableLoading": false + } + ], + "title": "Top memory consumers by type: $type; user: $user; query type: $query_type", + "transform": "table", + "type": "table-old" + }, + { + "columns": [], + "datasource": { + "uid": "$db" + }, + "fontSize": "100%", + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 8 + }, + "height": "400px", + "id": 20, + "links": [], + "scroll": true, + "showHeader": true, + "sort": { + "col": 3, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "t", + "type": "hidden" + }, + { + "alias": "type", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "type", + "thresholds": [], + "type": "number", + "unit": "none" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "count", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "database": "", + "datasource": { + "uid": "$db" + }, + "dateColDataType": "event_date", + "dateLoading": false, + "dateTimeColDataType": "event_time", + "dateTimeType": "DATETIME", + "datetimeLoading": false, + "expr": "", + "format": "table", + "formattedQuery": "SELECT
rand() as t,
substring(query, 1, 70) AS query,
type,
count() count
FROM $table
WHERE
$timeFilter
AND type in (3,4)
and initial_user in ($user)
and('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))
GROUP BY
query,
type
ORDER BY count desc
LIMIT $top", + "intervalFactor": 2, + "query": "SELECT\n rand() as t,\n substring(query, 1, 70) AS query,\n type,\n count() count\nFROM $table\nWHERE\n $timeFilter\n AND type IN (3,4)\n AND initial_user IN ($user)\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\nGROUP BY\n query,\n type\nORDER BY count desc\nLIMIT $top", + "rawQuery": "SELECT t, arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr) FROM ( SELECT t, groupArray((query, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t, substring(query, 1, 45) AS query, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234335) AND event_time >= toDateTime(1586234335) AND cityHash64(query) GLOBAL IN ( SELECT cityHash64(substring(query, 1, 45)) AS h FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234335) AND event_time >= toDateTime(1586234335) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY h ORDER BY count() DESC LIMIT 5 SETTINGS skip_unavailable_shards=1 ) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t, query ORDER BY t ) GROUP BY t ORDER BY t ) SETTINGS skip_unavailable_shards=1", + "refId": "A", + "resultFormat": "time_series", + "round": "0s", + "table": "cluster('all-sharded',system.query_log)", + "tableLoading": false + } + ], + "title": "Top failed queries by user: $user; query type: $query_type", + "transform": "table", + "type": "table-old" + }, + { + "collapsed": false, + "datasource": { + "uid": "$db" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 23, + "panels": [], + "targets": [ + { + "datasource": { + "uid": "$db" + }, + "refId": "A" + } + ], + "title": "Request charts", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$db" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 19 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "database": "", + "datasource": { + "uid": "$db" + }, + "dateColDataType": "event_date", + "dateLoading": false, + "dateTimeColDataType": "event_time", + "dateTimeType": "DATETIME", + "datetimeLoading": false, + "expr": "", + "format": "time_series", + "formattedQuery": "$rate(count() c)
FROM $table
where type in ($type)
and initial_user in ($user)
and('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))", + "interval": "", + "intervalFactor": 2, + "query": "$rate(count() c)\nFROM $table\nWHERE type IN($type)\n AND initial_user IN ($user)\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))", + "rawQuery": "SELECT t, c/runningDifference(t/1000) cRate FROM ( SELECT (intDiv(toUInt32(event_time), 120) * 120) * 1000 AS t, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586192459) AND event_time >= toDateTime(1586192459) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t ORDER BY t)", + "refId": "A", + "resultFormat": "time_series", + "round": "0s", + "table": "cluster('all-sharded',system.query_log)", + "tableLoading": false + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Reqs/s by type: $type; user: $user; query type: $query_type", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$db" + }, + "fill": 1, + "fillGradient": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 19 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/insert.+/", + "yaxis": 2 + }, + { + "alias": "select_quantile_50", + "color": "#FADE2A" + }, + { + "alias": "select_quantile_90", + "color": "#73BF69" + }, + { + "alias": "insert_quantile_50", + "color": "#FA6400" + }, + { + "alias": "insert_quantile_90", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "database": "", + "datasource": { + "uid": "$db" + }, + "dateColDataType": "event_date", + "dateLoading": false, + "dateTimeColDataType": "event_time", + "dateTimeType": "DATETIME", + "datetimeLoading": false, + "expr": "", + "format": "time_series", + "formattedQuery": "SELECT
$timeSeries as t,
avg(query_duration_ms) select_duration
FROM $table
WHERE
$timeFilter
AND type = 2
and positionCaseInsensitive(query, 'select') = 1
and initial_user in ($user)
GROUP BY t
ORDER BY t", + "intervalFactor": 2, + "query": "SELECT\n $timeSeries as t,\n quantile(0.5)(query_duration_ms) select_quantile_50,\n quantile(0.9)(query_duration_ms) select_quantile_90\nFROM $table\nWHERE\n $timeFilter\n AND type = 2\n AND positionCaseInsensitive(query, 'select') = 1\n AND initial_user IN ($user)\nGROUP BY t\nORDER BY t", + "rawQuery": "SELECT (intDiv(toUInt32(event_time), 10) * 10) * 1000 as t, quantile(0.5)(query_duration_ms) select_quantile_50, quantile(0.9)(query_duration_ms) select_quantile_90 FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1595573984) AND event_time >= toDateTime(1595573984) AND type = 2 AND positionCaseInsensitive(query, 'select') = 1 AND initial_user IN ('default') GROUP BY t ORDER BY t", + "refId": "A", + "resultFormat": "time_series", + "round": "0s", + "table": "cluster('all-sharded',system.query_log)", + "tableLoading": false + }, + { + "database": "", + "datasource": { + "uid": "$db" + }, + "dateColDataType": "event_date", + "dateLoading": false, + "dateTimeColDataType": "event_time", + "dateTimeType": "DATETIME", + "datetimeLoading": false, + "expr": "", + "format": "time_series", + "formattedQuery": "SELECT
$timeSeries as t,
avg(query_duration_ms) insert_duration
FROM $table
WHERE
$timeFilter
AND type = 2
and positionCaseInsensitive(query, 'insert into') = 1
and initial_user in ($user)
GROUP BY t
ORDER BY t", + "intervalFactor": 2, + "query": "SELECT\n $timeSeries as t,\n quantile(0.5)(query_duration_ms) insert_quantile_50,\n quantile(0.9)(query_duration_ms) insert_quantile_90\nFROM $table\nWHERE\n $timeFilter\n AND type = 2\nand positionCaseInsensitive(query, 'insert into') = 1\n and initial_user in ($user)\nGROUP BY t\nORDER BY t", + "rawQuery": "SELECT (intDiv(toUInt32(event_time), 10) * 10) * 1000 as t, quantile(0.5)(query_duration_ms) insert_quantile_50, quantile(0.9)(query_duration_ms) insert_quantile_90 FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1595573984) AND event_time >= toDateTime(1595573984) AND type = 2 and positionCaseInsensitive(query, 'insert into') = 1 and initial_user in ('default') GROUP BY t ORDER BY t", + "refId": "B", + "resultFormat": "time_series", + "round": "0s", + "table": "cluster('all-sharded',system.query_log)", + "tableLoading": false + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Query duration by type: $type; user: $user", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "show": true + }, + { + "format": "ms", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "uid": "$db" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 24, + "panels": [], + "targets": [ + { + "datasource": { + "uid": "$db" + }, + "refId": "A" + } + ], + "title": "Query log table", + "type": "row" + }, + { + "columns": [], + "datasource": { + "uid": "$db" + }, + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 21, + "links": [], + "scroll": true, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "duration", + "thresholds": [], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "memory", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "database": "", + "datasource": { + "uid": "$db" + }, + "dateColDataType": "event_date", + "dateLoading": false, + "dateTimeColDataType": "event_time", + "dateTimeType": "DATETIME", + "datetimeLoading": false, + "format": "time_series", + "formattedQuery": "SELECT
event_time,
user,
query_duration_ms duration,
memory_usage memory,
if(exception!='', 'fail', 'success') result,
concat(substring(query,1,120), '...') query
FROM $table
WHERE $timeFilter
AND type in ($type)
and initial_user in ($user)
and('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))
ORDER BY event_time DESC
LIMIT 1000", + "intervalFactor": 1, + "query": "SELECT\n event_time,\n hostName() host,\n user,\n query_duration_ms duration,\n memory_usage memory,\n if(exception!='', 'fail', 'success') result,\n concat(substring(query,1,120), '...') query\nFROM $table\nWHERE $timeFilter\n AND type in ($type)\n and initial_user in ($user)\n and('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\nORDER BY event_time DESC\nLIMIT 1000", + "rawQuery": "SELECT (intDiv(toUInt32(event_time), 10) * 10) * 1000 as t, avg(query_duration_ms) select_duration FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586232299) AND event_time >= toDateTime(1586232299) AND type = 2 and positionCaseInsensitive(query, 'select') = 1 and initial_user in ('default') GROUP BY t ORDER BY t", + "refId": "A", + "resultFormat": "time_series", + "round": "0s", + "table": "cluster('all-sharded',system.query_log)", + "tableLoading": false + } + ], + "title": "Query log by type: $type; user: $user; query type: $query_type", + "transform": "timeseries_to_columns", + "type": "table-old" + } + ], + "refresh": "5s", + "schemaVersion": 36, + "style": "dark", + "tags": [ + "Altinity", + "clickhouse", + "queries", + "performance" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "k8s-db-openreplay-clickhouse", + "value": "k8s-db-openreplay-clickhouse" + }, + "hide": 2, + "includeAll": false, + "label": "K8S ClickHouse installation", + "multi": false, + "name": "db", + "options": [], + "query": "vertamedia-clickhouse-datasource", + "refresh": 1, + "regex": "k8s-$exported_namespace-$chi", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "db", + "value": "db" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, exported_namespace)", + "hide": 0, + "includeAll": false, + "label": "K8S Namespace", + "multi": false, + "name": "exported_namespace", + "options": [], + "query": { + "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, exported_namespace)", + "refId": "Prometheus-exported_namespace-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "openreplay-clickhouse", + "value": "openreplay-clickhouse" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\", exported_namespace=\"$exported_namespace\"}, chi)", + "hide": 0, + "includeAll": false, + "label": "K8S Clickhouse Installation", + "multi": false, + "name": "chi", + "options": [], + "query": { + "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\",exported_namespace=\"$exported_namespace\"}, chi)", + "refId": "Prometheus-chi-Variable-Query" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": true, + "auto_count": 100, + "auto_min": "1m", + "current": { + "selected": false, + "text": "5m", + "value": "5m" + }, + "hide": 2, + "name": "interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + } + ], + "query": "5m", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "allValue": "1,2,3,4", + "current": { + "selected": true, + "tags": [], + "text": "All", + "value": [ + "$__all" + ] + }, + "hide": 0, + "includeAll": true, + "label": "type", + "multi": true, + "name": "type", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "1", + "value": "1" + }, + { + "selected": false, + "text": "2", + "value": "2" + }, + { + "selected": false, + "text": "3", + "value": "3" + }, + { + "selected": false, + "text": "4", + "value": "4" + } + ], + "query": "1,2,3,4", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": true, + "tags": [], + "text": "5", + "value": "5" + }, + "hide": 0, + "includeAll": false, + "label": "top elements", + "multi": false, + "name": "top", + "options": [ + { + "selected": true, + "text": "5", + "value": "5" + }, + { + "selected": false, + "text": "10", + "value": "10" + }, + { + "selected": false, + "text": "15", + "value": "15" + }, + { + "selected": false, + "text": "20", + "value": "20" + }, + { + "selected": false, + "text": "25", + "value": "25" + }, + { + "selected": false, + "text": "30", + "value": "30" + } + ], + "query": "5,10,15,20,25,30", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "uid": "$db" + }, + "definition": "SELECT DISTINCT initial_user FROM cluster('all-sharded',system.query_log) WHERE event_date >= today()-3 AND event_time > toDateTime(today()-3) SETTINGS skip_unavailable_shards=1", + "hide": 0, + "includeAll": true, + "label": "initial user", + "multi": true, + "name": "user", + "options": [], + "query": "SELECT DISTINCT initial_user FROM cluster('all-sharded',system.query_log) WHERE event_date >= today()-3 AND event_time > toDateTime(today()-3) SETTINGS skip_unavailable_shards=1", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": "", + "current": { + "selected": true, + "text": "all", + "value": "all" + }, + "hide": 0, + "includeAll": false, + "label": "query type", + "multi": false, + "name": "query_type", + "options": [ + { + "selected": true, + "text": "all", + "value": "all" + }, + { + "selected": false, + "text": "select", + "value": "select" + }, + { + "selected": false, + "text": "insert", + "value": "insert" + } + ], + "query": "all,select,insert", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "ClickHouse in Kubernetes Queries", + "uid": "clickhouse-queries-dashboard", + "version": 1, + "weekStart": "" +}