diff --git a/ee/scripts/helmcharts/manifests/observability-values.yaml b/ee/scripts/helmcharts/manifests/observability-values.yaml
index 5d2bcbf99..c49593480 100644
--- a/ee/scripts/helmcharts/manifests/observability-values.yaml
+++ b/ee/scripts/helmcharts/manifests/observability-values.yaml
@@ -27,6 +27,10 @@ kube-prometheus-stack:
editable: true
type: loki
url: http://loki.observability:3100
+ - name: k8s-db-openreplay-clickhouse
+ editable: true
+ type: vertamedia-clickhouse-datasource
+ url: http://clickhouse-openreplay-clickhouse.db:8123
plugins:
- grafana-piechart-panel
- vertamedia-clickhouse-datasource
@@ -50,6 +54,7 @@ kube-prometheus-stack:
storageSpec:
volumeClaimTemplate:
spec:
+ storageClassName: local-path
accessModes: ["ReadWriteOnce"]
resources:
requests:
diff --git a/ee/scripts/helmcharts/manifests/observability/dashboards/clickhouse/altinity-clickhouse-operator-dashboard.json b/ee/scripts/helmcharts/manifests/observability/dashboards/clickhouse/altinity-clickhouse-operator-dashboard.json
new file mode 100644
index 000000000..9ebfc8804
--- /dev/null
+++ b/ee/scripts/helmcharts/manifests/observability/dashboards/clickhouse/altinity-clickhouse-operator-dashboard.json
@@ -0,0 +1,4320 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "enable": true,
+ "expr": "ALERTS{app=~\"clickhouse-operator|zookeeper\"}",
+ "hide": false,
+ "iconColor": "rgba(255, 96, 96, 1)",
+ "limit": 100,
+ "name": "prometheus alerts",
+ "showIn": 0,
+ "step": "30s",
+ "tagKeys": "chi,pod,hostname,exported_namespace,namespace",
+ "tags": [],
+ "textFormat": "{{alertstate}}",
+ "titleFormat": "{{alertname}}",
+ "type": "tags"
+ },
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "Alitinity Clickhouse Operatoe metrics exported by Monitoring Agent",
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "gnetId": 882,
+ "graphTooltip": 1,
+ "id": 33,
+ "iteration": 1656599112302,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "columns": [
+ {
+ "text": "Current",
+ "value": "current"
+ }
+ ],
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "filterNull": false,
+ "fontSize": "100%",
+ "gridPos": {
+ "h": 4,
+ "w": 10,
+ "x": 0,
+ "y": 0
+ },
+ "id": 15,
+ "links": [],
+ "scroll": true,
+ "showHeader": true,
+ "sort": {
+ "col": 2,
+ "desc": false
+ },
+ "styles": [
+ {
+ "align": "auto",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "Time",
+ "type": "date"
+ },
+ {
+ "align": "auto",
+ "colorMode": "value",
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "decimals": 1,
+ "pattern": "/.*/",
+ "thresholds": [
+ "3600",
+ "86400"
+ ],
+ "type": "number",
+ "unit": "s"
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_Uptime{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "intervalFactor": 2,
+ "legendFormat": "{{hostname}}",
+ "metric": "chi_clickhouse_metric_Uptime",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "title": "Uptime",
+ "transform": "timeseries_aggregations",
+ "type": "table-old"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Clickhouse operator metrics-exporter fails when grab metrics from clickhouse-server\n\nPlease look pods status\n\nkubectl get pods --all-namespaces | grep clickhouse",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(50, 172, 45, 0.97)",
+ "value": null
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 1
+ },
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 3,
+ "x": 10,
+ "y": 0
+ },
+ "id": 47,
+ "interval": "",
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "metric_fetch_errors",
+ "url": "https://github.com/Altinity/clickhouse-operator/search?q=metric_fetch_errors"
+ }
+ ],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.0.1",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "sum(chi_clickhouse_metric_fetch_errors{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\",fetch_type=\"system.metrics\"})",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "title": "Failed Pods",
+ "type": "stat"
+ },
+ {
+ "columns": [
+ {
+ "text": "Current",
+ "value": "current"
+ }
+ ],
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "For example, version 11.22.33 is translated to 11022033",
+ "filterNull": false,
+ "fontSize": "100%",
+ "gridPos": {
+ "h": 4,
+ "w": 11,
+ "x": 13,
+ "y": 0
+ },
+ "hideTimeOverride": false,
+ "id": 17,
+ "links": [],
+ "scroll": false,
+ "showHeader": true,
+ "sort": {
+ "col": 3,
+ "desc": true
+ },
+ "styles": [
+ {
+ "align": "auto",
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "decimals": 0,
+ "pattern": "/.*/",
+ "thresholds": [],
+ "type": "number",
+ "unit": "none"
+ }
+ ],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_VersionInteger{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "intervalFactor": 2,
+ "legendFormat": "{{hostname}}",
+ "metric": "chi_clickhouse_metric_VersionInteger",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "title": "Version",
+ "transform": "timeseries_aggregations",
+ "type": "table-old"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Check Zookeeper connection, Disk Free space and network interconnection between replicas ASAP",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "rgba(50, 172, 45, 0.97)",
+ "value": null
+ },
+ {
+ "color": "rgba(237, 129, 40, 0.89)",
+ "value": 1
+ },
+ {
+ "color": "rgba(245, 54, 54, 0.9)",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 3,
+ "x": 10,
+ "y": 2
+ },
+ "id": 6,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "Restore After Failures",
+ "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/#recovery-after-failures"
+ },
+ {
+ "targetBlank": true,
+ "title": "Restore After Data Loss",
+ "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/#recovery-after-complete-data-loss"
+ }
+ ],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto"
+ },
+ "pluginVersion": "9.0.1",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "sum(chi_clickhouse_metric_ReadonlyReplica{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"})",
+ "intervalFactor": 2,
+ "legendFormat": "",
+ "refId": "A",
+ "step": 60
+ }
+ ],
+ "title": "ReadOnly replicas",
+ "type": "stat"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Show DNS errors and distributed server-server connections failures",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 4
+ },
+ "hiddenSeries": false,
+ "id": 21,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "hideZero": false,
+ "max": true,
+ "min": true,
+ "show": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "Manage Distributed tables",
+ "url": "https://clickhouse.tech/docs/en/sql-reference/statements/system/#query-language-system-distributed"
+ },
+ {
+ "targetBlank": true,
+ "title": "DNSError",
+ "url": "https://github.com/ClickHouse/ClickHouse/search?q=DNSError"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.1",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": true,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_NetworkErrors{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "intervalFactor": 2,
+ "legendFormat": "NetworkErrors {{hostname}}",
+ "metric": "chi_clickhouse_event_NetworkErrors",
+ "refId": "A",
+ "step": 120
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_DistributedConnectionFailAtAll{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "intervalFactor": 2,
+ "legendFormat": "DistributedConnectionFailAtAll {{hostname}}",
+ "metric": "chi_clickhouse_event_DistributedConnectionFailAtAll",
+ "refId": "B",
+ "step": 120
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_DistributedConnectionFailTry{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "intervalFactor": 2,
+ "legendFormat": "DistributedConnectionFailTry {{hostname}}",
+ "metric": "chi_clickhouse_event_DistributedConnectionFailTry",
+ "refId": "C",
+ "step": 120
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_DNSError{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "intervalFactor": 2,
+ "legendFormat": "DNSErrors {{hostname}}",
+ "metric": "chi_clickhouse_event_NetworkErrors",
+ "refId": "D",
+ "step": 120
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "DNS and Distributed Connection Errors",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Show readonly and partial shutdown replicas, zookeeer exceptions, zookeeer sessions, zookeeper init requests",
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 4
+ },
+ "hiddenSeries": false,
+ "id": 19,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "hideZero": false,
+ "max": true,
+ "min": true,
+ "show": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "Recommened Zookeeper Settings",
+ "url": "https://clickhouse.tech/docs/en/operations/tips/#zookeeper"
+ },
+ {
+ "targetBlank": true,
+ "title": "system.zookeeper",
+ "url": "https://clickhouse.tech/docs/en/operations/system-tables/#system-zookeeper"
+ },
+ {
+ "targetBlank": true,
+ "title": "Replication details",
+ "url": "https://www.slideshare.net/Altinity/introduction-to-the-mysteries-of-clickhouse-replication-by-robert-hodges-and-altinity-engineering-team"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.1",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": true,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_ReadonlyReplica{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "hide": false,
+ "intervalFactor": 2,
+ "legendFormat": "ReadonlyReplica {{hostname}}",
+ "metric": "chi_clickhouse_metric_ReadonlyReplica",
+ "refId": "D",
+ "step": 120
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_ReplicaPartialShutdown{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "intervalFactor": 2,
+ "legendFormat": "ReplicaPartialShutdown {{hostname}}",
+ "metric": "chi_clickhouse_event_ReplicaPartialShutdown",
+ "refId": "E",
+ "step": 120
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_ZooKeeperUserExceptions{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "intervalFactor": 2,
+ "legendFormat": "ZooKeeperUserExceptions {{hostname}}",
+ "metric": "chi_clickhouse_event_ZooKeeperUserExceptions",
+ "refId": "B",
+ "step": 120
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_ZooKeeperInit{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "intervalFactor": 2,
+ "legendFormat": "ZooKeeperInit {{hostname}}",
+ "metric": "chi_clickhouse_event_ZooKeeperInit",
+ "refId": "A",
+ "step": 120
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_metric_ZooKeeperSession{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "intervalFactor": 2,
+ "legendFormat": "ZooKeeperSession {{hostname}}",
+ "metric": "chi_clickhouse_metric_ZooKeeperSession",
+ "refId": "C",
+ "step": 120
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_ZooKeeperHardwareExceptions{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "intervalFactor": 2,
+ "legendFormat": "ZooKeeperHardwareExceptions {{hostname}}",
+ "metric": "chi_clickhouse_event_ZooKeeperUserExceptions",
+ "refId": "F",
+ "step": 120
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Replication and ZooKeeper Exceptions",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "delayed query\nNumber of INSERT queries that are throttled due to high number of active data parts for partition in a *MergeTree table.\n\ndelayed blocks\nNumber of times the INSERT of a block to a *MergeTree table was throttled due to high number of active data parts for partition. \n\nrejected blocks\nNumber of times the INSERT of a block to a MergeTree table was rejected with 'Too many parts' exception due to high number of active data parts for partition.\n\n\nplease look\nparts_to_delay_insert\nparts_to_throw_insert\n\nin system.merge_tree_settings table",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 4
+ },
+ "hiddenSeries": false,
+ "id": 5,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "system.parts_log",
+ "url": "https://clickhouse.tech/docs/en/operations/system-tables/#system_tables-part-log"
+ },
+ {
+ "targetBlank": true,
+ "title": "system.merge_tree_settings",
+ "url": "https://clickhouse.tech/docs/en/operations/system-tables/#system-merge_tree_settings"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.1",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_DelayedInserts{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "intervalFactor": 2,
+ "legendFormat": "delayed queries {{hostname}}",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_DelayedInserts{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "intervalFactor": 2,
+ "legendFormat": "delayed blocks {{hostname}}",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_RejectedInserts{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "intervalFactor": 2,
+ "legendFormat": "rejected blocks {{hostname}}",
+ "refId": "C",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_DistributedFilesToInsert{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "legendFormat": "delayed distributed files {{ hostname }}",
+ "refId": "D"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Delayed/Rejected Inserts",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Number of SELECT queries started to be interpreted and maybe executed. Does not include queries that are failed to parse, that are rejected due to AST size limits; rejected due to quota limits or limits on number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 11
+ },
+ "hiddenSeries": false,
+ "id": 1,
+ "isNew": true,
+ "legend": {
+ "avg": true,
+ "current": true,
+ "hideEmpty": true,
+ "hideZero": true,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "max_concurent_queries",
+ "url": "https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#max-concurrent-queries"
+ },
+ {
+ "targetBlank": true,
+ "title": "max_execution_time",
+ "url": "https://clickhouse.tech/docs/en/operations/settings/query-complexity/#max-execution-time"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.1",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": true,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_SelectQuery{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "intervalFactor": 2,
+ "legendFormat": "select {{hostname}}",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_Query{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "hide": true,
+ "intervalFactor": 2,
+ "legendFormat": "total {{hostname}}",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Select Queries",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Show how much bytes read and decompress via compressed buffer on each server in cluster",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 1,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 11
+ },
+ "hiddenSeries": false,
+ "id": 8,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideEmpty": true,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "I/O buffers architecture",
+ "url": "https://clickhouse.tech/docs/en/development/architecture/#io"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.1",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "/^uncompressed.+/",
+ "color": "#73BF69"
+ },
+ {
+ "alias": "/^(file descriptor|os).+/",
+ "color": "#F2495C"
+ },
+ {
+ "alias": "/^compressed.+/",
+ "color": "#FADE2A"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_CompressedReadBufferBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "hide": false,
+ "intervalFactor": 2,
+ "legendFormat": "uncompressed {{hostname}}",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_ReadCompressedBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "compressed {{hostname}}",
+ "refId": "C",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_ReadBufferFromFileDescriptorReadBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "file descriptor {{hostname}}",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_OSReadBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "hide": false,
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "os {{hostname}}",
+ "refId": "D",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Read Bytes",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "label": "",
+ "logBase": 10,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Total amount of memory (bytes) allocated in currently executing queries. \n\nNote that some memory allocations may not be accounted.",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 11
+ },
+ "hiddenSeries": false,
+ "id": 13,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "max_memory_usage",
+ "url": "https://clickhouse.tech/docs/en/operations/settings/query-complexity/#settings_max_memory_usage"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.1",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_MemoryTracking{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "intervalFactor": 2,
+ "legendFormat": "{{hostname}}",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Memory for Queries",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Number of INSERT queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 18
+ },
+ "hiddenSeries": false,
+ "id": 30,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "max_memory_usage",
+ "url": "https://clickhouse.tech/docs/en/operations/settings/query-complexity/#settings_max_memory_usage"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.1",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {},
+ {}
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_InsertQuery{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "Insert queries {{hostname}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Insert Queries",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "reqps",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "## Tracks amount of inserted data.",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 18
+ },
+ "hiddenSeries": false,
+ "id": 37,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "max_memory_usage",
+ "url": "https://clickhouse.tech/docs/en/operations/settings/query-complexity/#settings_max_memory_usage"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.1",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {},
+ {}
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_InsertedBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "Insert bytes {{hostname}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Bytes Inserted",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "reqps",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "## Tracks rows of inserted data.",
+ "editable": true,
+ "error": false,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 18
+ },
+ "hiddenSeries": false,
+ "id": 32,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "max_memory_usage",
+ "url": "https://clickhouse.tech/docs/en/operations/settings/query-complexity/#settings_max_memory_usage"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.1",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {},
+ {}
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_InsertedRows{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "Insert rows {{hostname}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Rows Inserted",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "reqps",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Show how intensive data exchange between replicas in parts",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 25
+ },
+ "hiddenSeries": false,
+ "id": 3,
+ "isNew": true,
+ "legend": {
+ "alignAsTable": false,
+ "avg": true,
+ "current": true,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "How replication works",
+ "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "/^max.+/",
+ "color": "#F2495C"
+ },
+ {
+ "alias": "/^check.+/",
+ "color": "#FF9830"
+ },
+ {
+ "alias": "/^fetch.+/",
+ "color": "#B877D9"
+ },
+ {
+ "alias": "/^send.+/",
+ "color": "#5794F2"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_ReplicasMaxQueueSize{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "legendFormat": "max queue size {{hostname}}",
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_ReplicatedChecks{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "intervalFactor": 2,
+ "legendFormat": "check {{hostname}}",
+ "refId": "A",
+ "step": 20
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_ReplicatedFetch{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "interval": "",
+ "intervalFactor": 2,
+ "legendFormat": "fetch {{hostname}}",
+ "refId": "B",
+ "step": 20
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_ReplicatedSend{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "intervalFactor": 2,
+ "legendFormat": "send {{hostname}}",
+ "refId": "C",
+ "step": 20
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Replication Queue Jobs",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Show seconds when replicated servers can be delayed relative to current time, when you insert directly in *ReplicatedMegreTree table on one server clickhouse need time to replicate new parts of data to another servers in same shard in background",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 25
+ },
+ "hiddenSeries": false,
+ "id": 7,
+ "isNew": true,
+ "legend": {
+ "avg": true,
+ "current": true,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "Replication architecture",
+ "url": "https://clickhouse.tech/docs/en/development/architecture/#replication"
+ },
+ {
+ "targetBlank": true,
+ "title": "ReplicatedMergeTree engine",
+ "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/replication/"
+ },
+ {
+ "targetBlank": true,
+ "title": "max_replica_delay_for_distributed_queries",
+ "url": "https://clickhouse.tech/docs/en/operations/settings/settings/#settings-max_replica_delay_for_distributed_queries"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "/^absolute.+/",
+ "color": "#F2495C"
+ },
+ {
+ "alias": "/^relative.+/",
+ "color": "#FADE2A"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_ReplicasMaxAbsoluteDelay{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "intervalFactor": 2,
+ "legendFormat": "absolute {{hostname}}",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_ReplicasMaxRelativeDelay{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "intervalFactor": 2,
+ "legendFormat": "relative {{hostname}}",
+ "refId": "B",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Max Replica Delay",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Number of requests to ZooKeeper transactions per seconds.",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 25
+ },
+ "hiddenSeries": false,
+ "id": 34,
+ "isNew": true,
+ "legend": {
+ "avg": true,
+ "current": true,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "Replication architecture",
+ "url": "https://clickhouse.tech/docs/en/development/architecture/#replication"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_ZooKeeperTransactions{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "transactions {{ hostname }}",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_ZooKeeperRequest{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "hide": true,
+ "legendFormat": "{{ hostname }}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Zookeeper Transactions",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Show how intensive background merge processes",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 32
+ },
+ "hiddenSeries": false,
+ "id": 2,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": true,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "START/STOP Merges",
+ "url": "https://clickhouse.tech/docs/en/sql-reference/statements/system/#query_language-system-stop-merges"
+ },
+ {
+ "targetBlank": true,
+ "title": "MegreTree Engine description",
+ "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": true,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_Merge{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "intervalFactor": 2,
+ "legendFormat": "merges {{hostname}}",
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Merges",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Show how intensive background merge processes",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 32
+ },
+ "hiddenSeries": false,
+ "id": 36,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": true,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "START/STOP Merges",
+ "url": "https://clickhouse.tech/docs/en/sql-reference/statements/system/#query_language-system-stop-merges"
+ },
+ {
+ "targetBlank": true,
+ "title": "MegreTree Engine description",
+ "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": true,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_MergedRows{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "intervalFactor": 2,
+ "legendFormat": "rows {{hostname}}",
+ "refId": "B",
+ "step": 4
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Merged Rows",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Show how intensive background merge processes",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 32
+ },
+ "hiddenSeries": false,
+ "id": 49,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": true,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "START/STOP Merges",
+ "url": "https://clickhouse.tech/docs/en/sql-reference/statements/system/#query_language-system-stop-merges"
+ },
+ {
+ "targetBlank": true,
+ "title": "MegreTree Engine description",
+ "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": true,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_MergedUncompressedBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "intervalFactor": 2,
+ "legendFormat": "bytes {{hostname}}",
+ "refId": "B",
+ "step": 4
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Merged Uncompressed Bytes",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 2,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "decbytes",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "decimals": 0,
+ "description": "",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 39
+ },
+ "hiddenSeries": false,
+ "id": 23,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": true,
+ "min": true,
+ "show": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "system.parts",
+ "url": "https://clickhouse.tech/docs/en/operations/system-tables/#system_tables-parts"
+ },
+ {
+ "targetBlank": true,
+ "title": "parts_to_delay_insert",
+ "url": "https://github.com/ClickHouse/ClickHouse/search?q=parts_to_delay_insert"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "sum by(hostname) (chi_clickhouse_table_parts{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\",active=\"1\"})",
+ "legendFormat": "Parts {{hostname}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Active Parts",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "decimals": 0,
+ "description": "",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 39
+ },
+ "hiddenSeries": false,
+ "id": 50,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": true,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": true,
+ "min": true,
+ "show": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "system.parts",
+ "url": "https://clickhouse.tech/docs/en/operations/system-tables/#system_tables-parts"
+ },
+ {
+ "targetBlank": true,
+ "title": "parts_to_delay_insert",
+ "url": "https://github.com/ClickHouse/ClickHouse/search?q=parts_to_delay_insert"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "sum by(hostname) (chi_clickhouse_table_parts{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\",active=\"0\"})",
+ "legendFormat": "Parts {{hostname}}",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Inactive Parts",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Each logical partition defined over `PARTITION BY` contains few physical data \"parts\" ",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 39
+ },
+ "hiddenSeries": false,
+ "id": 4,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": true,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "Custom Partitioning Key",
+ "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key/"
+ },
+ {
+ "targetBlank": true,
+ "title": "system.parts",
+ "url": "https://clickhouse.tech/docs/en/operations/system-tables/#system_tables-parts"
+ },
+ {
+ "targetBlank": true,
+ "title": "system.part_log",
+ "url": "https://clickhouse.tech/docs/en/operations/system-tables/#system_tables-part-log"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_MaxPartCountForPartition{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "intervalFactor": 2,
+ "legendFormat": "{{hostname}}",
+ "refId": "A",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Max Part count for Partition",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Memory size allocated for clickhouse-server process\nAvailable for ClickHouse 20.4+\n\nVIRT \nThe total amount of virtual memory used by the task. It includes all code, data and shared libraries plus pages that have been swapped out.\n\nVIRT = SWAP + RES\n\n\nSWAP -- Swapped size (kb)\nThe swapped out portion of a task's total virtual memory image.\n\nRES -- Resident size (kb)\nThe non-swapped physical memory a task has used.\nRES = CODE + USED DATA.\n\nCODE -- Code size (kb)\nThe amount of physical memory devoted to executable code, also known as the 'text resident set' size or TRS\n\nDATA -- Data+Stack size (kb)\nThe amount of physical memory allocated to other than executable code, also known as the 'data resident set' size or DRS.\n\nSHR -- Shared Mem size (kb)\nThe amount of shared memory used by a task. It simply reflects memory that could be potentially shared with other processes.",
+ "fill": 1,
+ "fillGradient": 2,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 46
+ },
+ "hiddenSeries": false,
+ "id": 46,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "Describe Linux Process Memory types",
+ "url": "https://elinux.org/Runtime_Memory_Measurement"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "/VIRT.+/",
+ "color": "#73BF69"
+ },
+ {
+ "alias": "/DATA.+/",
+ "color": "#C4162A"
+ },
+ {
+ "alias": "/CODE.+/",
+ "color": "#FF9830"
+ },
+ {
+ "alias": "/RES.+/",
+ "color": "#FADE2A"
+ },
+ {
+ "alias": "/SHR.+/",
+ "color": "#5794F2"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": true,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_MemoryCode{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "legendFormat": "CODE {{ hostname }}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_MemoryResident{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "legendFormat": "RES {{ hostname }}",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_MemoryShared{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "legendFormat": "SHR {{ hostname }}",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_MemoryDataAndStack{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "legendFormat": "DATA {{ hostname }}",
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_MemoryVirtual{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "legendFormat": "VIRT {{ hostname }}",
+ "refId": "E"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": " clickhouse-server Process Memory",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "decbytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Memory size allocated for primary keys",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 46
+ },
+ "hiddenSeries": false,
+ "id": 45,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "How to choose right primary key",
+ "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/#selecting-the-primary-key"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_MemoryPrimaryKeyBytesAllocated{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "legendFormat": "{{ hostname }}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Primary Keys Memory",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Memory size allocated for dictionaries",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 46
+ },
+ "hiddenSeries": false,
+ "id": 43,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "system.dictionaries",
+ "url": "https://clickhouse.tech/docs/en/operations/system-tables/#system_tables-dictionaries"
+ },
+ {
+ "targetBlank": true,
+ "title": "CREATE DICTIONARY",
+ "url": "https://clickhouse.tech/docs/en/sql-reference/statements/create/#create-dictionary-query"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_MemoryDictionaryBytesAllocated{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "legendFormat": "{{ hostname }}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Dictionary Memory",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "decbytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "shows how much space available in the kubernetes pod\n\nbe careful with multiple volumes configuration, kubernetes volume claims and S3 as storage backend",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 53
+ },
+ "hiddenSeries": false,
+ "id": 39,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "system.disks",
+ "url": "https://clickhouse.tech/docs/en/operations/system-tables/disks/"
+ },
+ {
+ "targetBlank": true,
+ "title": "Multiple Disk Volumes",
+ "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/#table_engine-mergetree-multiple-volumes"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_DiskFreeBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"} / chi_clickhouse_metric_DiskTotalBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "legendFormat": "{{ disk }} {{hostname}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Disk Space Free",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "logBase": 1,
+ "max": "1",
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Total data size for all ClickHouse *MergeTree tables\n\n",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 53
+ },
+ "hiddenSeries": false,
+ "id": 41,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "system.parts",
+ "url": "https://clickhouse.tech/docs/en/operations/system-tables/#system_tables-parts"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_DiskDataBytes{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "legendFormat": "{{ hostname }}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Clickhouse Data size on Disk",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "decbytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Show different types of connections for each server",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 53
+ },
+ "hiddenSeries": false,
+ "id": 48,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "max_connections",
+ "url": "https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#max-connections"
+ },
+ {
+ "targetBlank": true,
+ "title": "max_distributed_connections",
+ "url": "https://clickhouse.tech/docs/en/operations/settings/settings/#max-distributed-connections"
+ },
+ {
+ "targetBlank": true,
+ "title": "MySQL Protocol",
+ "url": "https://clickhouse.tech/docs/en/interfaces/mysql/"
+ },
+ {
+ "targetBlank": true,
+ "title": "HTTP Protocol",
+ "url": "https://clickhouse.tech/docs/en/interfaces/http/"
+ },
+ {
+ "targetBlank": true,
+ "title": "Native Protocol",
+ "url": "https://clickhouse.tech/docs/en/interfaces/tcp/"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": true,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_TCPConnection{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "intervalFactor": 2,
+ "legendFormat": "tcp {{hostname}}",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_HTTPConnection{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "intervalFactor": 2,
+ "legendFormat": "http {{hostname}}",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_InterserverConnection{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "intervalFactor": 2,
+ "legendFormat": "interserver {{hostname}}",
+ "refId": "C",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_MySQLConnection{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "intervalFactor": 2,
+ "legendFormat": "mysql {{hostname}}",
+ "refId": "D",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Connections",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "BackgroundPoolTask\t\n---\nNumber of active tasks in BackgroundProcessingPool (merges, mutations, fetches, or replication queue bookkeeping)\n\n\nBackgroundMovePoolTask\n---\nNumber of active tasks in BackgroundProcessingPool for moves\n\n\nBackgroundSchedulePoolTask\t\n---\nA number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc.",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 60
+ },
+ "hiddenSeries": false,
+ "id": 9,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": true,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "FETCH PARTITION",
+ "url": "https://clickhouse.tech/docs/en/sql-reference/statements/alter/partition/#alter_fetch-partition"
+ },
+ {
+ "targetBlank": true,
+ "title": "Mutations of data",
+ "url": "https://clickhouse.tech/docs/en/sql-reference/statements/alter/#mutations"
+ },
+ {
+ "targetBlank": true,
+ "title": "Data TTL",
+ "url": "https://clickhouse.tech/docs/en/engines/table-engines/mergetree-family/mergetree/#table_engine-mergetree-ttl"
+ },
+ {
+ "targetBlank": true,
+ "title": "MOVE PARTITION",
+ "url": "https://clickhouse.tech/docs/en/sql-reference/statements/alter/partition/#alter_move-partition"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": true,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_BackgroundPoolTask{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "intervalFactor": 2,
+ "legendFormat": "merge, mutate, fetch {{hostname}}",
+ "refId": "A",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_BackgroundSchedulePoolTask{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "intervalFactor": 2,
+ "legendFormat": "clean, alter, replica re-init {{hostname}}",
+ "refId": "B",
+ "step": 10
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "chi_clickhouse_metric_BackgroundMovePoolTask{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}",
+ "intervalFactor": 2,
+ "legendFormat": "moves {{hostname}}",
+ "refId": "C",
+ "step": 10
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Background Tasks",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Number of active mutations (ALTER DELETE/ALTER UPDATE) and parts to mutate",
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 60
+ },
+ "hiddenSeries": false,
+ "id": 26,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "Mutations",
+ "url": "https://clickhouse.tech/docs/en/sql-reference/statements/alter/#mutations"
+ },
+ {
+ "targetBlank": true,
+ "title": "system.mutations",
+ "url": "https://clickhouse.tech/docs/en/operations/system-tables/mutations/"
+ },
+ {
+ "targetBlank": true,
+ "title": "KILL MUTATION",
+ "url": "https://clickhouse.tech/docs/en/sql-reference/statements/kill/#kill-mutation"
+ }
+ ],
+ "nullPointMode": "null as zero",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": true,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "sum by (hostname) (chi_clickhouse_table_mutations{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"})",
+ "legendFormat": "mutations {{hostname}}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "sum by (hostname) (chi_clickhouse_table_mutations_parts_to_do{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"})",
+ "legendFormat": "parts_to_do {{hostname}}",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Mutations",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "Show which percent of mark files (.mrk) read from memory instead of disk",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 60
+ },
+ "hiddenSeries": false,
+ "id": 11,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "mark_cache_size",
+ "url": "https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-mark-cache-size"
+ },
+ {
+ "targetBlank": true,
+ "title": "MergeTree architecture",
+ "url": "https://clickhouse.tech/docs/en/development/architecture/#merge-tree"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_MarkCacheHits{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) / (irate(chi_clickhouse_event_MarkCacheHits{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) + irate(chi_clickhouse_event_MarkCacheMisses{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]))",
+ "hide": false,
+ "intervalFactor": 2,
+ "legendFormat": "{{hostname}}",
+ "refId": "A",
+ "step": 4
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Marks Cache Hit Rate",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "percentunit",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "The time which CPU spent on various types of activity ",
+ "fill": 1,
+ "fillGradient": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 67
+ },
+ "hiddenSeries": false,
+ "id": 51,
+ "interval": "",
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 2,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "/^Disk Read.+/",
+ "color": "#FF9830"
+ },
+ {
+ "alias": "/^Disk Write.+/",
+ "color": "#E0B400"
+ },
+ {
+ "alias": "/^Real Time.+/",
+ "color": "#73BF69"
+ },
+ {
+ "alias": "/^User Time.+/",
+ "color": "#FFF899"
+ },
+ {
+ "alias": "/^System Time.+/",
+ "color": "#F2495C"
+ },
+ {
+ "alias": "/^OS IO Wait.+/",
+ "color": "#C4162A"
+ },
+ {
+ "alias": "/^OS CPU Wait.+/",
+ "color": "rgb(95, 29, 29)"
+ },
+ {
+ "alias": "/^OS CPU Virtual.+/",
+ "color": "#B877D9"
+ },
+ {
+ "alias": "/^Network Receive.+/",
+ "color": "#C0D8FF"
+ },
+ {
+ "alias": "/^Network Send.+/",
+ "color": "#8AB8FF"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": true,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_DiskReadElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "hide": true,
+ "legendFormat": "Disk Read syscall {{hostname}}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_DiskWriteElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "hide": true,
+ "legendFormat": "Disk Write syscall {{hostname}}",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_NetworkReceiveElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "hide": true,
+ "legendFormat": "Network Receive {{hostname}}",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_NetworkSendElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "hide": true,
+ "legendFormat": "Network Send {{hostname}}",
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_RealTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "Real Time {{hostname}}",
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_UserTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "User Time {{hostname}}",
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_SystemTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "System Time {{hostname}}",
+ "refId": "G"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_OSIOWaitMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "OS IO Wait {{hostname}}",
+ "refId": "H"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_OSCPUWaitMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "OS CPU Wait {{hostname}}",
+ "refId": "I"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(chi_clickhouse_event_OSCPUVirtualTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "OS CPU Virtual {{hostname}}",
+ "refId": "J"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "CPU Time per second",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "µs",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "breakPoint": "50%",
+ "combine": {
+ "label": "Others",
+ "threshold": "0.01"
+ },
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "The time which CPU spent on various types of activity total for the selected period",
+ "fontSize": "80%",
+ "format": "µs",
+ "gridPos": {
+ "h": 7,
+ "w": 16,
+ "x": 8,
+ "y": 67
+ },
+ "id": 52,
+ "interval": "1m",
+ "legend": {
+ "header": "",
+ "percentage": true,
+ "show": true,
+ "sort": "total",
+ "sortDesc": true,
+ "values": true
+ },
+ "legendType": "Right side",
+ "links": [],
+ "nullPointMode": "connected",
+ "pieType": "pie",
+ "strokeWidth": "",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_DiskReadElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "Disk Read syscall {{hostname}}",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_DiskWriteElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "Disk Write syscall {{hostname}}",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_NetworkReceiveElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "Network Receive {{hostname}}",
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_NetworkSendElapsedMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "Network Send {{hostname}}",
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_RealTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "Real Time {{hostname}}",
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_UserTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "User Time {{hostname}}",
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_SystemTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "System Time {{hostname}}",
+ "refId": "G"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_OSIOWaitMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "OS IO Wait {{hostname}}",
+ "refId": "H"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_OSCPUWaitMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "OS CPU Wait {{hostname}}",
+ "refId": "I"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_OSCPUVirtualTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "OS CPU Virtual {{hostname}}",
+ "refId": "J"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_ThrottlerSleepMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "Throttler Sleep {{hostname}}",
+ "refId": "K"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_DelayedInsertsMilliseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000",
+ "legendFormat": "Delayed Insert {{hostname}}",
+ "refId": "L"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_ZooKeeperWaitMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "Zookeeper Wait {{hostname}}",
+ "refId": "M"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_CompileExpressionsMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "Compile Expressions {{hostname}}",
+ "refId": "N"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_MergesTimeMilliseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000",
+ "legendFormat": "Merges {{hostname}}",
+ "refId": "O"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_RWLockReadersWaitMilliseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000",
+ "legendFormat": "RWLock Reader Wait {{hostname}}",
+ "refId": "P"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_RWLockWritersWaitMilliseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m]) * 1000",
+ "legendFormat": "RWLock Writer Wait {{hostname}}",
+ "refId": "Q"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_SelectQueryTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "Select Query {{hostname}}",
+ "refId": "R"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_InsertQueryTimeMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "Insert Query {{hostname}}",
+ "refId": "S"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_S3ReadMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "S3 Read {{hostname}}",
+ "refId": "T"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "increase(chi_clickhouse_event_S3WriteMicroseconds{exported_namespace=~\"$exported_namespace\",chi=~\"$chi\",hostname=~\"$hostname\"}[1m])",
+ "legendFormat": "S3 Write {{hostname}}",
+ "refId": "U"
+ }
+ ],
+ "title": "CPU Time total",
+ "type": "grafana-piechart-panel",
+ "valueName": "total"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "description": "",
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "fillGradient": 0,
+ "grid": {},
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 74
+ },
+ "hiddenSeries": false,
+ "id": 24,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "hideEmpty": false,
+ "hideZero": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [
+ {
+ "targetBlank": true,
+ "title": "Howto show detail statsistic on grafana for golang process",
+ "url": "https://grafana.com/grafana/dashboards/6671"
+ }
+ ],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "irate(go_memstats_alloc_bytes_total{app=\"clickhouse-operator\",namespace=~\"$exported_namespace|kube-system\"}[1m])",
+ "hide": false,
+ "legendFormat": "{{ namespace }} GO malloc bytes / sec",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "expr": "process_resident_memory_bytes{app=\"clickhouse-operator\",namespace=~\"$exported_namespace|kube-system\"}",
+ "legendFormat": "{{ namespace }} RSS Memory",
+ "refId": "B"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Monitoring Agent",
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "refresh": "1m",
+ "schemaVersion": 36,
+ "style": "dark",
+ "tags": [
+ "Altinity",
+ "clickhouse",
+ "operator"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, exported_namespace)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "K8S Namespace",
+ "multi": true,
+ "name": "exported_namespace",
+ "options": [],
+ "query": {
+ "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, exported_namespace)",
+ "refId": "Prometheus-exported_namespace-Variable-Query"
+ },
+ "refresh": 2,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\", exported_namespace=~\"$exported_namespace\"}, chi)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "K8S Clickhouse Installation",
+ "multi": true,
+ "name": "chi",
+ "options": [],
+ "query": {
+ "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\", exported_namespace=~\"$exported_namespace\"}, chi)",
+ "refId": "Prometheus-chi-Variable-Query"
+ },
+ "refresh": 2,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\",exported_namespace=~\"$exported_namespace\",chi=~\"$chi\"}, hostname)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "Server",
+ "multi": true,
+ "name": "hostname",
+ "options": [],
+ "query": {
+ "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\",exported_namespace=~\"$exported_namespace\",chi=~\"$chi\"}, hostname)",
+ "refId": "Prometheus-hostname-Variable-Query"
+ },
+ "refresh": 2,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-30m",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "browser",
+ "title": "Altinity ClickHouse Operator Dashboard",
+ "uid": "clickhouse-operator-dashboard",
+ "version": 1,
+ "weekStart": ""
+}
diff --git a/ee/scripts/helmcharts/manifests/observability/dashboards/clickhouse/altinity-clickhouse-queries-dashboard.json b/ee/scripts/helmcharts/manifests/observability/dashboards/clickhouse/altinity-clickhouse-queries-dashboard.json
new file mode 100644
index 000000000..de9346844
--- /dev/null
+++ b/ee/scripts/helmcharts/manifests/observability/dashboards/clickhouse/altinity-clickhouse-queries-dashboard.json
@@ -0,0 +1,1188 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "target": {
+ "limit": 100,
+ "matchAny": false,
+ "tags": [],
+ "type": "dashboard"
+ },
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "Helps to visualize most frequent, slowest, failed queries.\r\nShows queries rate per second, table with last queries",
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "gnetId": 2515,
+ "graphTooltip": 1,
+ "id": 16,
+ "iteration": 1656598972006,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "collapsed": false,
+ "datasource": {
+ "uid": "$db"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 22,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$db"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Top charts",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$db"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 20,
+ "x": 0,
+ "y": 1
+ },
+ "hiddenSeries": false,
+ "id": 15,
+ "legend": {
+ "alignAsTable": true,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": true,
+ "show": true,
+ "sort": "avg",
+ "sortDesc": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "9.0.1",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "database": "",
+ "datasource": {
+ "uid": "$db"
+ },
+ "dateColDataType": "event_date",
+ "dateLoading": false,
+ "dateTimeColDataType": "event_time",
+ "dateTimeType": "DATETIME",
+ "datetimeLoading": false,
+ "expr": "",
+ "format": "time_series",
+ "formattedQuery": "SELECT * FROM ($rateColumns(\n substring(query, 1, 45) AS query,\n count() c)\nFROM $table\nWHERE\n cityHash64(query) global in (\n SELECT cityHash64(substring(query, 1, 45)) AS h\n FROM $table\n WHERE\n $timeFilter\n AND type in ($type)\n AND initial_user in ($user)\n AND('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))\n GROUP BY h\n ORDER BY count() desc\n LIMIT $top)\n AND type in ($type)\n AND initial_user in ($user)\n AND('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))) SETTINGS skip_unavailable_shards=1",
+ "intervalFactor": 2,
+ "query": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((query, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n substring(query, 1, 45) AS query,\r\n count() c\r\n FROM $table\r\n WHERE $timeFilter\r\n AND cityHash64(query) GLOBAL IN (\r\n SELECT cityHash64(substring(query, 1, 45)) AS h\r\n FROM $table\r\n WHERE $timeFilter\r\n AND type IN ($type)\r\n AND initial_user IN ($user)\r\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT $top\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n AND type IN ($type)\r\n AND initial_user IN ($user)\r\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1",
+ "rawQuery": "SELECT t, arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr) FROM ( SELECT t, groupArray((query, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t, substring(query, 1, 45) AS query, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586192379) AND event_time >= toDateTime(1586192379) AND cityHash64(query) GLOBAL IN ( SELECT cityHash64(substring(query, 1, 45)) AS h FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586192379) AND event_time >= toDateTime(1586192379) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY h ORDER BY count() DESC LIMIT 5 SETTINGS skip_unavailable_shards=1 ) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t, query ORDER BY t ) GROUP BY t ORDER BY t ) SETTINGS skip_unavailable_shards=1",
+ "refId": "A",
+ "resultFormat": "time_series",
+ "round": "0s",
+ "table": "cluster('all-sharded',system.query_log)",
+ "tableLoading": false
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Top $top request's rate by type: $type; user: $user; query type: $query_type",
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "uid": "$db"
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 4,
+ "x": 20,
+ "y": 1
+ },
+ "id": 17,
+ "links": [],
+ "options": {
+ "content": "1 - successful start of query execution\n\n2 - successful end of query execution\n\n3 - exception before start of query execution\n\n4 - exception while query execution",
+ "mode": "markdown"
+ },
+ "pluginVersion": "9.0.1",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$db"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Types",
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "columns": [],
+ "datasource": {
+ "uid": "$db"
+ },
+ "fontSize": "100%",
+ "gridPos": {
+ "h": 10,
+ "w": 8,
+ "x": 0,
+ "y": 8
+ },
+ "height": "400px",
+ "id": 18,
+ "links": [],
+ "scroll": true,
+ "showHeader": true,
+ "sort": {
+ "col": 2,
+ "desc": true
+ },
+ "styles": [
+ {
+ "alias": "Time",
+ "align": "auto",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "t",
+ "type": "hidden"
+ },
+ {
+ "alias": "duration",
+ "align": "auto",
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "pattern": "duration",
+ "thresholds": [],
+ "type": "number",
+ "unit": "ms"
+ },
+ {
+ "alias": "",
+ "align": "auto",
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "count",
+ "thresholds": [],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "",
+ "align": "auto",
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [],
+ "type": "number",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "database": "",
+ "datasource": {
+ "uid": "$db"
+ },
+ "dateColDataType": "event_date",
+ "dateLoading": false,
+ "dateTimeColDataType": "event_time",
+ "dateTimeType": "DATETIME",
+ "datetimeLoading": false,
+ "expr": "",
+ "format": "table",
+ "formattedQuery": "SELECT
rand() as t,
substring(query, 1, 70) AS query,
avg(query_duration_ms) duration,
count() count
FROM $table
WHERE
$timeFilter
AND type in ($type)
and initial_user in ($user)
and('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))
GROUP BY query
ORDER BY duration desc
LIMIT $top",
+ "intervalFactor": 2,
+ "query": "SELECT\n rand() as t,\n substring(query, 1, 70) AS query,\n avg(query_duration_ms) duration,\n count() count\nFROM $table\nWHERE\n $timeFilter\n AND type in ($type)\n and initial_user in ($user)\n and('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))\nGROUP BY query\nORDER BY duration desc\nLIMIT $top",
+ "rawQuery": "SELECT t, arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr) FROM ( SELECT t, groupArray((query, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t, substring(query, 1, 45) AS query, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234275) AND event_time >= toDateTime(1586234275) AND cityHash64(query) GLOBAL IN ( SELECT cityHash64(substring(query, 1, 45)) AS h FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234275) AND event_time >= toDateTime(1586234275) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY h ORDER BY count() DESC LIMIT 5 SETTINGS skip_unavailable_shards=1 ) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t, query ORDER BY t ) GROUP BY t ORDER BY t ) SETTINGS skip_unavailable_shards=1",
+ "refId": "A",
+ "resultFormat": "time_series",
+ "round": "0s",
+ "table": "cluster('all-sharded',system.query_log)",
+ "tableLoading": false
+ }
+ ],
+ "title": "Top slow queries by type: $type; user: $user; query type: $query_type",
+ "transform": "table",
+ "type": "table-old"
+ },
+ {
+ "columns": [],
+ "datasource": {
+ "uid": "$db"
+ },
+ "fontSize": "100%",
+ "gridPos": {
+ "h": 10,
+ "w": 8,
+ "x": 8,
+ "y": 8
+ },
+ "height": "400px",
+ "id": 19,
+ "links": [],
+ "scroll": true,
+ "showHeader": true,
+ "sort": {
+ "col": 2,
+ "desc": true
+ },
+ "styles": [
+ {
+ "alias": "Time",
+ "align": "auto",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "t",
+ "type": "hidden"
+ },
+ {
+ "alias": "usage",
+ "align": "auto",
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "pattern": "usage",
+ "thresholds": [],
+ "type": "number",
+ "unit": "bytes"
+ },
+ {
+ "alias": "",
+ "align": "auto",
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "count",
+ "thresholds": [],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "",
+ "align": "auto",
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [],
+ "type": "number",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "database": "",
+ "datasource": {
+ "uid": "$db"
+ },
+ "dateColDataType": "event_date",
+ "dateLoading": false,
+ "dateTimeColDataType": "event_time",
+ "dateTimeType": "DATETIME",
+ "datetimeLoading": false,
+ "expr": "",
+ "format": "table",
+ "formattedQuery": "SELECT
rand() as t,
substring(query, 1, 70) AS query,
avg(memory_usage) usage,
count() count
FROM $table
WHERE
$timeFilter
AND type in ($type)
and initial_user in ($user)
and('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))
GROUP BY query
ORDER BY usage desc
LIMIT $top",
+ "intervalFactor": 2,
+ "query": "SELECT\n rand() as t,\n substring(query, 1, 70) AS query,\n avg(memory_usage) usage,\n count() count\nFROM $table\nWHERE\n $timeFilter\n AND type IN ($type)\n AND initial_user IN ($user)\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\nGROUP BY query\nORDER BY usage DESC\nLIMIT $top",
+ "rawQuery": "SELECT t, arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr) FROM ( SELECT t, groupArray((query, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t, substring(query, 1, 45) AS query, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234300) AND event_time >= toDateTime(1586234300) AND cityHash64(query) GLOBAL IN ( SELECT cityHash64(substring(query, 1, 45)) AS h FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234300) AND event_time >= toDateTime(1586234300) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY h ORDER BY count() DESC LIMIT 5 SETTINGS skip_unavailable_shards=1 ) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t, query ORDER BY t ) GROUP BY t ORDER BY t ) SETTINGS skip_unavailable_shards=1",
+ "refId": "A",
+ "resultFormat": "time_series",
+ "round": "0s",
+ "table": "cluster('all-sharded',system.query_log)",
+ "tableLoading": false
+ }
+ ],
+ "title": "Top memory consumers by type: $type; user: $user; query type: $query_type",
+ "transform": "table",
+ "type": "table-old"
+ },
+ {
+ "columns": [],
+ "datasource": {
+ "uid": "$db"
+ },
+ "fontSize": "100%",
+ "gridPos": {
+ "h": 10,
+ "w": 8,
+ "x": 16,
+ "y": 8
+ },
+ "height": "400px",
+ "id": 20,
+ "links": [],
+ "scroll": true,
+ "showHeader": true,
+ "sort": {
+ "col": 3,
+ "desc": true
+ },
+ "styles": [
+ {
+ "alias": "Time",
+ "align": "auto",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "t",
+ "type": "hidden"
+ },
+ {
+ "alias": "type",
+ "align": "auto",
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "type",
+ "thresholds": [],
+ "type": "number",
+ "unit": "none"
+ },
+ {
+ "alias": "",
+ "align": "auto",
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "count",
+ "thresholds": [],
+ "type": "number",
+ "unit": "short"
+ },
+ {
+ "alias": "",
+ "align": "auto",
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [],
+ "type": "number",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "database": "",
+ "datasource": {
+ "uid": "$db"
+ },
+ "dateColDataType": "event_date",
+ "dateLoading": false,
+ "dateTimeColDataType": "event_time",
+ "dateTimeType": "DATETIME",
+ "datetimeLoading": false,
+ "expr": "",
+ "format": "table",
+ "formattedQuery": "SELECT
rand() as t,
substring(query, 1, 70) AS query,
type,
count() count
FROM $table
WHERE
$timeFilter
AND type in (3,4)
and initial_user in ($user)
and('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))
GROUP BY
query,
type
ORDER BY count desc
LIMIT $top",
+ "intervalFactor": 2,
+ "query": "SELECT\n rand() as t,\n substring(query, 1, 70) AS query,\n type,\n count() count\nFROM $table\nWHERE\n $timeFilter\n AND type IN (3,4)\n AND initial_user IN ($user)\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\nGROUP BY\n query,\n type\nORDER BY count desc\nLIMIT $top",
+ "rawQuery": "SELECT t, arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr) FROM ( SELECT t, groupArray((query, c)) AS groupArr FROM ( SELECT (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t, substring(query, 1, 45) AS query, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234335) AND event_time >= toDateTime(1586234335) AND cityHash64(query) GLOBAL IN ( SELECT cityHash64(substring(query, 1, 45)) AS h FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586234335) AND event_time >= toDateTime(1586234335) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY h ORDER BY count() DESC LIMIT 5 SETTINGS skip_unavailable_shards=1 ) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t, query ORDER BY t ) GROUP BY t ORDER BY t ) SETTINGS skip_unavailable_shards=1",
+ "refId": "A",
+ "resultFormat": "time_series",
+ "round": "0s",
+ "table": "cluster('all-sharded',system.query_log)",
+ "tableLoading": false
+ }
+ ],
+ "title": "Top failed queries by user: $user; query type: $query_type",
+ "transform": "table",
+ "type": "table-old"
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "uid": "$db"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 18
+ },
+ "id": 23,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$db"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Request charts",
+ "type": "row"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$db"
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 0,
+ "y": 19
+ },
+ "hiddenSeries": false,
+ "id": 14,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "database": "",
+ "datasource": {
+ "uid": "$db"
+ },
+ "dateColDataType": "event_date",
+ "dateLoading": false,
+ "dateTimeColDataType": "event_time",
+ "dateTimeType": "DATETIME",
+ "datetimeLoading": false,
+ "expr": "",
+ "format": "time_series",
+ "formattedQuery": "$rate(count() c)
FROM $table
where type in ($type)
and initial_user in ($user)
and('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))",
+ "interval": "",
+ "intervalFactor": 2,
+ "query": "$rate(count() c)\nFROM $table\nWHERE type IN($type)\n AND initial_user IN ($user)\n AND ('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))",
+ "rawQuery": "SELECT t, c/runningDifference(t/1000) cRate FROM ( SELECT (intDiv(toUInt32(event_time), 120) * 120) * 1000 AS t, count() c FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586192459) AND event_time >= toDateTime(1586192459) AND type IN (1,2,3,4) AND initial_user IN ('default') AND ('all' = 'all' OR (positionCaseInsensitive(query, 'all') = 1)) GROUP BY t ORDER BY t)",
+ "refId": "A",
+ "resultFormat": "time_series",
+ "round": "0s",
+ "table": "cluster('all-sharded',system.query_log)",
+ "tableLoading": false
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Reqs/s by type: $type; user: $user; query type: $query_type",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$db"
+ },
+ "fill": 1,
+ "fillGradient": 1,
+ "gridPos": {
+ "h": 7,
+ "w": 12,
+ "x": 12,
+ "y": 19
+ },
+ "hiddenSeries": false,
+ "id": 16,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null as zero",
+ "options": {
+ "dataLinks": []
+ },
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "alias": "/insert.+/",
+ "yaxis": 2
+ },
+ {
+ "alias": "select_quantile_50",
+ "color": "#FADE2A"
+ },
+ {
+ "alias": "select_quantile_90",
+ "color": "#73BF69"
+ },
+ {
+ "alias": "insert_quantile_50",
+ "color": "#FA6400"
+ },
+ {
+ "alias": "insert_quantile_90",
+ "color": "#C4162A"
+ }
+ ],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": true,
+ "targets": [
+ {
+ "database": "",
+ "datasource": {
+ "uid": "$db"
+ },
+ "dateColDataType": "event_date",
+ "dateLoading": false,
+ "dateTimeColDataType": "event_time",
+ "dateTimeType": "DATETIME",
+ "datetimeLoading": false,
+ "expr": "",
+ "format": "time_series",
+ "formattedQuery": "SELECT
$timeSeries as t,
avg(query_duration_ms) select_duration
FROM $table
WHERE
$timeFilter
AND type = 2
and positionCaseInsensitive(query, 'select') = 1
and initial_user in ($user)
GROUP BY t
ORDER BY t",
+ "intervalFactor": 2,
+ "query": "SELECT\n $timeSeries as t,\n quantile(0.5)(query_duration_ms) select_quantile_50,\n quantile(0.9)(query_duration_ms) select_quantile_90\nFROM $table\nWHERE\n $timeFilter\n AND type = 2\n AND positionCaseInsensitive(query, 'select') = 1\n AND initial_user IN ($user)\nGROUP BY t\nORDER BY t",
+ "rawQuery": "SELECT (intDiv(toUInt32(event_time), 10) * 10) * 1000 as t, quantile(0.5)(query_duration_ms) select_quantile_50, quantile(0.9)(query_duration_ms) select_quantile_90 FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1595573984) AND event_time >= toDateTime(1595573984) AND type = 2 AND positionCaseInsensitive(query, 'select') = 1 AND initial_user IN ('default') GROUP BY t ORDER BY t",
+ "refId": "A",
+ "resultFormat": "time_series",
+ "round": "0s",
+ "table": "cluster('all-sharded',system.query_log)",
+ "tableLoading": false
+ },
+ {
+ "database": "",
+ "datasource": {
+ "uid": "$db"
+ },
+ "dateColDataType": "event_date",
+ "dateLoading": false,
+ "dateTimeColDataType": "event_time",
+ "dateTimeType": "DATETIME",
+ "datetimeLoading": false,
+ "expr": "",
+ "format": "time_series",
+ "formattedQuery": "SELECT
$timeSeries as t,
avg(query_duration_ms) insert_duration
FROM $table
WHERE
$timeFilter
AND type = 2
and positionCaseInsensitive(query, 'insert into') = 1
and initial_user in ($user)
GROUP BY t
ORDER BY t",
+ "intervalFactor": 2,
+ "query": "SELECT\n $timeSeries as t,\n quantile(0.5)(query_duration_ms) insert_quantile_50,\n quantile(0.9)(query_duration_ms) insert_quantile_90\nFROM $table\nWHERE\n $timeFilter\n AND type = 2\nand positionCaseInsensitive(query, 'insert into') = 1\n and initial_user in ($user)\nGROUP BY t\nORDER BY t",
+ "rawQuery": "SELECT (intDiv(toUInt32(event_time), 10) * 10) * 1000 as t, quantile(0.5)(query_duration_ms) insert_quantile_50, quantile(0.9)(query_duration_ms) insert_quantile_90 FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1595573984) AND event_time >= toDateTime(1595573984) AND type = 2 and positionCaseInsensitive(query, 'insert into') = 1 and initial_user in ('default') GROUP BY t ORDER BY t",
+ "refId": "B",
+ "resultFormat": "time_series",
+ "round": "0s",
+ "table": "cluster('all-sharded',system.query_log)",
+ "tableLoading": false
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Query duration by type: $type; user: $user",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "ms",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "ms",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "uid": "$db"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 26
+ },
+ "id": 24,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$db"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "Query log table",
+ "type": "row"
+ },
+ {
+ "columns": [],
+ "datasource": {
+ "uid": "$db"
+ },
+ "fontSize": "100%",
+ "gridPos": {
+ "h": 7,
+ "w": 24,
+ "x": 0,
+ "y": 27
+ },
+ "id": 21,
+ "links": [],
+ "scroll": true,
+ "showHeader": true,
+ "sort": {
+ "col": 2,
+ "desc": true
+ },
+ "styles": [
+ {
+ "alias": "Time",
+ "align": "auto",
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "Time",
+ "type": "date"
+ },
+ {
+ "alias": "",
+ "align": "auto",
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "pattern": "duration",
+ "thresholds": [],
+ "type": "number",
+ "unit": "ms"
+ },
+ {
+ "alias": "",
+ "align": "auto",
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "dateFormat": "YYYY-MM-DD HH:mm:ss",
+ "decimals": 2,
+ "pattern": "memory",
+ "thresholds": [],
+ "type": "number",
+ "unit": "bytes"
+ },
+ {
+ "alias": "",
+ "align": "auto",
+ "colors": [
+ "rgba(245, 54, 54, 0.9)",
+ "rgba(237, 129, 40, 0.89)",
+ "rgba(50, 172, 45, 0.97)"
+ ],
+ "decimals": 2,
+ "pattern": "/.*/",
+ "thresholds": [],
+ "type": "number",
+ "unit": "short"
+ }
+ ],
+ "targets": [
+ {
+ "database": "",
+ "datasource": {
+ "uid": "$db"
+ },
+ "dateColDataType": "event_date",
+ "dateLoading": false,
+ "dateTimeColDataType": "event_time",
+ "dateTimeType": "DATETIME",
+ "datetimeLoading": false,
+ "format": "time_series",
+ "formattedQuery": "SELECT
event_time,
user,
query_duration_ms duration,
memory_usage memory,
if(exception!='', 'fail', 'success') result,
concat(substring(query,1,120), '...') query
FROM $table
WHERE $timeFilter
AND type in ($type)
and initial_user in ($user)
and('$query_type' = 'all' or(positionCaseInsensitive(query, '$query_type') = 1))
ORDER BY event_time DESC
LIMIT 1000",
+ "intervalFactor": 1,
+ "query": "SELECT\n event_time,\n hostName() host,\n user,\n query_duration_ms duration,\n memory_usage memory,\n if(exception!='', 'fail', 'success') result,\n concat(substring(query,1,120), '...') query\nFROM $table\nWHERE $timeFilter\n AND type in ($type)\n and initial_user in ($user)\n and('$query_type' = 'all' OR (positionCaseInsensitive(query, '$query_type') = 1))\nORDER BY event_time DESC\nLIMIT 1000",
+ "rawQuery": "SELECT (intDiv(toUInt32(event_time), 10) * 10) * 1000 as t, avg(query_duration_ms) select_duration FROM cluster('all-sharded',system.query_log) WHERE event_date >= toDate(1586232299) AND event_time >= toDateTime(1586232299) AND type = 2 and positionCaseInsensitive(query, 'select') = 1 and initial_user in ('default') GROUP BY t ORDER BY t",
+ "refId": "A",
+ "resultFormat": "time_series",
+ "round": "0s",
+ "table": "cluster('all-sharded',system.query_log)",
+ "tableLoading": false
+ }
+ ],
+ "title": "Query log by type: $type; user: $user; query type: $query_type",
+ "transform": "timeseries_to_columns",
+ "type": "table-old"
+ }
+ ],
+ "refresh": "5s",
+ "schemaVersion": 36,
+ "style": "dark",
+ "tags": [
+ "Altinity",
+ "clickhouse",
+ "queries",
+ "performance"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "k8s-db-openreplay-clickhouse",
+ "value": "k8s-db-openreplay-clickhouse"
+ },
+ "hide": 2,
+ "includeAll": false,
+ "label": "K8S ClickHouse installation",
+ "multi": false,
+ "name": "db",
+ "options": [],
+ "query": "vertamedia-clickhouse-datasource",
+ "refresh": 1,
+ "regex": "k8s-$exported_namespace-$chi",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "db",
+ "value": "db"
+ },
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, exported_namespace)",
+ "hide": 0,
+ "includeAll": false,
+ "label": "K8S Namespace",
+ "multi": false,
+ "name": "exported_namespace",
+ "options": [],
+ "query": {
+ "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\"}, exported_namespace)",
+ "refId": "Prometheus-exported_namespace-Variable-Query"
+ },
+ "refresh": 2,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "openreplay-clickhouse",
+ "value": "openreplay-clickhouse"
+ },
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "definition": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\", exported_namespace=\"$exported_namespace\"}, chi)",
+ "hide": 0,
+ "includeAll": false,
+ "label": "K8S Clickhouse Installation",
+ "multi": false,
+ "name": "chi",
+ "options": [],
+ "query": {
+ "query": "label_values({__name__ =~ \"chi_clickhouse_metric_Uptime|chi_clickhouse_metric_fetch_errors\",exported_namespace=\"$exported_namespace\"}, chi)",
+ "refId": "Prometheus-chi-Variable-Query"
+ },
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "auto": true,
+ "auto_count": 100,
+ "auto_min": "1m",
+ "current": {
+ "selected": false,
+ "text": "5m",
+ "value": "5m"
+ },
+ "hide": 2,
+ "name": "interval",
+ "options": [
+ {
+ "selected": false,
+ "text": "auto",
+ "value": "$__auto_interval_interval"
+ },
+ {
+ "selected": true,
+ "text": "5m",
+ "value": "5m"
+ }
+ ],
+ "query": "5m",
+ "refresh": 2,
+ "skipUrlSync": false,
+ "type": "interval"
+ },
+ {
+ "allValue": "1,2,3,4",
+ "current": {
+ "selected": true,
+ "tags": [],
+ "text": "All",
+ "value": [
+ "$__all"
+ ]
+ },
+ "hide": 0,
+ "includeAll": true,
+ "label": "type",
+ "multi": true,
+ "name": "type",
+ "options": [
+ {
+ "selected": true,
+ "text": "All",
+ "value": "$__all"
+ },
+ {
+ "selected": false,
+ "text": "1",
+ "value": "1"
+ },
+ {
+ "selected": false,
+ "text": "2",
+ "value": "2"
+ },
+ {
+ "selected": false,
+ "text": "3",
+ "value": "3"
+ },
+ {
+ "selected": false,
+ "text": "4",
+ "value": "4"
+ }
+ ],
+ "query": "1,2,3,4",
+ "skipUrlSync": false,
+ "type": "custom"
+ },
+ {
+ "current": {
+ "selected": true,
+ "tags": [],
+ "text": "5",
+ "value": "5"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "top elements",
+ "multi": false,
+ "name": "top",
+ "options": [
+ {
+ "selected": true,
+ "text": "5",
+ "value": "5"
+ },
+ {
+ "selected": false,
+ "text": "10",
+ "value": "10"
+ },
+ {
+ "selected": false,
+ "text": "15",
+ "value": "15"
+ },
+ {
+ "selected": false,
+ "text": "20",
+ "value": "20"
+ },
+ {
+ "selected": false,
+ "text": "25",
+ "value": "25"
+ },
+ {
+ "selected": false,
+ "text": "30",
+ "value": "30"
+ }
+ ],
+ "query": "5,10,15,20,25,30",
+ "skipUrlSync": false,
+ "type": "custom"
+ },
+ {
+ "allValue": "",
+ "current": {
+ "selected": false,
+ "text": "All",
+ "value": "$__all"
+ },
+ "datasource": {
+ "uid": "$db"
+ },
+ "definition": "SELECT DISTINCT initial_user FROM cluster('all-sharded',system.query_log) WHERE event_date >= today()-3 AND event_time > toDateTime(today()-3) SETTINGS skip_unavailable_shards=1",
+ "hide": 0,
+ "includeAll": true,
+ "label": "initial user",
+ "multi": true,
+ "name": "user",
+ "options": [],
+ "query": "SELECT DISTINCT initial_user FROM cluster('all-sharded',system.query_log) WHERE event_date >= today()-3 AND event_time > toDateTime(today()-3) SETTINGS skip_unavailable_shards=1",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": "",
+ "current": {
+ "selected": true,
+ "text": "all",
+ "value": "all"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "query type",
+ "multi": false,
+ "name": "query_type",
+ "options": [
+ {
+ "selected": true,
+ "text": "all",
+ "value": "all"
+ },
+ {
+ "selected": false,
+ "text": "select",
+ "value": "select"
+ },
+ {
+ "selected": false,
+ "text": "insert",
+ "value": "insert"
+ }
+ ],
+ "query": "all,select,insert",
+ "skipUrlSync": false,
+ "type": "custom"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-1h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "browser",
+ "title": "ClickHouse in Kubernetes Queries",
+ "uid": "clickhouse-queries-dashboard",
+ "version": 1,
+ "weekStart": ""
+}