diff --git a/monitor-snapshot/release-6.5/ansible-monitor.tar.gz b/monitor-snapshot/release-6.5/ansible-monitor.tar.gz index ba1ad70c..d071f450 100644 Binary files a/monitor-snapshot/release-6.5/ansible-monitor.tar.gz and b/monitor-snapshot/release-6.5/ansible-monitor.tar.gz differ diff --git a/monitor-snapshot/release-6.5/operator/dashboards/lightning.json b/monitor-snapshot/release-6.5/operator/dashboards/lightning.json index 794ba587..2ff5bc7f 100644 --- a/monitor-snapshot/release-6.5/operator/dashboards/lightning.json +++ b/monitor-snapshot/release-6.5/operator/dashboards/lightning.json @@ -93,15 +93,36 @@ "expr": "rate(tikv_import_write_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])", "format": "time_series", "intervalFactor": 2, - "legendFormat": "write from lightning", - "refId": "B" + "legendFormat": "write from lightning(importer)", + "refId": "A" }, { "expr": "sum(rate(tikv_import_upload_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "intervalFactor": 2, - "legendFormat": "upload to tikv", + "legendFormat": "upload to tikv(importer)", + "refId": "B" + }, + { + "expr": "rate(lightning_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", state=\"imported\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "write to tikv(lightning)", "refId": "C" + }, + { + "expr": "rate(lightning_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", state=\"restored\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "read from source(lightning)", + "refId": "D" + }, + { + "expr": "rate(lightning_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", state=\"written\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "write to local(lightning)", + "refId": "E" } ], "thresholds": [], @@ -507,7 +528,7 @@ "refId": "A" }, { - "expr": "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"lightning\"}", + "expr": "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb-lightning\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "lightning heap", @@ -582,7 +603,7 @@ "steppedLine": false, "targets": [ { - "expr": "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"lightning\"}", + "expr": "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb-lightning\"}", "format": "time_series", "instant": false, "intervalFactor": 2, @@ -661,7 +682,7 @@ "steppedLine": false, "targets": [ { - "expr": "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"lightning\"}[30s])*100", + "expr": "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb-lightning\"}[30s])*100", "format": "time_series", "intervalFactor": 2, "legendFormat": "Lightning", @@ -1366,15 +1387,29 @@ "expr": "rate(tikv_import_range_delivery_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_range_delivery_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 2, - "legendFormat": "range deliver", + "legendFormat": "range deliver(importer)", "refId": "A" }, { "expr": "rate(tikv_import_sst_delivery_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_sst_delivery_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", "format": "time_series", "intervalFactor": 2, - "legendFormat": "SST file deliver", + "legendFormat": "SST file deliver(importer)", "refId": "B" + }, + { + "expr": "rate(lightning_row_kv_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(lightning_row_kv_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "row deliver(lightning)", + "refId": "C" + }, + { + "expr": "rate(lightning_block_deliver_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(lightning_block_deliver_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block deliver(lightning)", + "refId": "D" } ], "thresholds": [], @@ -1450,35 +1485,58 @@ "stack": false, "steppedLine": false, "targets": [ + { + "expr": "rate(tikv_import_sst_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", + "legendFormat": "SST size(importer)", + "format": "time_series", + "intervalFactor": 2, + "refId": "A" + }, { "expr": "rate(tikv_import_split_sst_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_split_sst_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", + "legendFormat": "Split SST(importer)", "format": "time_series", "intervalFactor": 2, - "legendFormat": "Split SST", - "refId": "C" + "refId": "B" }, { "expr": "rate(tikv_import_sst_upload_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_sst_upload_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", + "legendFormat": "SST upload(importer)", "format": "time_series", - "interval": "", "intervalFactor": 2, - "legendFormat": "SST upload", - "refId": "D" + "refId": "C" }, { "expr": "rate(tikv_import_sst_ingest_duration_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s]) / rate(tikv_import_sst_ingest_duration_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", + "legendFormat": "SST ingest(importer)", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "refId": "D" + }, + { + "expr": "rate(lightning_sst_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", kind=\"split\"}[30s]) / rate(lightning_sst_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", kind=\"split\"}[30s])\n", + "legendFormat": "Region split and scatter(lightning)", "format": "time_series", "instant": false, "intervalFactor": 2, - "legendFormat": "SST ingest", "refId": "E" }, { - "expr": "rate(tikv_import_sst_chunk_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])", + "expr": "rate(lightning_sst_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", kind=\"write\"}[30s]) / rate(lightning_sst_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", kind=\"write\"}[30s])\n", + "legendFormat": "SST write(lightning)", "format": "time_series", + "instant": false, "intervalFactor": 2, - "legendFormat": "SST size", - "refId": "A" + "refId": "F" + }, + { + "expr": "rate(lightning_sst_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", kind=\"ingest\"}[30s]) / rate(lightning_sst_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", kind=\"ingest\"}[30s])", + "legendFormat": "SST ingest(lightning)", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "refId": "G" } ], "thresholds": [],