{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "Prometheus datasource scraping the Planeon api (:9090) and worker (:9091) ops endpoints", "type": "datasource", "pluginId": "prometheus" } ], "title": "Planeon Overview", "uid": "planeon-overview", "tags": ["planeon", "vdi"], "schemaVersion": 39, "editable": true, "refresh": "30s", "time": { "from": "now-6h", "to": "now" }, "templating": { "list": [ { "name": "job", "label": "job", "type": "query", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "query": "label_values(planeon_jobs_depth, job)", "definition": "label_values(planeon_jobs_depth, job)", "refresh": 2, "sort": 1, "multi": true, "includeAll": true, "allValue": ".*", "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, "options": [] }, { "name": "instance", "label": "instance", "type": "query", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "query": "label_values(planeon_jobs_depth{job=~\"$job\"}, instance)", "definition": "label_values(planeon_jobs_depth{job=~\"$job\"}, instance)", "refresh": 2, "sort": 1, "multi": true, "includeAll": true, "allValue": ".*", "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, "options": [] }, { "name": "cluster", "label": "cluster", "type": "query", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "query": "label_values(planeon_pve_requests_total, cluster)", "definition": "label_values(planeon_pve_requests_total, cluster)", "refresh": 2, "sort": 1, "multi": true, "includeAll": true, "allValue": ".*", "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, "options": [] }, { "name": "tenant", "label": "tenant", "type": "query", "description": "Forward-compatibility placeholder: metrics carry no tenant label yet, so this stays on All until tenants ship (v1.1+).", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "query": "label_values(planeon_sessions, tenant)", "definition": "label_values(planeon_sessions, tenant)", "refresh": 2, "sort": 1, "multi": true, "includeAll": true, "allValue": ".*", "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, "options": [] } ] }, "panels": [ { "id": 100, "type": "row", "title": "Availability", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "panels": [] }, { "id": 1, "type": "state-timeline", "title": "Scrape targets up", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 6, "w": 16, "x": 0, "y": 1 }, "fieldConfig": { "defaults": { "mappings": [ { "type": "value", "options": { "1": { "text": "up", "color": "green", "index": 0 }, "0": { "text": "down", "color": "red", "index": 1 } } } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single" } }, "targets": [ { "expr": "up{job=~\"$job\", instance=~\"$instance\"}", "legendFormat": "{{job}} {{instance}}" } ] }, { "id": 2, "type": "stat", "title": "Targets down", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 1 }, "fieldConfig": { "defaults": { "unit": "short", "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "targets": [ { "expr": "count(up{job=~\"$job\", instance=~\"$instance\"} == 0) OR on() vector(0)", "legendFormat": "targets down" } ] }, { "id": 101, "type": "row", "title": "Jobs", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 }, "panels": [] }, { "id": 3, "type": "timeseries", "title": "Job queue depth by kind", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max"] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum by (kind) (max by (kind, cluster) (planeon_jobs_depth{job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\", status=\"queued\"}))", "legendFormat": "{{kind}} queued" }, { "expr": "sum by (kind) (max by (kind, cluster) (planeon_jobs_depth{job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\", status=\"running\"}))", "legendFormat": "{{kind}} running" } ] }, { "id": 4, "type": "stat", "title": "Oldest queued job age", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 8 }, "fieldConfig": { "defaults": { "unit": "s", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 300 } ] } }, "overrides": [] }, "targets": [ { "expr": "max(planeon_jobs_oldest_queued_age_seconds{job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"})", "legendFormat": "oldest queued" } ] }, { "id": 5, "type": "timeseries", "title": "Job failures and retries (5m rate)", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 8 }, "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max"] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum by (kind) (rate(planeon_jobs_processed_total{job=~\"$job\", instance=~\"$instance\", outcome=\"failed\"}[5m]))", "legendFormat": "{{kind}} failed" }, { "expr": "sum by (kind) (rate(planeon_jobs_processed_total{job=~\"$job\", instance=~\"$instance\", outcome=\"retried\"}[5m]))", "legendFormat": "{{kind}} retried" } ] }, { "id": 6, "type": "timeseries", "title": "Job duration p95 by kind", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max"] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "histogram_quantile(0.95, sum by (kind, le) (rate(planeon_job_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[10m])))", "legendFormat": "{{kind}} p95" } ] }, { "id": 7, "type": "timeseries", "title": "Job lease reclaims (15m increase)", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["max"] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum by (kind) (increase(planeon_job_lease_reclaims_total{job=~\"$job\", instance=~\"$instance\"}[15m]))", "legendFormat": "{{kind}} reclaimed" } ] }, { "id": 102, "type": "row", "title": "Reconcile", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, "panels": [] }, { "id": 8, "type": "timeseries", "title": "Reconcile cycle duration p95", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 }, "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max"] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "histogram_quantile(0.95, sum by (le) (rate(planeon_reconcile_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\"}[10m])))", "legendFormat": "cycle p95" } ] }, { "id": 9, "type": "timeseries", "title": "Reconcile cycles by outcome (5m rate)", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 }, "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max"] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum by (outcome) (rate(planeon_reconcile_cycles_total{job=~\"$job\", instance=~\"$instance\"}[5m]))", "legendFormat": "{{outcome}}" } ] }, { "id": 103, "type": "row", "title": "PVE", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 33 }, "panels": [] }, { "id": 10, "type": "timeseries", "title": "PVE request rate by cluster and outcome", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 34 }, "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max"] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum by (cluster, outcome) (rate(planeon_pve_requests_total{job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"}[5m]))", "legendFormat": "{{cluster}} {{outcome}}" } ] }, { "id": 11, "type": "timeseries", "title": "PVE latency p95 by cluster", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 34 }, "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max"] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "histogram_quantile(0.95, sum by (cluster, le) (rate(planeon_pve_request_duration_seconds_bucket{job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"}[10m])))", "legendFormat": "{{cluster}} p95" } ] }, { "id": 12, "type": "timeseries", "title": "PVE error ratio by cluster (10m)", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 34 }, "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.05 }, { "color": "red", "value": 0.2 } ] }, "custom": { "thresholdsStyle": { "mode": "line" } } }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max"] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "(sum by (cluster) (rate(planeon_pve_requests_total{job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\", outcome!=\"success\"}[10m])) OR sum by (cluster) (rate(planeon_pve_requests_total{job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"}[10m])) * 0) / sum by (cluster) (rate(planeon_pve_requests_total{job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\"}[10m]))", "legendFormat": "{{cluster}} errors" } ] }, { "id": 104, "type": "row", "title": "Sessions", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 42 }, "panels": [] }, { "id": 13, "type": "timeseries", "title": "Sessions by state", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 43 }, "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max"] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum by (state) (max by (state, cluster) (planeon_sessions{job=~\"$job\", instance=~\"$instance\", cluster=~\"$cluster\", tenant=~\"$tenant\"}))", "legendFormat": "{{state}}" } ] }, { "id": 105, "type": "row", "title": "DB pool", "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 51 }, "panels": [] }, { "id": 14, "type": "timeseries", "title": "DB pool connections (per instance)", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 52 }, "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max"] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "planeon_db_pool_acquired_conns{job=~\"$job\", instance=~\"$instance\"}", "legendFormat": "{{instance}} acquired" }, { "expr": "planeon_db_pool_idle_conns{job=~\"$job\", instance=~\"$instance\"}", "legendFormat": "{{instance}} idle" }, { "expr": "planeon_db_pool_max_conns{job=~\"$job\", instance=~\"$instance\"}", "legendFormat": "{{instance}} max" } ] }, { "id": 15, "type": "timeseries", "title": "DB pool acquire pressure (5m rate)", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 52 }, "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": ["mean", "max"] }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "rate(planeon_db_pool_empty_acquire_total{job=~\"$job\", instance=~\"$instance\"}[5m])", "legendFormat": "{{instance}} waited" }, { "expr": "rate(planeon_db_pool_canceled_acquire_total{job=~\"$job\", instance=~\"$instance\"}[5m])", "legendFormat": "{{instance}} canceled" } ] } ] }