{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "Prometheus / Mimir datasource connected to your OTel collector", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.0.0" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" }, { "type": "panel", "id": "table", "name": "Table", "version": "" } ], "uid": "trustvc-fleet-health", "title": "TrustVC — Fleet & Chain Health", "description": "Fleet health, per-instance status, chain connection, state-change events, and escrow subscriptions", "schemaVersion": 39, "version": 1, "editable": true, "graphTooltip": 1, "time": { "from": "now-1h", "to": "now" }, "refresh": "15s", "tags": ["trustvc", "fleet"], "templating": { "list": [ { "name": "instance", "label": "Instance", "type": "query", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "query": "label_values(trustvc_instance_health, instance)", "includeAll": true, "allValue": ".*", "multi": true, "refresh": 2, "sort": 1, "current": {} } ] }, "panels": [ { "id": 100, "type": "row", "title": "Fleet Overview", "collapsed": false, "gridPos": { "x": 0, "y": 0, "w": 24, "h": 1 } }, { "id": 1, "type": "stat", "title": "Active Instances", "description": "Running process replicas currently emitting metrics", "gridPos": { "x": 0, "y": 1, "w": 4, "h": 4 }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "textMode": "auto" }, "fieldConfig": { "defaults": { "noValue": "0", "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } }, "overrides": [] }, "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(trustvc_instance_health{instance=~\"$instance\"})", "legendFormat": "", "refId": "A", "instant": true }] }, { "id": 2, "type": "stat", "title": "Healthy Instances", "description": "Instances where no chain has permanently failed", "gridPos": { "x": 4, "y": 1, "w": 4, "h": 4 }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none", "justifyMode": "center" }, "fieldConfig": { "defaults": { "noValue": "0", "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } }, "overrides": [] }, "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(trustvc_instance_health{instance=~\"$instance\"} == 1) or vector(0)", "legendFormat": "", "refId": "A", "instant": true }] }, { "id": 3, "type": "stat", "title": "Degraded Instances", "description": "Instances with at least one permanently failed chain", "gridPos": { "x": 8, "y": 1, "w": 4, "h": 4 }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none", "justifyMode": "center" }, "fieldConfig": { "defaults": { "noValue": "0", "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] } }, "overrides": [] }, "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(trustvc_instance_health{instance=~\"$instance\"} == 0) or vector(0)", "legendFormat": "", "refId": "A", "instant": true }] }, { "id": 4, "type": "stat", "title": "Total Active Chains", "description": "Sum of chains running across all instances", "gridPos": { "x": 12, "y": 1, "w": 4, "h": 4 }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none", "justifyMode": "center" }, "fieldConfig": { "defaults": { "noValue": "0", "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } }, "overrides": [] }, "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(trustvc_instance_active_chains{instance=~\"$instance\"}) or vector(0)", "legendFormat": "", "refId": "A", "instant": true }] }, { "id": 5, "type": "stat", "title": "Total Active Escrows", "description": "TitleEscrow subscriptions across all chains and instances", "gridPos": { "x": 16, "y": 1, "w": 4, "h": 4 }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, "fieldConfig": { "defaults": { "noValue": "0", "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] } }, "overrides": [] }, "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(trustvc_instance_total_escrows{instance=~\"$instance\"}) or vector(0)", "legendFormat": "", "refId": "A", "instant": true }] }, { "id": 6, "type": "stat", "title": "Active Worker Processes", "description": "Forked child worker processes across all instances", "gridPos": { "x": 20, "y": 1, "w": 4, "h": 4 }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none", "justifyMode": "center" }, "fieldConfig": { "defaults": { "noValue": "0", "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] } }, "overrides": [] }, "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(trustvc_instance_active_workers{instance=~\"$instance\"}) or vector(0)", "legendFormat": "", "refId": "A", "instant": true }] }, { "id": 101, "type": "row", "title": "Per-Instance Status", "collapsed": false, "gridPos": { "x": 0, "y": 5, "w": 24, "h": 1 } }, { "id": 7, "type": "table", "title": "Instance Status", "description": "One row per active process replica — health, uptime, chains, workers, escrows", "gridPos": { "x": 0, "y": 6, "w": 16, "h": 7 }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false }, "sortBy": [{ "desc": false, "displayName": "Instance" }] }, "fieldConfig": { "defaults": { "custom": { "align": "left", "displayMode": "auto" } }, "overrides": [ { "matcher": { "id": "byName", "options": "Health" }, "properties": [ { "id": "mappings", "value": [{ "type": "value", "options": { "1": { "text": "ok", "color": "green" }, "0": { "text": "degraded", "color": "red" } } }] }, { "id": "custom.displayMode", "value": "color-background" } ] }, { "matcher": { "id": "byName", "options": "Uptime" }, "properties": [{ "id": "unit", "value": "s" }, { "id": "custom.width", "value": 110 }] } ] }, "transformations": [ { "id": "labelsToFields", "options": { "valueLabel": "instance" } }, { "id": "merge", "options": {} }, { "id": "organize", "options": { "renameByName": { "trustvc_instance_health": "Health", "Value #A": "Health", "trustvc_instance_uptime_seconds": "Uptime", "Value #B": "Uptime", "trustvc_instance_active_chains": "Chains", "Value #C": "Chains", "trustvc_instance_active_workers": "Workers", "Value #D": "Workers", "trustvc_instance_total_escrows": "Escrows", "Value #E": "Escrows", "instance": "Instance" }, "excludeByName": { "Time": true, "__name__": true } } } ], "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "trustvc_instance_health{instance=~\"$instance\"}", "refId": "A", "instant": true, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "trustvc_instance_uptime_seconds{instance=~\"$instance\"}", "refId": "B", "instant": true, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "trustvc_instance_active_chains{instance=~\"$instance\"}", "refId": "C", "instant": true, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "trustvc_instance_active_workers{instance=~\"$instance\"}", "refId": "D", "instant": true, "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "trustvc_instance_total_escrows{instance=~\"$instance\"}", "refId": "E", "instant": true, "format": "table" } ] }, { "id": 8, "type": "timeseries", "title": "Instance Uptime", "description": "Gaps indicate process restarts", "gridPos": { "x": 16, "y": 6, "w": 8, "h": 7 }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "options": { "tooltip": { "mode": "multi", "sort": "none" }, "legend": { "displayMode": "list", "placement": "bottom" } }, "fieldConfig": { "defaults": { "unit": "s", "custom": { "lineWidth": 2, "fillOpacity": 10 } }, "overrides": [] }, "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "trustvc_instance_uptime_seconds{instance=~\"$instance\"}", "legendFormat": "{{instance}}", "refId": "A" }] }, { "id": 102, "type": "row", "title": "Chain Health", "collapsed": false, "gridPos": { "x": 0, "y": 13, "w": 24, "h": 1 } }, { "id": 9, "type": "timeseries", "title": "Chain RPC Connection Status", "description": "1 = connected, 0 = not connected. Drops indicate disconnections.", "gridPos": { "x": 0, "y": 14, "w": 12, "h": 8 }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "options": { "tooltip": { "mode": "multi", "sort": "none" }, "legend": { "displayMode": "list", "placement": "bottom" } }, "fieldConfig": { "defaults": { "min": 0, "max": 1, "custom": { "lineWidth": 2, "fillOpacity": 15, "drawStyle": "line", "spanNulls": false }, "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } }, "overrides": [] }, "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "trustvc_chain_connected{instance=~\"$instance\"}", "legendFormat": "{{chain}} ({{instance}})", "refId": "A" }] }, { "id": 10, "type": "timeseries", "title": "Chain Reconnect Attempts", "description": "Rising line = persistent connection trouble", "gridPos": { "x": 12, "y": 14, "w": 12, "h": 8 }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "options": { "tooltip": { "mode": "multi", "sort": "none" }, "legend": { "displayMode": "list", "placement": "bottom" } }, "fieldConfig": { "defaults": { "custom": { "lineWidth": 2, "fillOpacity": 10 }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 3 }, { "color": "red", "value": 10 }] } }, "overrides": [] }, "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "trustvc_chain_reconnect_attempts{instance=~\"$instance\"}", "legendFormat": "{{chain}} ({{instance}})", "refId": "A" }] }, { "id": 103, "type": "row", "title": "State-Change Events", "collapsed": false, "gridPos": { "x": 0, "y": 22, "w": 24, "h": 1 } }, { "id": 11, "type": "timeseries", "title": "Chain State Transitions (rate)", "description": "Spikes = flapping connection. Labels show from_status → to_status.", "gridPos": { "x": 0, "y": 23, "w": 14, "h": 8 }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "options": { "tooltip": { "mode": "multi", "sort": "none" }, "legend": { "displayMode": "list", "placement": "bottom" } }, "fieldConfig": { "defaults": { "unit": "ops", "custom": { "lineWidth": 2, "fillOpacity": 10, "drawStyle": "bars", "barAlignment": 0 } }, "overrides": [] }, "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(trustvc_chain_state_changes_total{instance=~\"$instance\"}[5m])", "legendFormat": "{{chain}}: {{from_status}} → {{to_status}}", "refId": "A" }] }, { "id": 12, "type": "table", "title": "State Transition Counts", "description": "Total transitions since process start", "gridPos": { "x": 14, "y": 23, "w": 10, "h": 8 }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false } }, "fieldConfig": { "defaults": { "custom": { "align": "left", "displayMode": "auto" } }, "overrides": [] }, "transformations": [ { "id": "organize", "options": { "renameByName": { "chain": "Chain", "from_status": "From", "to_status": "To", "Value": "Count" }, "excludeByName": { "Time": true, "__name__": true } } } ], "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "trustvc_chain_state_changes_total{instance=~\"$instance\"}", "refId": "A", "instant": true, "format": "table" }] }, { "id": 104, "type": "row", "title": "Event Processing", "collapsed": false, "gridPos": { "x": 0, "y": 31, "w": 24, "h": 1 } }, { "id": 13, "type": "timeseries", "title": "Active Escrows per Chain", "description": "Grows as new tokens are minted", "gridPos": { "x": 0, "y": 32, "w": 12, "h": 8 }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "options": { "tooltip": { "mode": "multi", "sort": "none" }, "legend": { "displayMode": "list", "placement": "bottom" } }, "fieldConfig": { "defaults": { "min": 0, "custom": { "lineWidth": 2, "fillOpacity": 15 } }, "overrides": [] }, "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "trustvc_chain_active_escrows{instance=~\"$instance\"}", "legendFormat": "{{chain}} ({{instance}})", "refId": "A" }] }, { "id": 14, "type": "timeseries", "title": "Last Seen Block per Chain", "description": "Flat line = listener stalled", "gridPos": { "x": 12, "y": 32, "w": 12, "h": 8 }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "options": { "tooltip": { "mode": "multi", "sort": "none" }, "legend": { "displayMode": "list", "placement": "bottom" } }, "fieldConfig": { "defaults": { "custom": { "lineWidth": 2, "fillOpacity": 10 } }, "overrides": [] }, "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "trustvc_chain_last_seen_block{instance=~\"$instance\"}", "legendFormat": "{{chain}} ({{instance}})", "refId": "A" }] } ], "annotations": { "list": [] }, "links": [] }