vmon: Missed block metric (#5913)

Validator monitoring gained 2 new metrics for tracking when blocks are
included or not on the head chain.

Similar to attestations, if the block is produced in epoch N, reporting
will use the state when switching to epoch N+2 to do the reporting (so
as to reasonably stabilise the block inclusion in the face of reorgs).

beacon_chain/consensus_object_pools/blockchain_dag.nim

@@ -956,9 +956,14 @@ proc advanceSlots*(
       # which is an acceptable tradeoff for monitoring.
       withState(state):
         let postEpoch = forkyState.data.slot.epoch
-        if preEpoch != postEpoch:
+        if preEpoch != postEpoch and postEpoch >= 2:
+          var proposers: array[SLOTS_PER_EPOCH, Opt[ValidatorIndex]]
+          let epochRef = dag.findEpochRef(stateBid, postEpoch - 2)
+          if epochRef.isSome():
+            proposers = epochRef[][].beacon_proposers
+
           dag.validatorMonitor[].registerEpochInfo(
-            postEpoch, info, forkyState.data)
+            forkyState.data, proposers, info)
 
 proc applyBlock(
     dag: ChainDAGRef, state: var ForkedHashedBeaconState, bid: BlockId,

beacon_chain/validators/validator_monitor.nim

@@ -149,6 +149,11 @@ declareCounter validator_monitor_proposer_slashing,
 declareCounter validator_monitor_attester_slashing,
   "Number of attester slashings seen", labels = ["src", "validator"]
 
+declareCounter validator_monitor_block_hit,
+  "Number of times a block proposed by the validator was included an epoch later", labels = ["validator"]
+declareCounter validator_monitor_block_miss,
+  "Number of times the validator was expected to propose a block but no block was included", labels = ["validator"]
+
 const
   total = "total" # what we use for label when using "totals" mode
 
@@ -405,12 +410,15 @@ func is_active_unslashed_in_previous_epoch(status: ParticipationInfo): bool =
   ParticipationFlag.eligible in status.flags
 
 proc registerEpochInfo*(
-    self: var ValidatorMonitor, epoch: Epoch, info: ForkedEpochInfo,
-    state: ForkyBeaconState) =
+    self: var ValidatorMonitor, state: ForkyBeaconState,
+    proposers: array[SLOTS_PER_EPOCH, Opt[ValidatorIndex]],
+    info: ForkedEpochInfo) =
   # Register rewards, as computed during the epoch transition that lands in
   # `epoch` - the rewards will be from attestations that were created at
   # `epoch - 2`.
 
+  let epoch = state.slot.epoch
+
   if epoch < 2 or self.monitors.len == 0:
     return
 
@@ -442,6 +450,24 @@ proc registerEpochInfo*(
         # attestations.
         continue
 
+      # Check that block proposals are sticky an epoch later
+      for i in 0..<SLOTS_PER_EPOCH:
+        let slot = prev_epoch.start_slot + i
+        if slot == 0:
+          continue
+
+        if proposers[i] == Opt.some(idx):
+          let hasBlock =
+            # When a block is missing in a slot, the beacon root repeats
+            get_block_root_at_slot(state, slot - 1) !=
+              get_block_root_at_slot(state, slot)
+          if hasBlock:
+            validator_monitor_block_hit.inc(1, [metricId])
+            info "Block proposal included", slot, validator = id
+          else:
+            validator_monitor_block_miss.inc(1, [metricId])
+            notice "Block proposal missing", slot, validator = id
+
       let
         previous_epoch_matched_source = status.is_previous_epoch_source_attester()
         previous_epoch_matched_target = status.is_previous_epoch_target_attester()

grafana/beacon_nodes_Grafana_dashboard.json

@@ -5997,7 +5997,7 @@
         "x": 0,
         "y": 111
       },
-      "id": 89,
+      "id": 125,
       "options": {
         "legend": {
           "calcs": [],
@@ -6017,37 +6017,17 @@
             "type": "prometheus",
             "uid": "${DS_PROMETHEUS-PROXY}"
           },
+          "editorMode": "code",
           "exemplar": true,
-          "expr": "sum(rate(validator_monitor_prev_epoch_on_chain_head_attester_miss_total{instance=\"${instance}\",container=\"${container}\"}[$__rate_interval]))*384",
-          "interval": "384s",
-          "legendFormat": "head",
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS-PROXY}"
-          },
-          "exemplar": true,
-          "expr": "sum(rate(validator_monitor_prev_epoch_on_chain_target_attester_miss_total{instance=\"${instance}\",container=\"${container}\"}[$__rate_interval]))*384",
-          "interval": "384s",
-          "legendFormat": "target",
-          "refId": "B"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${DS_PROMETHEUS-PROXY}"
-          },
-          "exemplar": true,
-          "expr": "sum(rate(validator_monitor_prev_epoch_on_chain_source_attester_miss_total{instance=\"${instance}\",container=\"${container}\"}[$__rate_interval]))*384",
+          "expr": "sum(rate(validator_monitor_block_miss_total{instance=\"${instance}\",container=\"${container}\"}[$__rate_interval]))*384",
           "hide": false,
           "interval": "384s",
-          "legendFormat": "source",
+          "legendFormat": "miss ({{validator}})",
+          "range": true,
           "refId": "C"
         }
       ],
-      "title": "Attestation misses",
+      "title": "Block misses",
       "type": "timeseries"
     },
     {
@@ -6145,6 +6125,141 @@
       "title": "Balance",
       "type": "timeseries"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS-PROXY}"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "bars",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineStyle": {
+              "fill": "solid"
+            },
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "source"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "red",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 119
+      },
+      "id": 89,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "8.0.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS-PROXY}"
+          },
+          "exemplar": true,
+          "expr": "sum(rate(validator_monitor_prev_epoch_on_chain_head_attester_miss_total{instance=\"${instance}\",container=\"${container}\"}[$__rate_interval]))*384",
+          "interval": "384s",
+          "legendFormat": "head",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS-PROXY}"
+          },
+          "exemplar": true,
+          "expr": "sum(rate(validator_monitor_prev_epoch_on_chain_target_attester_miss_total{instance=\"${instance}\",container=\"${container}\"}[$__rate_interval]))*384",
+          "interval": "384s",
+          "legendFormat": "target",
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS-PROXY}"
+          },
+          "exemplar": true,
+          "expr": "sum(rate(validator_monitor_prev_epoch_on_chain_source_attester_miss_total{instance=\"${instance}\",container=\"${container}\"}[$__rate_interval]))*384",
+          "hide": false,
+          "interval": "384s",
+          "legendFormat": "source",
+          "refId": "C"
+        }
+      ],
+      "title": "Attestation misses",
+      "type": "timeseries"
+    },
     {
       "collapsed": true,
       "datasource": {
@@ -6155,7 +6270,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 119
+        "y": 127
       },
       "id": 75,
       "panels": [
@@ -6495,6 +6610,6 @@
   "timezone": "",
   "title": "NBC local testnet/sim (all nodes)",
   "uid": "pgeNfj2Wz23",
-  "version": 15,
+  "version": 16,
   "weekStart": ""
 }