sender: landing UDP votes

and (tmp) init poh_slot from repair

Author: akhinvasara-jumptrading

book/api/metrics-generated.md

@@ -584,3 +584,11 @@
 | repair_​sent_​pkt_​types_​needed_​window | `counter` | What types of client messages are we sending (Need Window) |
 | repair_​sent_​pkt_​types_​needed_​highest_​window | `counter` | What types of client messages are we sending (Need Highest Window) |
 | repair_​sent_​pkt_​types_​needed_​orphan | `counter` | What types of client messages are we sending (Need Orphans) |
+
+## Sender Tile
+| Metric | Type | Description |
+|--------|------|-------------|
+| sender_​txns_​sent_​to_​leader | `counter` | Total count of transactions sent to leader |
+| sender_​leader_​sched_​not_​found | `counter` | Total count of times leader schedule not found |
+| sender_​leader_​not_​found | `counter` | Total count of times leader not found for given slot |
+| sender_​leader_​contact_​not_​found | `counter` | Total count of times leader contact info not found |

src/app/firedancer-dev/config/default.toml

@@ -29,7 +29,7 @@
         prometheus_listen_address = "0.0.0.0"
         prometheus_listen_port = 7999
 [consensus]
-    vote = false
+    vote = true
     expected_shred_version = 64475
 [paths]
     identity_key = "${KEYS}/fd-identity-keypair.json"

src/app/firedancer-dev/config/private.toml

@@ -24,7 +24,7 @@
         prometheus_listen_address = "0.0.0.0"
         prometheus_listen_port = 7999
 [consensus]
-    vote = false
+    vote = true
     expected_shred_version = 27972
 [paths]
     identity_key = "${KEYS}/fd-identity-keypair.json"

src/app/firedancer/topology.c

@@ -17,8 +17,7 @@
    them out. */
 
 #define IN_KIND_GOSSIP (0UL)
-#define IN_KIND_VOTER  (1UL)
-#define IN_KIND_VERIFY (2UL)
+#define IN_KIND_VERIFY (1UL)
 
 /* fd_dedup_in_ctx_t is a context object for each in (producer) mcache
    connected to the dedup tile. */
@@ -113,8 +112,8 @@ during_frag( fd_dedup_ctx_t * ctx,
   uchar * src = (uchar *)fd_chunk_to_laddr( ctx->in[ in_idx ].mem, chunk );
   uchar * dst = (uchar *)fd_chunk_to_laddr( ctx->out_mem, ctx->out_chunk );
 
-  if( FD_UNLIKELY( ctx->in_kind[ in_idx ]==IN_KIND_GOSSIP || ctx->in_kind[ in_idx ]==IN_KIND_VOTER ) ) {
-    if( FD_UNLIKELY( sz>FD_TPU_MTU ) ) FD_LOG_ERR(( "received a gossip or voter transaction that was too large" ));
+  if( FD_UNLIKELY( ctx->in_kind[ in_idx ]==IN_KIND_GOSSIP ) ) {
+    if( FD_UNLIKELY( sz>FD_TPU_MTU ) ) FD_LOG_ERR(( "received a gossip transaction that was too large" ));
 
     fd_txn_m_t * txnm = (fd_txn_m_t *)dst;
     txnm->payload_sz = (ushort)sz;
@@ -161,7 +160,7 @@ after_frag( fd_dedup_ctx_t *    ctx,
     return;
   }
 
-  if( FD_UNLIKELY( ctx->in_kind[ in_idx ]==IN_KIND_GOSSIP || ctx->in_kind[ in_idx]==IN_KIND_VOTER ) ) {
+  if( FD_UNLIKELY( ctx->in_kind[ in_idx ]==IN_KIND_GOSSIP ) ) {
     /* Transactions coming in from these links are not parsed.
 
        We'll need to parse it so it's ready for downstream consumers.
@@ -170,7 +169,7 @@ after_frag( fd_dedup_ctx_t *    ctx,
     txnm->txn_t_sz = (ushort)fd_txn_parse( fd_txn_m_payload( txnm ), txnm->payload_sz, txn, NULL );
     if( FD_UNLIKELY( !txnm->txn_t_sz ) ) FD_LOG_ERR(( "fd_txn_parse failed for vote transactions that should have been sigverified" ));
 
-    if( FD_UNLIKELY( ctx->in_kind[ in_idx ]==IN_KIND_GOSSIP ) ) FD_MCNT_INC( DEDUP, GOSSIPED_VOTES_RECEIVED, 1UL );
+    FD_MCNT_INC( DEDUP, GOSSIPED_VOTES_RECEIVED, 1UL );
   }
 
   int is_dup = 0;
@@ -251,8 +250,6 @@ unprivileged_init( fd_topo_t *      topo,
 
     if( !strcmp( link->name, "gossip_dedup" ) ) {
       ctx->in_kind[ i ] = IN_KIND_GOSSIP;
-    } else if( !strcmp( link->name, "voter_dedup" ) ) {
-      ctx->in_kind[ i ] = IN_KIND_VOTER;
     } else if( !strcmp( link->name, "verify_dedup" ) ) {
       ctx->in_kind[ i ] = IN_KIND_VERIFY;
     } else {

src/disco/dedup/fd_dedup_tile.c

@@ -25,7 +25,8 @@ class Tile(Enum):
     GOSSIP = 19
     NETLNK = 20
     SOCK = 21,
-    REPAIR = 22
+    REPAIR = 22,
+    SENDER = 23
 
 class MetricType(Enum):
     COUNTER = 0

src/disco/metrics/generate/types.py

@@ -53,6 +53,7 @@ const char * FD_METRICS_TILE_KIND_NAMES[FD_METRICS_TILE_KIND_CNT] = {
     "netlnk",
     "sock",
     "repair",
+    "sender",
 };
 
 const ulong FD_METRICS_TILE_KIND_SIZES[FD_METRICS_TILE_KIND_CNT] = {
@@ -73,6 +74,7 @@ const ulong FD_METRICS_TILE_KIND_SIZES[FD_METRICS_TILE_KIND_CNT] = {
     FD_METRICS_NETLNK_TOTAL,
     FD_METRICS_SOCK_TOTAL,
     FD_METRICS_REPAIR_TOTAL,
+    FD_METRICS_SENDER_TOTAL,
 };
 const fd_metrics_meta_t * FD_METRICS_TILE_KIND_METRICS[FD_METRICS_TILE_KIND_CNT] = {
     FD_METRICS_NET,
@@ -92,4 +94,5 @@ const fd_metrics_meta_t * FD_METRICS_TILE_KIND_METRICS[FD_METRICS_TILE_KIND_CNT]
     FD_METRICS_NETLNK,
     FD_METRICS_SOCK,
     FD_METRICS_REPAIR,
+    FD_METRICS_SENDER,
 };

src/disco/metrics/generated/fd_metrics_all.c

@@ -5,6 +5,7 @@
 #include "fd_metrics_net.h"
 #include "fd_metrics_sock.h"
 #include "fd_metrics_quic.h"
+#include "fd_metrics_sender.h"
 #include "fd_metrics_bundle.h"
 #include "fd_metrics_verify.h"
 #include "fd_metrics_dedup.h"
@@ -155,7 +156,7 @@ extern const fd_metrics_meta_t FD_METRICS_ALL_LINK_OUT[FD_METRICS_ALL_LINK_OUT_T
 
 #define FD_METRICS_TOTAL_SZ (8UL*246UL)
 
-#define FD_METRICS_TILE_KIND_CNT 17
+#define FD_METRICS_TILE_KIND_CNT 18
 extern const char * FD_METRICS_TILE_KIND_NAMES[FD_METRICS_TILE_KIND_CNT];
 extern const ulong FD_METRICS_TILE_KIND_SIZES[FD_METRICS_TILE_KIND_CNT];
 extern const fd_metrics_meta_t * FD_METRICS_TILE_KIND_METRICS[FD_METRICS_TILE_KIND_CNT];

src/disco/metrics/generated/fd_metrics_all.h

@@ -0,0 +1,9 @@
+/* THIS FILE IS GENERATED BY gen_metrics.py. DO NOT HAND EDIT. */
+#include "fd_metrics_sender.h"
+
+const fd_metrics_meta_t FD_METRICS_SENDER[FD_METRICS_SENDER_TOTAL] = {
+    DECLARE_METRIC( SENDER_TXNS_SENT_TO_LEADER, COUNTER ),
+    DECLARE_METRIC( SENDER_LEADER_SCHED_NOT_FOUND, COUNTER ),
+    DECLARE_METRIC( SENDER_LEADER_NOT_FOUND, COUNTER ),
+    DECLARE_METRIC( SENDER_LEADER_CONTACT_NOT_FOUND, COUNTER ),
+};

src/disco/metrics/generated/fd_metrics_sender.c

@@ -0,0 +1,31 @@
+/* THIS FILE IS GENERATED BY gen_metrics.py. DO NOT HAND EDIT. */
+
+#include "../fd_metrics_base.h"
+#include "fd_metrics_enums.h"
+
+#define FD_METRICS_COUNTER_SENDER_TXNS_SENT_TO_LEADER_OFF  (16UL)
+#define FD_METRICS_COUNTER_SENDER_TXNS_SENT_TO_LEADER_NAME "sender_txns_sent_to_leader"
+#define FD_METRICS_COUNTER_SENDER_TXNS_SENT_TO_LEADER_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_SENDER_TXNS_SENT_TO_LEADER_DESC "Total count of transactions sent to leader"
+#define FD_METRICS_COUNTER_SENDER_TXNS_SENT_TO_LEADER_CVT  (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_COUNTER_SENDER_LEADER_SCHED_NOT_FOUND_OFF  (17UL)
+#define FD_METRICS_COUNTER_SENDER_LEADER_SCHED_NOT_FOUND_NAME "sender_leader_sched_not_found"
+#define FD_METRICS_COUNTER_SENDER_LEADER_SCHED_NOT_FOUND_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_SENDER_LEADER_SCHED_NOT_FOUND_DESC "Total count of times leader schedule not found"
+#define FD_METRICS_COUNTER_SENDER_LEADER_SCHED_NOT_FOUND_CVT  (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_COUNTER_SENDER_LEADER_NOT_FOUND_OFF  (18UL)
+#define FD_METRICS_COUNTER_SENDER_LEADER_NOT_FOUND_NAME "sender_leader_not_found"
+#define FD_METRICS_COUNTER_SENDER_LEADER_NOT_FOUND_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_SENDER_LEADER_NOT_FOUND_DESC "Total count of times leader not found for given slot"
+#define FD_METRICS_COUNTER_SENDER_LEADER_NOT_FOUND_CVT  (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_COUNTER_SENDER_LEADER_CONTACT_NOT_FOUND_OFF  (19UL)
+#define FD_METRICS_COUNTER_SENDER_LEADER_CONTACT_NOT_FOUND_NAME "sender_leader_contact_not_found"
+#define FD_METRICS_COUNTER_SENDER_LEADER_CONTACT_NOT_FOUND_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_SENDER_LEADER_CONTACT_NOT_FOUND_DESC "Total count of times leader contact info not found"
+#define FD_METRICS_COUNTER_SENDER_LEADER_CONTACT_NOT_FOUND_CVT  (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_SENDER_TOTAL (4UL)
+extern const fd_metrics_meta_t FD_METRICS_SENDER[FD_METRICS_SENDER_TOTAL];

src/disco/metrics/generated/fd_metrics_sender.h

@@ -210,6 +210,13 @@ metric introduced.
     <counter name="PktRetransmissions" summary="Number of QUIC packets that retransmitted." />
 </tile>
 
+<tile name="sender">
+    <counter name="TxnsSentToLeader" summary="Total count of transactions sent to leader" />
+    <counter name="LeaderSchedNotFound" summary="Total count of times leader schedule not found" />
+    <counter name="LeaderNotFound" summary="Total count of times leader not found for given slot" />
+    <counter name="LeaderContactNotFound" summary="Total count of times leader contact info not found" />
+</tile>
+
 <tile name="bundle">
     <counter name="TransactionReceived" summary="Total count of transactions received, including transactions within bundles" />
     <counter name="PacketReceived" summary="Total count of packets received" />

src/disco/metrics/metrics.xml

@@ -279,9 +279,9 @@ unprivileged_init_sensitive( fd_topo_t *      topo,
       FD_TEST( !strcmp( out_link->name, "sign_repair" ) );
       FD_TEST( in_link->mtu==2048UL );
       FD_TEST( out_link->mtu==64UL );
-    } else if ( !strcmp(in_link->name, "voter_sign" ) ) {
+    } else if ( !strcmp(in_link->name, "send_sign"  ) ) {
       ctx->in_role[ i ] = FD_KEYGUARD_ROLE_VOTER;
-      FD_TEST( !strcmp( out_link->name, "sign_voter"  ) );
+      FD_TEST( !strcmp( out_link->name, "sign_send"  ) );
       FD_TEST( in_link->mtu==FD_TXN_MTU  );
       FD_TEST( out_link->mtu==64UL );
     } else if( !strcmp(in_link->name, "bundle_sign" ) ) {

src/disco/sign/fd_sign_tile.c

@@ -7,7 +7,7 @@
 #define IN_KIND_QUIC   (0UL)
 #define IN_KIND_BUNDLE (1UL)
 #define IN_KIND_GOSSIP (2UL)
-
+#define IN_KIND_SENDER (3UL)
 /* The verify tile is a wrapper around the mux tile, that also verifies
    incoming transaction signatures match the data being signed.
    Non-matching transactions are filtered out of the frag stream. */
@@ -69,7 +69,8 @@ during_frag( fd_verify_ctx_t * ctx,
              ulong             sz,
              ulong             ctl FD_PARAM_UNUSED ) {
 
-  if( FD_UNLIKELY( ctx->in_kind[ in_idx ]==IN_KIND_QUIC || ctx->in_kind[ in_idx ]==IN_KIND_GOSSIP ) ) {
+  ulong in_kind = ctx->in_kind[ in_idx ];
+  if( FD_UNLIKELY( in_kind==IN_KIND_QUIC || in_kind==IN_KIND_GOSSIP || in_kind==IN_KIND_SENDER ) ) {
     if( FD_UNLIKELY( chunk<ctx->in[in_idx].chunk0 || chunk>ctx->in[in_idx].wmark || sz>FD_TPU_MTU ) )
       FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->in[in_idx].chunk0, ctx->in[in_idx].wmark ));
 
@@ -79,7 +80,7 @@ during_frag( fd_verify_ctx_t * ctx,
     dst->payload_sz = (ushort)sz;
     dst->block_engine.bundle_id = 0UL;
     fd_memcpy( fd_txn_m_payload( dst ), src, sz );
-  } else if( FD_UNLIKELY( ctx->in_kind[ in_idx ]==IN_KIND_BUNDLE ) ) {
+  } else if( FD_UNLIKELY( in_kind==IN_KIND_BUNDLE ) ) {
     if( FD_UNLIKELY( chunk<ctx->in[in_idx].chunk0 || chunk>ctx->in[in_idx].wmark || sz>FD_TPU_RAW_MTU ) )
       FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu,%lu]", chunk, sz, ctx->in[in_idx].chunk0, ctx->in[in_idx].wmark, FD_TPU_RAW_MTU ));
 
@@ -205,6 +206,7 @@ unprivileged_init( fd_topo_t *      topo,
     if(      !strcmp( link->name, "quic_verify"  ) ) ctx->in_kind[ i ] = IN_KIND_QUIC;
     else if( !strcmp( link->name, "bundle_verif" ) ) ctx->in_kind[ i ] = IN_KIND_BUNDLE;
     else if( !strcmp( link->name, "gossip_verif" ) ) ctx->in_kind[ i ] = IN_KIND_GOSSIP;
+    else if( !strcmp( link->name, "send_txns"    ) ) ctx->in_kind[ i ] = IN_KIND_SENDER;
     else FD_LOG_ERR(( "unexpected link name %s", link->name ));
   }
 

src/disco/verify/fd_verify_tile.c

@@ -67,23 +67,20 @@ scratch_footprint( fd_topo_tile_t const * tile FD_PARAM_UNUSED ) {
 }
 
 static inline void
-handle_new_cluster_contact_info( fd_eqvoc_tile_ctx_t * ctx, uchar const * buf, ulong buf_sz ) {
+handle_new_cluster_contact_info( fd_eqvoc_tile_ctx_t * ctx,
+                                 uchar const *         buf,
+                                 ulong                 buf_sz ) {
   ulong const * header = (ulong const *)fd_type_pun_const( buf );
 
-  ulong dest_cnt = buf_sz;
-
-  if( dest_cnt >= MAX_SHRED_DESTS )
-    FD_LOG_ERR(( "Cluster nodes had %lu destinations, which was more than the max of %lu",
-                  dest_cnt,
-                  MAX_SHRED_DESTS ));
+  ulong dest_cnt = buf_sz / sizeof(fd_shred_dest_wire_t);
 
   fd_shred_dest_wire_t const * in_dests = fd_type_pun_const( header );
-  fd_shred_dest_weighted_t *   dests    = fd_stake_ci_dest_add_init( ctx->stake_ci );
+  fd_shred_dest_weighted_t * dests = fd_stake_ci_dest_add_init( ctx->stake_ci );
 
   ctx->new_dest_ptr = dests;
   ctx->new_dest_cnt = dest_cnt;
 
-  for( ulong i = 0UL; i < dest_cnt; i++ ) {
+  for( ulong i=0UL; i<dest_cnt; i++ ) {
     memcpy( dests[i].pubkey.uc, in_dests[i].pubkey, 32UL );
     dests[i].ip4  = in_dests[i].ip4_addr;
     dests[i].port = in_dests[i].udp_port;
@@ -229,7 +226,7 @@ unprivileged_init( fd_topo_t *      topo,
   ctx->stake_ci = fd_stake_ci_join( fd_stake_ci_new( stake_ci_mem, ctx->identity_key ) );
   ctx->eqvoc    = fd_eqvoc_join( fd_eqvoc_new( eqvoc_mem, 1 << 10, 1 << 10, 0 ) );
 
-  ctx->contact_in_idx = fd_topo_find_tile_in_link( topo, tile, "gossip_voter", 0 );
+  ctx->contact_in_idx = fd_topo_find_tile_in_link( topo, tile, "gossip_send", 0 );
   FD_TEST( ctx->contact_in_idx != ULONG_MAX );
   fd_topo_link_t * contact_in_link = &topo->links[tile->in_link_id[ctx->contact_in_idx]];
   ctx->contact_in_mem = topo->workspaces[topo->objs[contact_in_link->dcache_obj_id].wksp_id].wksp;