Skip to content

Commit c035d95

Browse files
topointon-jumptopointon-jump
committed
flamenco, groove: initial Funk/Groove wiring
1 parent ec1baa3 commit c035d95

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+1800
-355
lines changed

‎src/app/firedancer-dev/Local.mk

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ $(call add-objs,commands/dev,fd_firedancer_dev)
1515
$(call add-objs,commands/sim,fd_firedancer_dev)
1616
$(call add-objs,commands/backtest,fd_firedancer_dev)
1717

18-
$(call make-bin,firedancer-dev,main,fd_firedancer_dev fd_firedancer fddev_shared fdctl_shared fd_discof fd_disco fd_choreo fd_flamenco fd_funk fd_quic fd_tls fd_reedsol fd_ballet fd_waltz fd_tango fd_util firedancer_version, $(SECP256K1_LIBS) $(ROCKSDB_LIBS))
18+
$(call make-bin,firedancer-dev,main,fd_firedancer_dev fd_firedancer fddev_shared fdctl_shared fd_discof fd_disco fd_choreo fd_flamenco fd_funk fd_groove fd_quic fd_tls fd_reedsol fd_ballet fd_waltz fd_tango fd_util firedancer_version, $(SECP256K1_LIBS) $(ROCKSDB_LIBS))
1919

2020
firedancer-dev: $(OBJDIR)/bin/firedancer-dev
2121

‎src/app/firedancer-dev/config/default.toml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@
2525
funk_sz_gb = 140
2626
funk_rec_max = 150000000
2727
funk_txn_max = 2000
28+
funk_file = "${LEDGER_DIR}/funk"
29+
[tiles.groove]
30+
cold_store_dir = "${LEDGER_DIR}/cold_store"
2831
[tiles.metric]
2932
prometheus_listen_address = "0.0.0.0"
3033
prometheus_listen_port = 7999

‎src/app/firedancer-dev/main.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ extern fd_topo_run_tile_t fd_tile_gossip;
8383
extern fd_topo_run_tile_t fd_tile_repair;
8484
extern fd_topo_run_tile_t fd_tile_storei;
8585
extern fd_topo_run_tile_t fd_tile_replay;
86+
extern fd_topo_run_tile_t fd_tile_groove;
8687
extern fd_topo_run_tile_t fd_tile_execor;
8788
extern fd_topo_run_tile_t fd_tile_writer;
8889
extern fd_topo_run_tile_t fd_tile_batch;
@@ -121,6 +122,7 @@ fd_topo_run_tile_t * TILES[] = {
121122
&fd_tile_repair,
122123
&fd_tile_storei,
123124
&fd_tile_replay,
125+
&fd_tile_groove,
124126
&fd_tile_execor,
125127
&fd_tile_writer,
126128
&fd_tile_batch,

‎src/app/firedancer/Local.mk

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ $(call add-objs,callbacks,fd_firedancer)
3737
$(call make-lib,firedancer_version)
3838
$(call add-objs,version,firedancer_version)
3939

40-
$(call make-bin,firedancer,main,fd_firedancer fdctl_shared fd_discof fd_disco fd_choreo fd_flamenco fd_funk fd_quic fd_tls fd_reedsol fd_ballet fd_waltz fd_tango fd_util firedancer_version, $(SECP256K1_LIBS))
40+
$(call make-bin,firedancer,main,fd_firedancer fdctl_shared fd_discof fd_disco fd_choreo fd_flamenco fd_funk fd_groove fd_quic fd_tls fd_reedsol fd_ballet fd_waltz fd_tango fd_util firedancer_version, $(SECP256K1_LIBS))
4141

4242
firedancer: $(OBJDIR)/bin/firedancer
4343
else

‎src/app/firedancer/main.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ extern fd_topo_run_tile_t fd_tile_gossip;
7070
extern fd_topo_run_tile_t fd_tile_repair;
7171
extern fd_topo_run_tile_t fd_tile_storei;
7272
extern fd_topo_run_tile_t fd_tile_replay;
73+
extern fd_topo_run_tile_t fd_tile_groove;
7374
extern fd_topo_run_tile_t fd_tile_execor;
7475
extern fd_topo_run_tile_t fd_tile_writer;
7576
extern fd_topo_run_tile_t fd_tile_batch;
@@ -97,6 +98,7 @@ fd_topo_run_tile_t * TILES[] = {
9798
&fd_tile_repair,
9899
&fd_tile_storei,
99100
&fd_tile_replay,
101+
&fd_tile_groove,
100102
&fd_tile_execor,
101103
&fd_tile_writer,
102104
&fd_tile_batch,

‎src/app/firedancer/topology.c

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,7 @@ fd_topo_initialize( config_t * config ) {
243243

244244
fd_topob_wksp( topo, "replay_exec" );
245245
fd_topob_wksp( topo, "replay_wtr" );
246+
fd_topob_wksp( topo, "replay_grv" );
246247
fd_topob_wksp( topo, "exec_writer" );
247248

248249
fd_topob_wksp( topo, "voter_sign" );
@@ -292,6 +293,7 @@ fd_topo_initialize( config_t * config ) {
292293
fd_topob_wksp( topo, "runtime_pub" );
293294
fd_topob_wksp( topo, "exec" );
294295
fd_topob_wksp( topo, "writer" );
296+
fd_topob_wksp( topo, "groove" );
295297
fd_topob_wksp( topo, "bstore" );
296298
fd_topob_wksp( topo, "tcache" );
297299
fd_topob_wksp( topo, "pohi" );
@@ -305,6 +307,7 @@ fd_topo_initialize( config_t * config ) {
305307
fd_topob_wksp( topo, "exec_spad" );
306308
fd_topob_wksp( topo, "exec_fseq" );
307309
fd_topob_wksp( topo, "writer_fseq" );
310+
fd_topob_wksp( topo, "groove_fseq" );
308311

309312
if( enable_rpc ) fd_topob_wksp( topo, "rpcsrv" );
310313

@@ -332,6 +335,7 @@ fd_topo_initialize( config_t * config ) {
332335
message that is outbound from the replay to exec. */
333336
FOR(exec_tile_cnt) fd_topob_link( topo, "replay_exec", "replay_exec", 128UL, 10240UL, exec_tile_cnt );
334337
FOR(writer_tile_cnt) fd_topob_link( topo, "replay_wtr", "replay_wtr", 128UL, FD_REPLAY_WRITER_MTU, 1UL );
338+
/**/ fd_topob_link( topo, "replay_grv", "replay_grv", 128UL, FD_REPLAY_GROOVE_MTU, 1UL );
335339
/* Assuming the number of writer tiles is sufficient to keep up with
336340
the number of exec tiles, under equilibrium, we should have at least
337341
enough link space to buffer worst case input shuffling done by the
@@ -437,7 +441,7 @@ fd_topo_initialize( config_t * config ) {
437441
fd_topo_tile_t * replay_tile = fd_topob_tile( topo, "replay", "replay", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
438442
FOR(exec_tile_cnt) fd_topob_tile( topo, "exec", "exec", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
439443
FOR(writer_tile_cnt) fd_topob_tile( topo, "writer", "writer", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
440-
fd_topo_tile_t * batch_tile = fd_topob_tile( topo, "batch", "batch", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
444+
fd_topo_tile_t * groove_tile = fd_topob_tile( topo, "groove", "groove", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
441445

442446
if( enable_rstart ) /* */ fd_topob_tile( topo, "rstart", "restart", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
443447

@@ -552,6 +556,12 @@ fd_topo_initialize( config_t * config ) {
552556
fd_topob_tile_uses( topo, batch_tile, constipated_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
553557
FD_TEST( fd_pod_insertf_ulong( topo->props, constipated_obj->id, "constipate" ) );
554558

559+
/* This fseq communicates the progress Groove is making to the Replay tile */
560+
fd_topo_obj_t * groove_fseq_obj = fd_topob_obj( topo, "fseq", "groove_fseq" );
561+
fd_topob_tile_uses( topo, groove_tile, groove_fseq_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
562+
fd_topob_tile_uses( topo, replay_tile, groove_fseq_obj, FD_SHMEM_JOIN_MODE_READ_ONLY );
563+
FD_TEST( fd_pod_insertf_ulong( topo->props, groove_fseq_obj->id, "groove_fseq" ) );
564+
555565
if( FD_LIKELY( !is_auto_affinity ) ) {
556566
if( FD_UNLIKELY( affinity_tile_cnt<topo->tile_cnt ) )
557567
FD_LOG_ERR(( "The topology you are using has %lu tiles, but the CPU affinity specified in the config tile as [layout.affinity] only provides for %lu cores. "
@@ -649,6 +659,7 @@ fd_topo_initialize( config_t * config ) {
649659
FOR(bank_tile_cnt) fd_topob_tile_out( topo, "replay", 0UL, "replay_poh", i );
650660
FOR(exec_tile_cnt) fd_topob_tile_out( topo, "replay", 0UL, "replay_exec", i ); /* TODO check order in fd_replay.c macros*/
651661
FOR(writer_tile_cnt) fd_topob_tile_out( topo, "replay", 0UL, "replay_wtr", i );
662+
/**/ fd_topob_tile_out( topo, "replay", 0UL, "replay_grv", 0UL );
652663

653664
FOR(exec_tile_cnt) fd_topob_tile_in( topo, "exec", i, "metric_in", "replay_exec", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
654665
FOR(exec_tile_cnt) fd_topob_tile_out( topo, "exec", i, "exec_writer", i );
@@ -657,6 +668,7 @@ fd_topo_initialize( config_t * config ) {
657668
FOR(writer_tile_cnt) for( ulong j=0UL; j<exec_tile_cnt; j++ )
658669
fd_topob_tile_in( topo, "writer", i, "metric_in", "exec_writer", j, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
659670
FOR(writer_tile_cnt) fd_topob_tile_in( topo, "writer", i, "metric_in", "replay_wtr", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
671+
/**/ fd_topob_tile_in( topo, "groove", 0UL, "metric_in", "replay_grv", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
660672
661673
/**/ fd_topob_tile_in( topo, "sender", 0UL, "metric_in", "stake_out", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */
662674
/**/ fd_topob_tile_in( topo, "sender", 0UL, "metric_in", "gossip_voter", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */
@@ -938,6 +950,9 @@ fd_topo_initialize( config_t * config ) {
938950
strncpy( tile->exec.funk_file, config->tiles.replay.funk_file, sizeof(tile->exec.funk_file) );
939951
} else if( FD_UNLIKELY( !strcmp( tile->name, "writer" ) ) ) {
940952
strncpy( tile->writer.funk_file, config->tiles.replay.funk_file, sizeof(tile->writer.funk_file) );
953+
} else if( FD_UNLIKELY( !strcmp( tile->name, "groove" ) ) ) {
954+
strncpy( tile->groove.funk_file, config->tiles.replay.funk_file, sizeof(tile->groove.funk_file) );
955+
strncpy( tile->groove.cold_store_dir, config->tiles.groove.cold_store_dir, sizeof(tile->groove.cold_store_dir) );
941956
} else if( FD_UNLIKELY( !strcmp( tile->name, "rstart" ) ) ) {
942957
strncpy( tile->restart.funk_file, config->tiles.replay.funk_file, sizeof(tile->replay.funk_file) );
943958
strncpy( tile->restart.tower_checkpt, config->tiles.replay.tower_checkpt, sizeof(tile->replay.tower_checkpt) );

‎src/app/ledger/Local.mk

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
ifdef FD_HAS_ROCKSDB
22

33
ifdef FD_HAS_SECP256K1
4-
$(call make-bin,fd_ledger,main,fd_flamenco fd_ballet fd_reedsol fd_funk fd_tango fd_choreo fd_waltz fd_util fd_disco,$(ROCKSDB_LIBS) $(SECP256K1_LIBS))
4+
$(call make-bin,fd_ledger,main,fd_flamenco fd_ballet fd_reedsol fd_funk fd_groove fd_tango fd_choreo fd_waltz fd_util fd_disco,$(ROCKSDB_LIBS) $(SECP256K1_LIBS))
55
else
66
$(warning ledger tool build disabled due to lack of secp256k1)
77
endif

‎src/app/ledger/main.c

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1125,8 +1125,6 @@ ingest( fd_ledger_args_t * args ) {
11251125
args->verify_acc_hash,
11261126
args->check_acc_hash ,
11271127
FD_SNAPSHOT_TYPE_FULL,
1128-
args->exec_spads,
1129-
args->exec_spad_cnt,
11301128
args->runtime_spad );
11311129
FD_LOG_NOTICE(( "imported records from snapshot" ));
11321130
}
@@ -1140,8 +1138,6 @@ ingest( fd_ledger_args_t * args ) {
11401138
args->verify_acc_hash,
11411139
args->check_acc_hash,
11421140
FD_SNAPSHOT_TYPE_INCREMENTAL,
1143-
args->exec_spads,
1144-
args->exec_spad_cnt,
11451141
args->runtime_spad );
11461142
FD_LOG_NOTICE(( "imported records from incremental snapshot" ));
11471143
}
@@ -1316,8 +1312,6 @@ replay( fd_ledger_args_t * args ) {
13161312
args->verify_acc_hash,
13171313
args->check_acc_hash,
13181314
FD_SNAPSHOT_TYPE_FULL,
1319-
args->exec_spads,
1320-
args->exec_spad_cnt,
13211315
args->runtime_spad );
13221316
FD_LOG_NOTICE(( "imported from snapshot" ));
13231317
if( args->incremental ) {
@@ -1330,8 +1324,6 @@ replay( fd_ledger_args_t * args ) {
13301324
args->verify_acc_hash,
13311325
args->check_acc_hash,
13321326
FD_SNAPSHOT_TYPE_INCREMENTAL,
1333-
args->exec_spads,
1334-
args->exec_spad_cnt,
13351327
args->runtime_spad );
13361328
FD_LOG_NOTICE(( "imported from snapshot" ));
13371329
}

‎src/app/rpcserver/Local.mk

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
ifdef FD_HAS_HOSTED
22
ifdef FD_HAS_INT128
33
ifdef FD_HAS_SSE
4-
$(call make-bin,fd_rpcserver,main,fd_discof fd_disco fd_flamenco fd_reedsol fd_ballet fd_funk fd_tango fd_choreo fd_waltz fd_util, $(SECP256K1_LIBS))
4+
$(call make-bin,fd_rpcserver,main,fd_discof fd_disco fd_flamenco fd_reedsol fd_ballet fd_funk fd_groove fd_tango fd_choreo fd_waltz fd_util, $(SECP256K1_LIBS))
55
endif
66
endif
77
endif

‎src/app/shared/fd_config.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,10 @@ struct fd_config {
408408
char archiver_path[ PATH_MAX ];
409409
} archiver;
410410

411+
struct {
412+
char cold_store_dir[ PATH_MAX ];
413+
} groove;
414+
411415
} tiles;
412416
};
413417

‎src/app/shared/fd_config_parse.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -475,6 +475,8 @@ fd_config_extract_pod( uchar * pod,
475475
CFG_POP ( ulong, tiles.archiver.end_slot );
476476
CFG_POP ( cstr, tiles.archiver.archiver_path );
477477

478+
CFG_POP ( cstr, tiles.groove.cold_store_dir );
479+
478480
CFG_POP ( bool, development.sandbox );
479481
CFG_POP ( bool, development.no_clone );
480482
CFG_POP ( bool, development.core_dump );

‎src/disco/fd_disco_base.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ FD_STATIC_ASSERT( FD_SHRED_REPAIR_MTU == 120 , update FD_SHRED_REPAIR_MTU );
6262
#define FD_REPLAY_WRITER_MTU (128UL)
6363
#define FD_EXEC_WRITER_MTU (128UL)
6464

65+
/* Maximum size of frags going into the Groove tile. */
66+
#define FD_REPLAY_GROOVE_MTU (48UL)
6567

6668
#define FD_NETMUX_SIG_MIN_HDR_SZ ( 42UL) /* The default header size, which means no vlan tags and no IP options. */
6769
#define FD_NETMUX_SIG_IGNORE_HDR_SZ (102UL) /* Outside the allowable range, but still fits in 4 bits when compressed */

‎src/disco/topo/fd_topo.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,11 @@ typedef struct {
435435
int archive_fd;
436436
} archiver;
437437

438+
struct {
439+
char funk_file[ PATH_MAX ];
440+
char cold_store_dir[ PATH_MAX ];
441+
} groove;
442+
438443
};
439444
} fd_topo_tile_t;
440445

‎src/disco/topo/fd_topob.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,7 @@ fd_topob_auto_layout( fd_topo_t * topo,
373373
"eqvoc", /* FIREDANCER only */
374374
"rpcsrv", /* FIREDANCER only */
375375
"batch", /* FIREDANCER only */
376+
"groove", /* FIREDANCER only */
376377
"pktgen",
377378
};
378379

‎src/discof/groove/Local.mk

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
ifdef FD_HAS_INT128
2+
$(call add-objs,fd_groove_tile,fd_discof)
3+
endif

‎src/discof/groove/fd_groove_messages.h

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#include <linux/limits.h>
2+
#include "../../flamenco/types/fd_types.h"
3+
#include "../../funk/fd_funk_txn.h"
4+
5+
/******************************** Groove Tile API **************************************/
6+
7+
/* Signatures */
8+
#define FD_GROOVE_TILE_PREFETCH_SIGNATURE (0UL)
9+
#define FD_GROOVE_TILE_BLOCKING_PREFETCH_SIGNATURE (1UL)
10+
#define FD_GROOVE_TILE_LOAD_SNAPSHOT_SIGNATURE (2UL)
11+
12+
/* Fseqs */
13+
#define FD_MSG_GROOVE_REPLAY_FSEQ_INIT (0UL)
14+
#define FD_MSG_GROOVE_REPLAY_FSEQ_LOAD_SNAPSHOT_DONE (1UL)
15+
16+
/* Tango Messages */
17+
struct fd_msg_groove_replay_load_snapshot_req {
18+
char snapshot_path[PATH_MAX];
19+
char snapshot_dir[PATH_MAX];
20+
int snapshot_src_type;
21+
};
22+
typedef struct fd_msg_groove_replay_load_snapshot_req fd_msg_groove_replay_load_snapshot_req_t;
23+
24+
struct fd_msg_groove_prefetch_account_req {
25+
fd_pubkey_t pubkey;
26+
fd_funk_txn_xid_t funk_txn_xid;
27+
};
28+
typedef struct fd_msg_groove_prefetch_account_req fd_msg_groove_prefetch_account_req_t;

0 commit comments

Comments
 (0)