diff --git a/src/app/firedancer-dev/commands/backtest.c b/src/app/firedancer-dev/commands/backtest.c index aa3207fa5a..7d99386816 100644 --- a/src/app/firedancer-dev/commands/backtest.c +++ b/src/app/firedancer-dev/commands/backtest.c @@ -66,7 +66,6 @@ setup_topo_blockstore( fd_topo_t * topo, ulong shred_max, ulong block_max, ulong idx_max, - ulong txn_max, ulong alloc_max ) { fd_topo_obj_t * obj = fd_topob_obj( topo, "blockstore", wksp_name ); @@ -78,12 +77,11 @@ setup_topo_blockstore( fd_topo_t * topo, FD_TEST( fd_pod_insertf_ulong( topo->props, shred_max, "obj.%lu.shred_max", obj->id ) ); FD_TEST( fd_pod_insertf_ulong( topo->props, block_max, "obj.%lu.block_max", obj->id ) ); FD_TEST( fd_pod_insertf_ulong( topo->props, idx_max, "obj.%lu.idx_max", obj->id ) ); - FD_TEST( fd_pod_insertf_ulong( topo->props, txn_max, "obj.%lu.txn_max", obj->id ) ); FD_TEST( fd_pod_insertf_ulong( topo->props, alloc_max, "obj.%lu.alloc_max", obj->id ) ); /* DO NOT MODIFY LOOSE WITHOUT CHANGING HOW BLOCKSTORE ALLOCATES INTERNAL STRUCTURES */ - ulong blockstore_footprint = fd_blockstore_footprint( shred_max, block_max, idx_max, txn_max ) + alloc_max; + ulong blockstore_footprint = fd_blockstore_footprint( shred_max, block_max, idx_max ) + alloc_max; FD_TEST( fd_pod_insertf_ulong( topo->props, blockstore_footprint, "obj.%lu.loose", obj->id ) ); return obj; @@ -344,7 +342,6 @@ backtest_topo( config_t * config ) { config->firedancer.blockstore.shred_max, config->firedancer.blockstore.block_max, config->firedancer.blockstore.idx_max, - config->firedancer.blockstore.txn_max, config->firedancer.blockstore.alloc_max ); fd_topob_tile_uses( topo, replay_tile, blockstore_obj, FD_SHMEM_JOIN_MODE_READ_WRITE ); fd_topob_tile_uses( topo, backtest_tile, blockstore_obj, FD_SHMEM_JOIN_MODE_READ_WRITE ); diff --git a/src/app/firedancer-dev/commands/sim.c b/src/app/firedancer-dev/commands/sim.c index 0417dfc3fe..0f3403595b 100644 --- a/src/app/firedancer-dev/commands/sim.c +++ b/src/app/firedancer-dev/commands/sim.c @@ -66,7 +66,6 @@ setup_topo_blockstore( fd_topo_t * topo, ulong shred_max, ulong block_max, ulong idx_max, - ulong txn_max, ulong alloc_max ) { fd_topo_obj_t * obj = fd_topob_obj( topo, "blockstore", wksp_name ); @@ -78,12 +77,11 @@ setup_topo_blockstore( fd_topo_t * topo, FD_TEST( fd_pod_insertf_ulong( topo->props, shred_max, "obj.%lu.shred_max", obj->id ) ); FD_TEST( fd_pod_insertf_ulong( topo->props, block_max, "obj.%lu.block_max", obj->id ) ); FD_TEST( fd_pod_insertf_ulong( topo->props, idx_max, "obj.%lu.idx_max", obj->id ) ); - FD_TEST( fd_pod_insertf_ulong( topo->props, txn_max, "obj.%lu.txn_max", obj->id ) ); FD_TEST( fd_pod_insertf_ulong( topo->props, alloc_max, "obj.%lu.alloc_max", obj->id ) ); /* DO NOT MODIFY LOOSE WITHOUT CHANGING HOW BLOCKSTORE ALLOCATES INTERNAL STRUCTURES */ - ulong blockstore_footprint = fd_blockstore_footprint( shred_max, block_max, idx_max, txn_max ) + alloc_max; + ulong blockstore_footprint = fd_blockstore_footprint( shred_max, block_max, idx_max ) + alloc_max; FD_TEST( fd_pod_insertf_ulong( topo->props, blockstore_footprint, "obj.%lu.loose", obj->id ) ); return obj; @@ -260,7 +258,6 @@ sim_topo( config_t * config ) { config->firedancer.blockstore.shred_max, config->firedancer.blockstore.block_max, config->firedancer.blockstore.idx_max, - config->firedancer.blockstore.txn_max, config->firedancer.blockstore.alloc_max ); fd_topo_obj_t * poh_shred_obj = fd_topob_obj( topo, "fseq", "poh_shred" ); fd_topo_obj_t * root_slot_obj = fd_topob_obj( topo, "fseq", "root_slot" ); diff --git a/src/app/firedancer/callbacks.c b/src/app/firedancer/callbacks.c index a382322e3e..ab487807c1 100644 --- a/src/app/firedancer/callbacks.c +++ b/src/app/firedancer/callbacks.c @@ -40,7 +40,7 @@ fd_topo_obj_callbacks_t fd_obj_cb_runtime_pub = { static ulong blockstore_footprint( fd_topo_t const * topo, fd_topo_obj_t const * obj ) { - return fd_blockstore_footprint( VAL("shred_max"), VAL("block_max"), VAL("idx_max"), VAL("txn_max") ) + VAL("alloc_max"); + return fd_blockstore_footprint( VAL("shred_max"), VAL("block_max"), VAL("idx_max") ) + VAL("alloc_max"); } static ulong @@ -52,7 +52,7 @@ blockstore_align( fd_topo_t const * topo FD_FN_UNUSED, static void blockstore_new( fd_topo_t const * topo, fd_topo_obj_t const * obj ) { - FD_TEST( fd_blockstore_new( fd_topo_obj_laddr( topo, obj->id ), VAL("wksp_tag"), VAL("seed"), VAL("shred_max"), VAL("block_max"), VAL("idx_max"), VAL("txn_max") ) ); + FD_TEST( fd_blockstore_new( fd_topo_obj_laddr( topo, obj->id ), VAL("wksp_tag"), VAL("seed"), VAL("shred_max"), VAL("block_max"), VAL("idx_max") ) ); } fd_topo_obj_callbacks_t fd_obj_cb_blockstore = { diff --git a/src/app/firedancer/topology.c b/src/app/firedancer/topology.c index 39c4acfe5b..f5b8504f6a 100644 --- a/src/app/firedancer/topology.c +++ b/src/app/firedancer/topology.c @@ -27,7 +27,6 @@ setup_topo_blockstore( fd_topo_t * topo, ulong shred_max, ulong block_max, ulong idx_max, - ulong txn_max, ulong alloc_max ) { fd_topo_obj_t * obj = fd_topob_obj( topo, "blockstore", wksp_name ); @@ -39,12 +38,11 @@ setup_topo_blockstore( fd_topo_t * topo, FD_TEST( fd_pod_insertf_ulong( topo->props, shred_max, "obj.%lu.shred_max", obj->id ) ); FD_TEST( fd_pod_insertf_ulong( topo->props, block_max, "obj.%lu.block_max", obj->id ) ); FD_TEST( fd_pod_insertf_ulong( topo->props, idx_max, "obj.%lu.idx_max", obj->id ) ); - FD_TEST( fd_pod_insertf_ulong( topo->props, txn_max, "obj.%lu.txn_max", obj->id ) ); FD_TEST( fd_pod_insertf_ulong( topo->props, alloc_max, "obj.%lu.alloc_max", obj->id ) ); /* DO NOT MODIFY LOOSE WITHOUT CHANGING HOW BLOCKSTORE ALLOCATES INTERNAL STRUCTURES */ - ulong blockstore_footprint = fd_blockstore_footprint( shred_max, block_max, idx_max, txn_max ) + alloc_max; + ulong blockstore_footprint = fd_blockstore_footprint( shred_max, block_max, idx_max ) + alloc_max; FD_TEST( fd_pod_insertf_ulong( topo->props, blockstore_footprint, "obj.%lu.loose", obj->id ) ); return obj; @@ -460,7 +458,6 @@ fd_topo_initialize( config_t * config ) { config->firedancer.blockstore.shred_max, config->firedancer.blockstore.block_max, config->firedancer.blockstore.idx_max, - config->firedancer.blockstore.txn_max, config->firedancer.blockstore.alloc_max ); fd_topob_tile_uses( topo, store_tile, blockstore_obj, FD_SHMEM_JOIN_MODE_READ_WRITE ); fd_topob_tile_uses( topo, replay_tile, blockstore_obj, FD_SHMEM_JOIN_MODE_READ_WRITE ); diff --git a/src/app/ledger/main.c b/src/app/ledger/main.c index 8e2ab880d9..dc4e3a1acb 100644 --- a/src/app/ledger/main.c +++ b/src/app/ledger/main.c @@ -358,7 +358,6 @@ runtime_replay( fd_ledger_args_t * ledger_args ) { if( !block_exists && slot_meta.slot == slot ) { int err = fd_rocksdb_import_block_blockstore( &rocks_db, &slot_meta, blockstore, - ledger_args->copy_txn_status, slot == (ledger_args->trash_hash) ? trash_hash_buf : NULL, ledger_args->valloc ); if( FD_UNLIKELY( err ) ) { @@ -761,7 +760,6 @@ ingest_rocksdb( char const * file, ulong start_slot, ulong end_slot, fd_blockstore_t * blockstore, - int txn_status, ulong trash_hash, fd_valloc_t valloc ) { @@ -818,7 +816,6 @@ ingest_rocksdb( char const * file, int err = fd_rocksdb_import_block_blockstore( &rocks_db, &slot_meta, blockstore, - txn_status, (slot == trash_hash) ? trash_hash_buf : NULL, valloc ); if( FD_UNLIKELY( err ) ) { @@ -935,12 +932,11 @@ init_blockstore( fd_ledger_args_t * args ) { } FD_LOG_NOTICE(( "joined blockstore" )); } else { - ulong txn_max = 256UL; - shmem = fd_wksp_alloc_laddr( args->wksp, fd_blockstore_align(), fd_blockstore_footprint( args->shred_max, args->slot_history_max, 16, txn_max ), blockstore_tag ); + shmem = fd_wksp_alloc_laddr( args->wksp, fd_blockstore_align(), fd_blockstore_footprint( args->shred_max, args->slot_history_max, 16 ), blockstore_tag ); if( shmem == NULL ) { FD_LOG_ERR(( "failed to allocate a blockstore" )); } - args->blockstore = fd_blockstore_join( &args->blockstore_ljoin, fd_blockstore_new( shmem, 1, args->hashseed, args->shred_max, args->slot_history_max, 16, txn_max ) ); + args->blockstore = fd_blockstore_join( &args->blockstore_ljoin, fd_blockstore_new( shmem, 1, args->hashseed, args->shred_max, args->slot_history_max, 16 ) ); if( args->blockstore->shmem->magic != FD_BLOCKSTORE_MAGIC ) { fd_wksp_free_laddr( shmem ); FD_LOG_ERR(( "failed to allocate a blockstore" )); @@ -1050,13 +1046,9 @@ minify( fd_ledger_args_t * args ) { args->start_slot, args->end_slot, args->blockstore, - 0, ULONG_MAX, args->valloc ); - fd_rocksdb_copy_over_txn_status_range( &big_rocksdb, &mini_rocksdb, args->blockstore, - args->start_slot, args->end_slot ); - FD_LOG_NOTICE(( "copied over all transaction statuses" )); } else { FD_LOG_NOTICE(( "skipping copying of transaction statuses" )); } @@ -1172,7 +1164,7 @@ ingest( fd_ledger_args_t * args ) { args->end_slot = slot_ctx->slot_bank.slot + args->slot_history_max - 1; } ingest_rocksdb( args->rocksdb_list[ 0UL ], args->start_slot, args->end_slot, - blockstore, args->copy_txn_status, args->trash_hash, args->valloc ); + blockstore, args->trash_hash, args->valloc ); } /* Verification */ diff --git a/src/app/shredcap/main.c b/src/app/shredcap/main.c index e44df2755f..6345e567b3 100644 --- a/src/app/shredcap/main.c +++ b/src/app/shredcap/main.c @@ -81,12 +81,12 @@ main( int argc, char ** argv ) { FD_LOG_ERR(( "failed to join a blockstore" )); } } else { - shmem = fd_wksp_alloc_laddr( wksp, fd_blockstore_align(), fd_blockstore_footprint( shred_max, slot_history_max, 16, shred_max ), FD_BLOCKSTORE_MAGIC ); + shmem = fd_wksp_alloc_laddr( wksp, fd_blockstore_align(), fd_blockstore_footprint( shred_max, slot_history_max, 16 ), FD_BLOCKSTORE_MAGIC ); if ( shmem == NULL ) { FD_LOG_ERR(( "failed to allocate a blockstore" )); } - blockstore = fd_blockstore_join( &blockstore_ljoin, fd_blockstore_new( shmem, 1, hashseed, shred_max, slot_history_max, 16, shred_max ) ); + blockstore = fd_blockstore_join( &blockstore_ljoin, fd_blockstore_new( shmem, 1, hashseed, shred_max, slot_history_max, 16 ) ); if ( blockstore == NULL ) { fd_wksp_free_laddr( shmem ); FD_LOG_ERR(( "failed to allocate a blockstore" )); diff --git a/src/discof/rpcserver/fd_block_to_json.c b/src/discof/rpcserver/fd_block_to_json.c index 5c758f0fd0..fada1bf402 100644 --- a/src/discof/rpcserver/fd_block_to_json.c +++ b/src/discof/rpcserver/fd_block_to_json.c @@ -672,8 +672,6 @@ fd_txn_to_json( fd_webserver_t * ws, const char* fd_block_to_json( fd_webserver_t * ws, - fd_blockstore_t * blockstore, - int blockstore_fd, const char * call_id, const uchar * blk_data, ulong blk_sz, @@ -766,8 +764,6 @@ fd_block_to_json( fd_webserver_t * ws, EMIT_SIMPLE("\"transactions\":["); - fd_wksp_t * blockstore_wksp = fd_blockstore_wksp( blockstore ); - int first_txn = 1; ulong blockoff = 0; while (blockoff < blk_sz) { @@ -801,6 +797,7 @@ fd_block_to_json( fd_webserver_t * ws, } else EMIT_SIMPLE(",{"); + /* FIXME uchar const * sig_p = raw + ((fd_txn_t *)txn_out)->signature_off; fd_txn_map_t elem; uchar flags; @@ -809,6 +806,7 @@ fd_block_to_json( fd_webserver_t * ws, const char * err = fd_txn_meta_to_json( ws, meta, elem.meta_sz ); if ( err ) return err; } + */ const char * err = fd_txn_to_json( ws, (fd_txn_t *)txn_out, raw, pay_sz, encoding, maxvers, detail, spad ); if ( err ) return err; diff --git a/src/discof/rpcserver/fd_block_to_json.h b/src/discof/rpcserver/fd_block_to_json.h index 6adcf11903..e39ee3bbd3 100644 --- a/src/discof/rpcserver/fd_block_to_json.h +++ b/src/discof/rpcserver/fd_block_to_json.h @@ -29,8 +29,6 @@ const char* fd_txn_to_json( fd_webserver_t * ws, fd_spad_t * spad ); const char* fd_block_to_json( fd_webserver_t * ws, - fd_blockstore_t * blockstore, - int blockstore_fd, const char * call_id, const uchar * blk_data, ulong blk_sz, diff --git a/src/discof/rpcserver/fd_rpc_service.c b/src/discof/rpcserver/fd_rpc_service.c index 0d18597ddb..c4d6347f80 100644 --- a/src/discof/rpcserver/fd_rpc_service.c +++ b/src/discof/rpcserver/fd_rpc_service.c @@ -245,6 +245,7 @@ read_slot_bank( fd_rpc_ctx_t * ctx, ulong slot ) { return slot_bank; } +/* static const char * block_flags_to_confirmation_status( uchar flags ) { if( flags & (1U << FD_BLOCK_FLAG_FINALIZED) ) return "\"finalized\""; @@ -252,6 +253,7 @@ block_flags_to_confirmation_status( uchar flags ) { if( flags & (1U << FD_BLOCK_FLAG_PROCESSED) ) return "\"processed\""; return "null"; } +*/ // Implementation of the "getAccountInfo" method // curl http://localhost:8123 -X POST -H "Content-Type: application/json" -d '{ "jsonrpc": "2.0", "id": 1, "method": "getAccountInfo", "params": [ "21bVZhkqPJRVYDG3YpYtzHLMvkc7sa4KB7fMwGekTquG", { "encoding": "base64" } ] }' @@ -484,8 +486,6 @@ method_getBlock(struct json_values* values, fd_rpc_ctx_t * ctx) { } const char * err = fd_block_to_json(ws, - blockstore, - ctx->global->blockstore_fd, ctx->call_id, blk_data, blk_sz, @@ -1305,7 +1305,6 @@ method_getSignaturesForAddress(struct json_values* values, fd_rpc_ctx_t * ctx) { static int method_getSignatureStatuses(struct json_values* values, fd_rpc_ctx_t * ctx) { fd_webserver_t * ws = &ctx->global->ws; - fd_blockstore_t * blockstore = ctx->global->blockstore; fd_web_reply_sprintf(ws, "{\"jsonrpc\":\"2.0\",\"result\":{\"context\":{\"apiVersion\":\"" FIREDANCER_VERSION "\",\"slot\":%lu},\"value\":[", ctx->global->last_slot); @@ -1331,6 +1330,7 @@ method_getSignatureStatuses(struct json_values* values, fd_rpc_ctx_t * ctx) { fd_web_reply_sprintf(ws, "null"); continue; } + /* FIXME fd_txn_map_t elem; uchar flags; if( fd_blockstore_txn_query_volatile( blockstore, ctx->global->blockstore_fd, key, &elem, NULL, &flags, NULL ) ) { @@ -1341,6 +1341,7 @@ method_getSignatureStatuses(struct json_values* values, fd_rpc_ctx_t * ctx) { // TODO other fields fd_web_reply_sprintf(ws, "{\"slot\":%lu,\"confirmations\":null,\"err\":null,\"status\":{\"Ok\":null},\"confirmationStatus\":%s}", elem.slot, block_flags_to_confirmation_status(flags)); + */ } fd_web_reply_sprintf(ws, "]},\"id\":%s}" CRLF, ctx->call_id); @@ -1586,6 +1587,8 @@ method_getTransaction(struct json_values* values, fd_rpc_ctx_t * ctx) { fd_web_reply_sprintf(ws, "{\"jsonrpc\":\"2.0\",\"result\":null,\"id\":%s}" CRLF, ctx->call_id); return 0; } + (void)enc; +#if 0 /* FIXME */ fd_txn_map_t elem; long blk_ts; uchar blk_flags; @@ -1620,6 +1623,7 @@ method_getTransaction(struct json_values* values, fd_rpc_ctx_t * ctx) { fd_method_error(ctx, -1, "%s", err); return 0; } +#endif fd_web_reply_sprintf(ws, "},\"id\":%s}" CRLF, ctx->call_id); return 0; diff --git a/src/discof/writer/fd_writer_tile.c b/src/discof/writer/fd_writer_tile.c index f1f894b179..36e7171f37 100644 --- a/src/discof/writer/fd_writer_tile.c +++ b/src/discof/writer/fd_writer_tile.c @@ -189,7 +189,7 @@ during_frag( fd_writer_tile_ctx_t * ctx, FD_SPIN_PAUSE(); } FD_SPAD_FRAME_BEGIN( ctx->spad ) { - fd_runtime_finalize_txn( ctx->slot_ctx, NULL, &info, ctx->spad ); + fd_runtime_finalize_txn( ctx->slot_ctx, &info, ctx->spad ); } FD_SPAD_FRAME_END; } /* Notify the replay tile. */ diff --git a/src/flamenco/runtime/fd_blockstore.c b/src/flamenco/runtime/fd_blockstore.c index 1e8b4c746f..c1c93f1941 100644 --- a/src/flamenco/runtime/fd_blockstore.c +++ b/src/flamenco/runtime/fd_blockstore.c @@ -11,8 +11,7 @@ fd_blockstore_new( void * shmem, ulong seed, ulong shred_max, ulong block_max, - ulong idx_max, - ulong txn_max ) { + ulong idx_max ) { /* TODO temporary fix to make sure block_max is a power of 2, as required for slot map para. We should change to err in config verification eventually */ @@ -47,7 +46,7 @@ fd_blockstore_new( void * shmem, FD_LOG_WARNING(( "blockstore implementation requires shred_max to be a power of two, rounding it up to %lu", shred_max )); } - fd_memset( blockstore_shmem, 0, fd_blockstore_footprint( shred_max, block_max, idx_max, txn_max ) ); + fd_memset( blockstore_shmem, 0, fd_blockstore_footprint( shred_max, block_max, idx_max ) ); int lg_idx_max = fd_ulong_find_msb( fd_ulong_pow2_up( idx_max ) ); @@ -60,10 +59,9 @@ fd_blockstore_new( void * shmem, void * block_map = FD_SCRATCH_ALLOC_APPEND( l, fd_block_map_align(), fd_block_map_footprint( block_max, lock_cnt, BLOCK_INFO_PROBE_CNT ) ); void * block_idx = FD_SCRATCH_ALLOC_APPEND( l, fd_block_idx_align(), fd_block_idx_footprint( lg_idx_max ) ); void * slot_deque = FD_SCRATCH_ALLOC_APPEND( l, fd_slot_deque_align(), fd_slot_deque_footprint( block_max ) ); - void * txn_map = FD_SCRATCH_ALLOC_APPEND( l, fd_txn_map_align(), fd_txn_map_footprint( txn_max ) ); void * alloc = FD_SCRATCH_ALLOC_APPEND( l, fd_alloc_align(), fd_alloc_footprint() ); ulong top = FD_SCRATCH_ALLOC_FINI( l, fd_blockstore_align() ); - FD_TEST( fd_ulong_align_up( top - (ulong)shmem, fd_alloc_align() ) == fd_ulong_align_up( fd_blockstore_footprint( shred_max, block_max, idx_max, txn_max ), fd_alloc_align() ) ); + FD_TEST( fd_ulong_align_up( top - (ulong)shmem, fd_alloc_align() ) == fd_ulong_align_up( fd_blockstore_footprint( shred_max, block_max, idx_max ), fd_alloc_align() ) ); (void)shreds; fd_buf_shred_pool_new( shred_pool ); @@ -82,12 +80,10 @@ fd_blockstore_new( void * shmem, blockstore_shmem->block_idx_gaddr = fd_wksp_gaddr( wksp, fd_block_idx_join( fd_block_idx_new( block_idx, lg_idx_max ) ) ); blockstore_shmem->slot_deque_gaddr = fd_wksp_gaddr( wksp, fd_slot_deque_join (fd_slot_deque_new( slot_deque, block_max ) ) ); - blockstore_shmem->txn_map_gaddr = fd_wksp_gaddr( wksp, fd_txn_map_join (fd_txn_map_new( txn_map, txn_max, seed ) ) ); blockstore_shmem->alloc_gaddr = fd_wksp_gaddr( wksp, fd_alloc_join (fd_alloc_new( alloc, wksp_tag ), wksp_tag ) ); FD_TEST( blockstore_shmem->block_idx_gaddr ); FD_TEST( blockstore_shmem->slot_deque_gaddr ); - FD_TEST( blockstore_shmem->txn_map_gaddr ); FD_TEST( blockstore_shmem->alloc_gaddr ); blockstore_shmem->blockstore_gaddr = fd_wksp_gaddr_fast( wksp, blockstore_shmem ); @@ -108,7 +104,6 @@ fd_blockstore_new( void * shmem, blockstore_shmem->shred_max = shred_max; blockstore_shmem->block_max = block_max; blockstore_shmem->idx_max = idx_max; - blockstore_shmem->txn_max = txn_max; FD_COMPILER_MFENCE(); FD_VOLATILE( blockstore_shmem->magic ) = FD_BLOCKSTORE_MAGIC; @@ -189,7 +184,6 @@ fd_blockstore_leave( fd_blockstore_t * blockstore ) { FD_TEST( fd_block_map_leave( blockstore->block_map ) ); FD_TEST( fd_block_idx_leave( fd_blockstore_block_idx( blockstore ) ) ); FD_TEST( fd_slot_deque_leave( fd_blockstore_slot_deque( blockstore ) ) ); - FD_TEST( fd_txn_map_leave( fd_blockstore_txn_map( blockstore ) ) ); FD_TEST( fd_alloc_leave( fd_blockstore_alloc( blockstore ) ) ); return (void *)blockstore; @@ -227,7 +221,6 @@ fd_blockstore_delete( void * shblockstore ) { FD_TEST( fd_block_map_delete( &blockstore->block_map ) ); FD_TEST( fd_block_idx_delete( fd_blockstore_block_idx( blockstore ) ) ); FD_TEST( fd_slot_deque_delete( fd_blockstore_slot_deque( blockstore ) ) ); - FD_TEST( fd_txn_map_delete( fd_blockstore_txn_map( blockstore ) ) ); FD_TEST( fd_alloc_delete( fd_blockstore_alloc( blockstore ) ) ); FD_COMPILER_MFENCE(); @@ -320,25 +313,7 @@ fd_blockstore_fini( fd_blockstore_t * blockstore ) { } } -/* txn map helpers */ - -FD_FN_PURE int -fd_txn_key_equal( fd_txn_key_t const * k0, fd_txn_key_t const * k1 ) { - for( ulong i = 0; i < FD_ED25519_SIG_SZ / sizeof( ulong ); ++i ) - if( k0->v[i] != k1->v[i] ) return 0; - return 1; -} - -FD_FN_PURE ulong -fd_txn_key_hash( fd_txn_key_t const * k, ulong seed ) { - ulong h = seed; - for( ulong i = 0; i < FD_ED25519_SIG_SZ / sizeof( ulong ); ++i ) - h ^= k->v[i]; - return h; -} - -/* Remove a slot from blockstore. Needs to currently be under a blockstore_write - lock due to txn_map access. */ +/* Remove a slot from blockstore */ void fd_blockstore_slot_remove( fd_blockstore_t * blockstore, ulong slot ) { FD_LOG_DEBUG(( "[%s] slot: %lu", __func__, slot )); @@ -931,106 +906,6 @@ fd_blockstore_block_map_query_volatile( fd_blockstore_t * blockstore, return FD_BLOCKSTORE_SUCCESS; } -fd_txn_map_t * -fd_blockstore_txn_query( fd_blockstore_t * blockstore, uchar const sig[FD_ED25519_SIG_SZ] ) { - fd_txn_key_t key; - fd_memcpy( &key, sig, sizeof( key ) ); - return fd_txn_map_query( fd_blockstore_txn_map( blockstore ), &key, NULL ); -} - -int -fd_blockstore_txn_query_volatile( fd_blockstore_t * blockstore, - int fd, - uchar const sig[FD_ED25519_SIG_SZ], - fd_txn_map_t * txn_out, - long * blk_ts, - uchar * blk_flags, - uchar txn_data_out[FD_TXN_MTU] ) { - /* WARNING: this code is extremely delicate. Do NOT modify without - understanding all the invariants. In particular, we must never - dereference through a corrupt pointer. It's OK for the - destination data to be overwritten/invalid as long as the memory - location is valid. As long as we don't crash, we can validate the - data after it is read. */ - (void)blockstore; - (void)fd; - (void)sig; - (void)txn_out; - (void)blk_ts; - (void)blk_flags; - (void)txn_data_out; - return FD_BLOCKSTORE_ERR_SLOT_MISSING; -#if BLOCK_ARCHIVING - fd_wksp_t * wksp = fd_blockstore_wksp( blockstore ); - fd_txn_map_t * txn_map = fd_blockstore_txn_map( blockstore ); - - for(;;) { - fd_txn_key_t key; - memcpy( &key, sig, sizeof(key) ); - fd_txn_map_t const * txn_map_entry = fd_txn_map_query_safe( txn_map, &key, NULL ); - if( FD_UNLIKELY( txn_map_entry == NULL ) ) return FD_BLOCKSTORE_ERR_TXN_MISSING; - memcpy( txn_out, txn_map_entry, sizeof(fd_txn_map_t) ); - break; - } - - fd_block_idx_t * block_idx = fd_blockstore_block_idx( blockstore ); - - ulong off = ULONG_MAX; - for(;;) { - fd_block_idx_t * idx_entry = fd_block_idx_query( block_idx, txn_out->slot, NULL ); - if( FD_LIKELY( idx_entry ) ) off = idx_entry->off; - break; - } - - if ( FD_UNLIKELY( off < ULONG_MAX ) ) { /* optimize for non-archival */ - if( FD_UNLIKELY( lseek( fd, (long)off, SEEK_SET ) == -1 ) ) { - FD_LOG_WARNING(( "failed to seek" )); - return FD_BLOCKSTORE_ERR_SLOT_MISSING; - } - fd_block_info_t block_info; - ulong rsz; int err; - err = fd_io_read( fd, &block_info, sizeof(fd_block_info_t), sizeof(fd_block_info_t), &rsz ); - check_read_write_err( err ); - err = fd_io_read( fd, txn_data_out, txn_out->sz, txn_out->sz, &rsz ); - check_read_write_err( err ); - err = (int)lseek( fd, (long)off + (long)txn_out->offset, SEEK_SET ); - check_read_write_err( err ); - err = fd_io_read( fd, txn_data_out, txn_out->sz, txn_out->sz, &rsz ); - check_read_write_err( err); - return FD_BLOCKSTORE_SUCCESS; - } - - for(;;) { - fd_block_map_query_t quer[1] = { 0 }; - fd_block_map_query_try( blockstore->block_map, &txn_out->slot, NULL, quer, 0 ); - fd_block_info_t const * query = fd_block_map_query_ele_const( quer ); - - if( FD_UNLIKELY( !query ) ) return FD_BLOCKSTORE_ERR_TXN_MISSING; - ulong blk_gaddr = query->block_gaddr; - if( FD_UNLIKELY( !blk_gaddr ) ) return FD_BLOCKSTORE_ERR_TXN_MISSING; - - if( fd_block_map_query_test( quer ) ) continue; - - fd_block_t * blk = fd_wksp_laddr_fast( wksp, blk_gaddr ); - if( blk_ts ) *blk_ts = query->ts; - if( blk_flags ) *blk_flags = query->flags; - ulong ptr = blk->data_gaddr; - ulong sz = blk->data_sz; - if( txn_out->offset + txn_out->sz > sz || txn_out->sz > FD_TXN_MTU ) continue; - - if( FD_UNLIKELY( fd_block_map_query_test( quer ) ) ) continue; - - if( txn_data_out == NULL ) return FD_BLOCKSTORE_SUCCESS; - uchar const * data = fd_wksp_laddr_fast( wksp, ptr ); - fd_memcpy( txn_data_out, data + txn_out->offset, txn_out->sz ); - - if( FD_UNLIKELY( fd_block_map_query_test( quer ) ) ) continue; - - return FD_BLOCKSTORE_SUCCESS; - } -#endif -} - void fd_blockstore_block_height_update( fd_blockstore_t * blockstore, ulong slot, ulong height ) { fd_block_map_query_t query[1] = { 0 }; @@ -1115,14 +990,6 @@ fd_blockstore_log_mem_usage( fd_blockstore_t * blockstore ) { slot_map_max, (100U*slot_map_cnt)/slot_map_max )); */ - fd_txn_map_t * txn_map = fd_blockstore_txn_map( blockstore ); - ulong txn_map_cnt = fd_txn_map_key_cnt( txn_map ); - ulong txn_map_max = fd_txn_map_key_max( txn_map ); - FD_LOG_NOTICE(( "txn map footprint: %s (%lu entries used out of %lu, %lu%%)", - fd_smart_size( fd_txn_map_footprint( txn_map_max ), tmp1, sizeof(tmp1) ), - txn_map_cnt, - txn_map_max, - (100U*txn_map_cnt)/txn_map_max )); ulong block_cnt = 0; ulong * q = fd_blockstore_slot_deque( blockstore ); diff --git a/src/flamenco/runtime/fd_blockstore.h b/src/flamenco/runtime/fd_blockstore.h index f57d9f6a54..0f4d0f1568 100644 --- a/src/flamenco/runtime/fd_blockstore.h +++ b/src/flamenco/runtime/fd_blockstore.h @@ -34,7 +34,6 @@ /* DO NOT MODIFY. */ // #define FD_BUF_SHRED_MAP_MAX (1UL << 24UL) /* 16 million shreds can be buffered */ -// #define FD_TXN_MAP_LG_MAX (24) /* 16 million txns can be stored in the txn map */ /* TODO this can be removed if we explicitly manage a memory pool for the fd_block_map_t entries */ @@ -83,10 +82,8 @@ #define FD_BLOCKSTORE_ERR_KEY (-6) #define FD_BLOCKSTORE_ERR_SHRED_FULL -1 /* no space left for shreds */ #define FD_BLOCKSTORE_ERR_SLOT_FULL -2 /* no space left for slots */ -#define FD_BLOCKSTORE_ERR_TXN_FULL -3 /* no space left for txns */ #define FD_BLOCKSTORE_ERR_SHRED_MISSING -4 #define FD_BLOCKSTORE_ERR_SLOT_MISSING -5 -#define FD_BLOCKSTORE_ERR_TXN_MISSING -6 #define FD_BLOCKSTORE_ERR_SHRED_INVALID -7 /* shred was invalid */ #define FD_BLOCKSTORE_ERR_DESHRED_INVALID -8 /* deshredded block was invalid */ #define FD_BLOCKSTORE_ERR_NO_MEM -9 /* no mem */ @@ -208,16 +205,6 @@ struct fd_block_micro { }; typedef struct fd_block_micro fd_block_micro_t; -/* fd_block_txn_t is a transaction that has been parsed and is part of a - block. The transaction begins at `off` relative to the start of the - block's data region. */ -struct fd_block_txn { - ulong txn_off; /* offset into block data of transaction */ - ulong id_off; /* offset into block data of transaction identifiers */ - ulong sz; -}; -typedef struct fd_block_txn fd_block_txn_t; - /* If the 0th bit is set, this indicates the block is preparing, which means it might be partially executed e.g. a subset of the microblocks have been executed. It is not safe to remove, relocate, or modify @@ -374,33 +361,6 @@ typedef struct fd_block_idx fd_block_idx_t; #define MAP_KEY_INVAL(k) (k == ULONG_MAX) #include "../../util/tmpl/fd_map_dynamic.c" -struct fd_txn_key { - ulong v[FD_ED25519_SIG_SZ / sizeof( ulong )]; -}; -typedef struct fd_txn_key fd_txn_key_t; - -struct fd_txn_map { - fd_txn_key_t sig; - ulong next; - ulong slot; - ulong offset; - ulong sz; - ulong meta_gaddr; /* ptr to the transaction metadata */ - ulong meta_sz; /* metadata size */ -}; -typedef struct fd_txn_map fd_txn_map_t; - -FD_FN_PURE int fd_txn_key_equal(fd_txn_key_t const * k0, fd_txn_key_t const * k1); -FD_FN_PURE ulong fd_txn_key_hash(fd_txn_key_t const * k, ulong seed); - -#define MAP_NAME fd_txn_map -#define MAP_T fd_txn_map_t -#define MAP_KEY sig -#define MAP_KEY_T fd_txn_key_t -#define MAP_KEY_EQ(k0,k1) fd_txn_key_equal(k0,k1) -#define MAP_KEY_HASH(k,seed) fd_txn_key_hash(k, seed) -#include "../../util/tmpl/fd_map_giant.c" - /* fd_blockstore_archiver outlines the format of metadata at the start of an archive file - needed so that archive files can be read back on initialization. */ @@ -454,15 +414,12 @@ struct __attribute__((aligned(FD_BLOCKSTORE_ALIGN))) fd_blockstore_shmem { ulong shred_max; /* maximum # of shreds that can be held in memory */ ulong block_max; /* maximum # of blocks that can be held in memory */ ulong idx_max; /* maximum # of blocks that can be indexed from the archival file */ - ulong txn_max; /* maximum # of transactions that can be indexed from blocks */ ulong alloc_max; /* maximum bytes that can be allocated */ //ulong block_map_gaddr; /* map of slot->(slot_meta, block) */ ulong block_idx_gaddr; /* map of slot->byte offset in archival file */ ulong slot_deque_gaddr; /* deque of slot numbers */ - /* IMPORTANT: the txn_map is not safe to write to from multiple threads. */ - ulong txn_map_gaddr; ulong alloc_gaddr; }; typedef struct fd_blockstore_shmem fd_blockstore_shmem_t; @@ -498,7 +455,7 @@ fd_blockstore_align( void ) { including data structures. */ FD_FN_CONST static inline ulong -fd_blockstore_footprint( ulong shred_max, ulong block_max, ulong idx_max, ulong txn_max ) { +fd_blockstore_footprint( ulong shred_max, ulong block_max, ulong idx_max ) { /* TODO -- when removing, make change in fd_blockstore_new as well */ block_max = fd_ulong_pow2_up( block_max ); ulong lock_cnt = fd_ulong_min( block_max, BLOCK_INFO_LOCK_CNT ); @@ -514,7 +471,6 @@ fd_blockstore_footprint( ulong shred_max, ulong block_max, ulong idx_max, ulong FD_LAYOUT_APPEND( FD_LAYOUT_APPEND( FD_LAYOUT_APPEND( - FD_LAYOUT_APPEND( FD_LAYOUT_INIT, alignof(fd_blockstore_shmem_t), sizeof(fd_blockstore_shmem_t) ), alignof(fd_buf_shred_t), sizeof(fd_buf_shred_t) * shred_max ), @@ -524,7 +480,6 @@ fd_blockstore_footprint( ulong shred_max, ulong block_max, ulong idx_max, ulong fd_block_map_align(), fd_block_map_footprint( block_max, lock_cnt, BLOCK_INFO_PROBE_CNT ) ), fd_block_idx_align(), fd_block_idx_footprint( lg_idx_max ) ), fd_slot_deque_align(), fd_slot_deque_footprint( block_max ) ), - fd_txn_map_align(), fd_txn_map_footprint( txn_max ) ), fd_alloc_align(), fd_alloc_footprint() ), fd_blockstore_align() ); } @@ -542,8 +497,7 @@ fd_blockstore_new( void * shmem, ulong seed, ulong shred_max, ulong block_max, - ulong idx_max, - ulong txn_max ); + ulong idx_max ); /* fd_blockstore_join joins a blockstore. ljoin points to a fd_blockstore_t compatible memory region in the caller's address @@ -633,15 +587,6 @@ fd_blockstore_slot_deque( fd_blockstore_t * blockstore ) { return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore), blockstore->shmem->slot_deque_gaddr ); } -/* fd_txn_map returns a pointer in the caller's address space to the blockstore's - block map. Assumes blockstore is local join. Lifetime of the returned pointer is that of the - local join. */ - -FD_FN_PURE static inline fd_txn_map_t * -fd_blockstore_txn_map( fd_blockstore_t * blockstore ) { - return fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore), blockstore->shmem->txn_map_gaddr ); -} - /* fd_blockstore_alloc returns a pointer in the caller's address space to the blockstore's allocator. */ @@ -787,27 +732,6 @@ fd_blockstore_block_map_query_volatile( fd_blockstore_t * blockstore, ulong slot, fd_block_info_t * block_info_out ) ; -/* fd_blockstore_txn_query queries the transaction data for the given - signature. - - IMPORTANT! Caller MUST hold the read lock when calling this - function. */ - -fd_txn_map_t * -fd_blockstore_txn_query( fd_blockstore_t * blockstore, uchar const sig[static FD_ED25519_SIG_SZ] ); - -/* Query the transaction data for the given signature in a thread - safe manner. The transaction data is copied out. txn_data_out can - be NULL if you are only interested in the transaction metadata. */ -int -fd_blockstore_txn_query_volatile( fd_blockstore_t * blockstore, - int fd, - uchar const sig[static FD_ED25519_SIG_SZ], - fd_txn_map_t * txn_out, - long * blk_ts, - uchar * blk_flags, - uchar txn_data_out[FD_TXN_MTU] ); - /* fd_blockstore_block_info_test tests if a block meta entry exists for the given slot. Returns 1 if the entry exists and 0 otherwise. diff --git a/src/flamenco/runtime/fd_blockstore_tool.c b/src/flamenco/runtime/fd_blockstore_tool.c index d737ff657a..88aa0bec3d 100644 --- a/src/flamenco/runtime/fd_blockstore_tool.c +++ b/src/flamenco/runtime/fd_blockstore_tool.c @@ -43,13 +43,11 @@ usage( void ) { ulong shred_max = 1 << 17; \ ulong idx_max = 1 << 12; \ ulong block_max = 1 << 17; \ - ulong txn_max = 1 << 17; \ void * mem = fd_wksp_alloc_laddr( wksp, \ fd_blockstore_align(), \ fd_blockstore_footprint( shred_max, \ block_max, \ - idx_max, \ - txn_max ), \ + idx_max ), \ 1UL ); \ FD_TEST( mem ); \ void * shblockstore = fd_blockstore_new( mem, \ @@ -57,8 +55,7 @@ usage( void ) { 0UL, \ shred_max, \ block_max, \ - idx_max, \ - txn_max ); \ + idx_max ); \ \ FD_TEST( shblockstore ); \ fd_blockstore_t blockstore_ljoin; \ @@ -191,7 +188,7 @@ initialize_rocksdb( fd_wksp_t * wksp, if( err < 0 ) continue; - err = fd_rocksdb_import_block_blockstore( &rocks_db, &slot_meta, blockstore, 1, trash_hash_buf, valloc ); + err = fd_rocksdb_import_block_blockstore( &rocks_db, &slot_meta, blockstore, trash_hash_buf, valloc ); if( FD_UNLIKELY( err != 0) ) { FD_LOG_ERR(( "Failed to import block %lu", slot )); } diff --git a/src/flamenco/runtime/fd_rocksdb.c b/src/flamenco/runtime/fd_rocksdb.c index fd0f547ac6..55720b0423 100644 --- a/src/flamenco/runtime/fd_rocksdb.c +++ b/src/flamenco/runtime/fd_rocksdb.c @@ -433,66 +433,6 @@ fd_rocksdb_copy_over_slot_indexed_range( fd_rocksdb_t * src, return 0; } -int -fd_rocksdb_copy_over_txn_status_range( fd_rocksdb_t * src, - fd_rocksdb_t * dst, - fd_blockstore_t * blockstore, - ulong start_slot, - ulong end_slot ) { - /* Look up the blocks data and iterate through its transactions */ - fd_wksp_t * wksp = fd_blockstore_wksp( blockstore ); - - for ( ulong slot = start_slot; slot <= end_slot; ++slot ) { - FD_LOG_NOTICE(( "fd_rocksdb_copy_over_txn_status_range: %lu", slot )); - fd_block_info_t * block_entry = fd_blockstore_block_map_query( blockstore, slot ); - if( FD_LIKELY( block_entry && fd_blockstore_shreds_complete( blockstore, slot) ) ) { - fd_block_t * blk = fd_wksp_laddr_fast( wksp, block_entry->block_gaddr ); - uchar * data = fd_wksp_laddr_fast( wksp, blk->data_gaddr ); - fd_block_txn_t * txns = fd_wksp_laddr_fast( wksp, blk->txns_gaddr ); - /* TODO: change txn indexing after blockstore refactor */ - ulong last_txn_off = ULONG_MAX; - for ( ulong j = 0; j < blk->txns_cnt; ++j ) { - fd_txn_key_t sig; - fd_memcpy( &sig, data + txns[j].id_off, sizeof(sig) ); - if( txns[j].txn_off != last_txn_off ) { - last_txn_off = txns[j].txn_off; - fd_rocksdb_copy_over_txn_status( src, dst, slot, &sig ); - } - } - } - } - return 0; -} - -void -fd_rocksdb_copy_over_txn_status( fd_rocksdb_t * src, - fd_rocksdb_t * dst, - ulong slot, - void const * sig ) { - ulong slot_be = fd_ulong_bswap( slot ); - - /* Construct RocksDB query key */ - /* TODO: Replace with constants */ - char key[ 72 ]; - memcpy( key, sig, 64UL ); - memcpy( key+64UL, &slot_be, 8UL ); - - /* Query record */ - ulong sz; - char * err = NULL; - char * res = rocksdb_get_cf( - src->db, src->ro, src->cf_handles[ FD_ROCKSDB_CFIDX_TRANSACTION_STATUS ], - key, 72UL, &sz, &err ); - - if( FD_UNLIKELY( err ) ) { - FD_LOG_WARNING(("err=%s", err)); - free( err ); - return; - } - - fd_rocksdb_insert_entry( dst, FD_ROCKSDB_CFIDX_TRANSACTION_STATUS, key, 72UL, res, sz ); -} - int fd_rocksdb_insert_entry( fd_rocksdb_t * db, ulong cf_idx, @@ -517,9 +457,6 @@ fd_blockstore_scan_block( fd_blockstore_t * blockstore, ulong slot, fd_block_t * fd_block_micro_t * micros = fd_alloc_malloc( fd_blockstore_alloc( blockstore ), alignof( fd_block_micro_t ), sizeof( *micros ) * FD_MICROBLOCK_MAX_PER_SLOT ); - fd_block_txn_t * txns = fd_alloc_malloc( fd_blockstore_alloc( blockstore ), - alignof( fd_block_txn_t ), - sizeof( *txns ) * FD_TXN_MAX_PER_SLOT ); /* * Agave decodes precisely one array of microblocks from each batch. @@ -537,7 +474,6 @@ fd_blockstore_scan_block( fd_blockstore_t * blockstore, ulong slot, fd_block_t * ulong const batch_cnt = block->batch_cnt; ulong micros_cnt = 0UL; - ulong txns_cnt = 0UL; ulong blockoff = 0UL; for( ulong batch_i = 0UL; batch_i < batch_cnt; batch_i++ ) { ulong const batch_end_off = batch_laddr[ batch_i ].end_off; @@ -573,7 +509,6 @@ fd_blockstore_scan_block( fd_blockstore_t * blockstore, ulong slot, fd_block_t * slot, txn_sz )); } - fd_txn_t const * txn = (fd_txn_t const *)txn_out; if( pay_sz == 0UL ) FD_LOG_ERR(( "failed to parse transaction %lu in microblock %lu in slot %lu", @@ -581,32 +516,6 @@ fd_blockstore_scan_block( fd_blockstore_t * blockstore, ulong slot, fd_block_t * mblk, slot )); - fd_txn_key_t const * sigs = - (fd_txn_key_t const *)( (ulong)raw + (ulong)txn->signature_off ); - fd_txn_map_t * txn_map = fd_blockstore_txn_map( blockstore ); - for( ulong j = 0; j < txn->signature_cnt; j++ ) { - if( FD_UNLIKELY( fd_txn_map_key_cnt( txn_map ) == - fd_txn_map_key_max( txn_map ) ) ) { - break; - } - fd_txn_key_t sig; - fd_memcpy( &sig, sigs + j, sizeof( sig ) ); - if( fd_txn_map_query( txn_map, &sig, NULL ) != NULL ) continue; - fd_txn_map_t * elem = fd_txn_map_insert( txn_map, &sig ); - if( elem == NULL ) { break; } - elem->slot = slot; - elem->offset = blockoff; - elem->sz = pay_sz; - elem->meta_gaddr = 0; - elem->meta_sz = 0; - if( txns_cnt < FD_TXN_MAX_PER_SLOT ) { - fd_block_txn_t * ref = &txns[txns_cnt++]; - ref->txn_off = blockoff; - ref->id_off = (ulong)( sigs + j ) - (ulong)data; - ref->sz = pay_sz; - } - } - blockoff += pay_sz; } } @@ -632,16 +541,7 @@ fd_blockstore_scan_block( fd_blockstore_t * blockstore, ulong slot, fd_block_t * block->micros_gaddr = fd_wksp_gaddr_fast( fd_blockstore_wksp( blockstore ), micros_laddr ); block->micros_cnt = micros_cnt; - fd_block_txn_t * txns_laddr = - fd_alloc_malloc( fd_blockstore_alloc( blockstore ), - alignof( fd_block_txn_t ), - sizeof( fd_block_txn_t ) * txns_cnt ); - fd_memcpy( txns_laddr, txns, sizeof( fd_block_txn_t ) * txns_cnt ); - block->txns_gaddr = fd_wksp_gaddr_fast( fd_blockstore_wksp( blockstore ), txns_laddr ); - block->txns_cnt = txns_cnt; - fd_alloc_free( fd_blockstore_alloc( blockstore ), micros ); - fd_alloc_free( fd_blockstore_alloc( blockstore ), txns ); } static int @@ -793,22 +693,12 @@ fd_blockstore_block_allocs_remove( fd_blockstore_t * blockstore, fd_wksp_t * wksp = fd_blockstore_wksp( blockstore ); fd_alloc_t * alloc = fd_blockstore_alloc( blockstore ); - fd_txn_map_t * txn_map = fd_blockstore_txn_map( blockstore ); fd_block_t * block = fd_wksp_laddr_fast( wksp, block_gaddr ); /* DO THIS FIRST FOR THREAD SAFETY */ FD_COMPILER_MFENCE(); //block_info->block_gaddr = 0; - uchar * data = fd_wksp_laddr_fast( wksp, block->data_gaddr ); - fd_block_txn_t * txns = fd_wksp_laddr_fast( wksp, block->txns_gaddr ); - for( ulong j = 0; j < block->txns_cnt; ++j ) { - fd_txn_key_t sig; - fd_memcpy( &sig, data + txns[j].id_off, sizeof( sig ) ); - fd_txn_map_remove( txn_map, &sig ); - } - if( block->micros_gaddr ) fd_alloc_free( alloc, fd_wksp_laddr_fast( wksp, block->micros_gaddr ) ); - if( block->txns_gaddr ) fd_alloc_free( alloc, txns ); ulong mgaddr = block->txns_meta_gaddr; while( mgaddr ) { ulong * laddr = fd_wksp_laddr_fast( wksp, mgaddr ); @@ -823,7 +713,6 @@ int fd_rocksdb_import_block_blockstore( fd_rocksdb_t * db, fd_slot_meta_t * m, fd_blockstore_t * blockstore, - int txnstatus, const uchar * hash_override, fd_valloc_t valloc ) { ulong slot = m->slot; @@ -892,7 +781,6 @@ fd_rocksdb_import_block_blockstore( fd_rocksdb_t * db, rocksdb_iter_destroy(iter); - fd_wksp_t * wksp = fd_blockstore_wksp( blockstore ); fd_block_info_t * block_info = fd_blockstore_block_map_query( blockstore, slot ); if( FD_LIKELY( block_info && fd_blockstore_shreds_complete( blockstore, slot ) ) ) { deshred( blockstore, slot ); @@ -972,72 +860,6 @@ fd_rocksdb_import_block_blockstore( fd_rocksdb_t * db, } } - if( txnstatus && FD_LIKELY( block_info && fd_blockstore_shreds_complete( blockstore, slot ) ) ) { - fd_block_t * blk = fd_wksp_laddr_fast( wksp, block_info->block_gaddr ); - uchar * data = fd_wksp_laddr_fast( wksp, blk->data_gaddr ); - fd_block_txn_t * txns = fd_wksp_laddr_fast( wksp, blk->txns_gaddr ); - - /* TODO: change txn indexing after blockstore refactor */ - /* Compute the total size of the logs */ - ulong tot_meta_sz = 2*sizeof(ulong); - for ( ulong j = 0; j < blk->txns_cnt; ++j ) { - if( j == 0 || txns[j].txn_off != txns[j-1].txn_off ) { - fd_txn_key_t sig; - fd_memcpy( &sig, data + txns[j].id_off, sizeof( sig ) ); - ulong sz; - void * raw = fd_rocksdb_get_txn_status_raw( db, slot, &sig, &sz ); - if( raw != NULL ) { - free(raw); - tot_meta_sz += sz; - } - } - } - fd_alloc_t * alloc = fd_blockstore_alloc( blockstore ); - uchar * cur_laddr = fd_alloc_malloc( alloc, 1, tot_meta_sz ); - if( cur_laddr == NULL ) { - return 0; - } - ((ulong*)cur_laddr)[0] = blk->txns_meta_gaddr; /* Link to previous allocation */ - ((ulong*)cur_laddr)[1] = blk->txns_meta_sz; - blk->txns_meta_gaddr = fd_wksp_gaddr_fast( wksp, cur_laddr ); - blk->txns_meta_sz = tot_meta_sz; - cur_laddr += 2*sizeof(ulong); - - /* Copy over the logs */ - fd_txn_map_t * txn_map = fd_blockstore_txn_map( blockstore ); - ulong meta_gaddr = 0; - ulong meta_sz = 0; - fd_txn_key_t sig = { 0 }; - for ( ulong j = 0; j < blk->txns_cnt; ++j ) { - if( j == 0 || txns[j].txn_off != txns[j-1].txn_off ) { - fd_memcpy( &sig, data + txns[j].id_off, sizeof( sig ) ); - ulong sz; - void * raw = fd_rocksdb_get_txn_status_raw( db, slot, &sig, &sz ); - if( raw == NULL ) { - meta_gaddr = 0; - meta_sz = 0; - } else { - fd_memcpy(cur_laddr, raw, sz); - free(raw); - meta_gaddr = fd_wksp_gaddr_fast( wksp, cur_laddr ); - meta_sz = sz; - cur_laddr += sz; - } - } - fd_txn_map_t * txn_map_entry = fd_txn_map_query( txn_map, &sig, NULL ); - if( FD_UNLIKELY( !txn_map_entry ) ) { - char sig_str[ FD_BASE58_ENCODED_64_SZ ]; - fd_base58_encode_64( fd_type_pun_const( sig.v ), NULL, sig_str ); - FD_LOG_WARNING(( "missing transaction %s", sig_str )); - continue; - } - txn_map_entry->meta_gaddr = meta_gaddr; - txn_map_entry->meta_sz = meta_sz; - } - - FD_TEST( blk->txns_meta_gaddr + blk->txns_meta_sz == fd_wksp_gaddr_fast( wksp, cur_laddr ) ); - } - blockstore->shmem->lps = slot; blockstore->shmem->hcs = slot; blockstore->shmem->wmk = slot; diff --git a/src/flamenco/runtime/fd_rocksdb.h b/src/flamenco/runtime/fd_rocksdb.h index 35b0217b88..a37b94ff06 100644 --- a/src/flamenco/runtime/fd_rocksdb.h +++ b/src/flamenco/runtime/fd_rocksdb.h @@ -245,18 +245,6 @@ fd_rocksdb_copy_over_slot_indexed_range( fd_rocksdb_t * src, ulong start_slot, ulong end_slot ); -/* fd_rocksdb_copy_over_txn_status_range copies over all transaction statuses - within a block range assuming the blockstore contains relevant pointers to - the transactions within the range. The blockstore object must be populated - with the relevant block range. */ - -int -fd_rocksdb_copy_over_txn_status_range( fd_rocksdb_t * src, - fd_rocksdb_t * dst, - fd_blockstore_t * blockstore, - ulong start_slot, - ulong end_slot ); - /* fd_rocksdb_copy_over_txn_status constructs a key to query a transaction status and copies over the entry into another rocksdb. The index is used to specify which transaction. */ @@ -283,7 +271,6 @@ int fd_rocksdb_import_block_blockstore( fd_rocksdb_t * db, fd_slot_meta_t * m, fd_blockstore_t * blockstore, - int txnstatus, const uchar * hash_override, fd_valloc_t valloc ); diff --git a/src/flamenco/runtime/fd_runtime.c b/src/flamenco/runtime/fd_runtime.c index 2a38bc01cf..b1227ba620 100644 --- a/src/flamenco/runtime/fd_runtime.c +++ b/src/flamenco/runtime/fd_runtime.c @@ -694,279 +694,6 @@ fd_runtime_collect_rent_from_account( ulong slot, #undef FD_RENT_EXEMPT -void -fd_runtime_write_transaction_status( fd_capture_ctx_t * capture_ctx, - fd_exec_slot_ctx_t * slot_ctx, - fd_exec_txn_ctx_t * txn_ctx, - int exec_txn_err) { - /* TODO: The blockstore txn_map is unsafe to write to from multiple threads, - so we use this lock to protect it. This function is the only place - we write to the txn_map in parallel, and when the ledger replay code - is ripped out we can also remove this lock. */ - fd_capture_ctx_txn_status_start_write(); - - /* Look up solana-side transaction status details */ - fd_blockstore_t * blockstore = slot_ctx->blockstore; - uchar * sig = (uchar *)txn_ctx->_txn_raw->raw + txn_ctx->txn_descriptor->signature_off; - fd_txn_map_t * txn_map_entry = fd_blockstore_txn_query( blockstore, sig ); - if( FD_LIKELY( txn_map_entry != NULL ) ) { - void * meta = fd_wksp_laddr_fast( fd_blockstore_wksp( blockstore ), txn_map_entry->meta_gaddr ); - - fd_solblock_TransactionStatusMeta txn_status = {0}; - /* Need to handle case for ledgers where transaction status is not available. - This case will be handled in fd_solcap_diff. */ - ulong fd_cus_consumed = txn_ctx->compute_unit_limit - txn_ctx->compute_meter; - ulong solana_cus_consumed = ULONG_MAX; - ulong solana_txn_err = ULONG_MAX; - if( FD_LIKELY( meta != NULL ) ) { - pb_istream_t stream = pb_istream_from_buffer( meta, txn_map_entry->meta_sz ); - if ( pb_decode( &stream, fd_solblock_TransactionStatusMeta_fields, &txn_status ) == false ) { - FD_LOG_WARNING(("no txn_status decoding found sig=%s (%s)", FD_BASE58_ENC_64_ALLOCA( sig ), PB_GET_ERROR(&stream))); - } - if ( txn_status.has_compute_units_consumed ) { - solana_cus_consumed = txn_status.compute_units_consumed; - } - if ( txn_status.has_err ) { - solana_txn_err = txn_status.err.err->bytes[0]; - } - - fd_solcap_Transaction txn = { - .slot = slot_ctx->slot_bank.slot, - .fd_txn_err = exec_txn_err, - .fd_custom_err = txn_ctx->custom_err, - .solana_txn_err = solana_txn_err, - .fd_cus_used = fd_cus_consumed, - .solana_cus_used = solana_cus_consumed, - .instr_err_idx = txn_ctx->instr_err_idx == INT_MAX ? -1 : txn_ctx->instr_err_idx, - }; - memcpy( txn.txn_sig, sig, sizeof(fd_signature_t) ); - - fd_exec_instr_ctx_t const * failed_instr = txn_ctx->failed_instr; - if( failed_instr ) { - FD_TEST( failed_instr->depth < 4 ); - txn.instr_err = failed_instr->instr_err; - txn.failed_instr_path_count = failed_instr->depth + 1; - for( long j = failed_instr->depth; j>=0L; j-- ) { - txn.failed_instr_path[j] = failed_instr->index; - failed_instr = failed_instr->parent; - } - } - - fd_solcap_write_transaction2( capture_ctx->capture, &txn ); - } - } - - fd_capture_ctx_txn_status_end_write(); -} - -static bool -encode_return_data( pb_ostream_t *stream, const pb_field_t *field, void * const *arg ) { - fd_exec_txn_ctx_t * txn_ctx = (fd_exec_txn_ctx_t *)(*arg); - pb_encode_tag_for_field(stream, field); - pb_encode_string(stream, txn_ctx->return_data.data, txn_ctx->return_data.len ); - return 1; -} - -static ulong -fd_txn_copy_meta( fd_exec_txn_ctx_t * txn_ctx, uchar * dest, ulong dest_sz ) { - fd_solblock_TransactionStatusMeta txn_status = {0}; - - txn_status.has_fee = 1; - txn_status.fee = txn_ctx->execution_fee + txn_ctx->priority_fee; - - txn_status.has_compute_units_consumed = 1; - txn_status.compute_units_consumed = txn_ctx->compute_unit_limit - txn_ctx->compute_meter; - - ulong readonly_cnt = 0; - ulong writable_cnt = 0; - if( txn_ctx->txn_descriptor->transaction_version == FD_TXN_V0 ) { - fd_txn_acct_addr_lut_t const * addr_luts = fd_txn_get_address_tables_const( txn_ctx->txn_descriptor ); - for( ulong i = 0; i < txn_ctx->txn_descriptor->addr_table_lookup_cnt; i++ ) { - fd_txn_acct_addr_lut_t const * addr_lut = &addr_luts[i]; - readonly_cnt += addr_lut->readonly_cnt; - writable_cnt += addr_lut->writable_cnt; - } - } - - typedef PB_BYTES_ARRAY_T(32) my_ba_t; - typedef union { my_ba_t my; pb_bytes_array_t normal; } union_ba_t; - union_ba_t writable_ba[writable_cnt]; - pb_bytes_array_t * writable_baptr[writable_cnt]; - txn_status.loaded_writable_addresses_count = (uint)writable_cnt; - txn_status.loaded_writable_addresses = writable_baptr; - ulong idx2 = txn_ctx->txn_descriptor->acct_addr_cnt; - for (ulong idx = 0; idx < writable_cnt; idx++) { - pb_bytes_array_t * ba = writable_baptr[ idx ] = &writable_ba[ idx ].normal; - ba->size = 32; - fd_memcpy(ba->bytes, &txn_ctx->account_keys[idx2++], 32); - } - - union_ba_t readonly_ba[readonly_cnt]; - pb_bytes_array_t * readonly_baptr[readonly_cnt]; - txn_status.loaded_readonly_addresses_count = (uint)readonly_cnt; - txn_status.loaded_readonly_addresses = readonly_baptr; - for (ulong idx = 0; idx < readonly_cnt; idx++) { - pb_bytes_array_t * ba = readonly_baptr[ idx ] = &readonly_ba[ idx ].normal; - ba->size = 32; - fd_memcpy(ba->bytes, &txn_ctx->account_keys[idx2++], 32); - } - ulong acct_cnt = txn_ctx->accounts_cnt; - FD_TEST(acct_cnt == idx2); - - txn_status.pre_balances_count = txn_status.post_balances_count = (pb_size_t)acct_cnt; - uint64_t pre_balances[acct_cnt]; - txn_status.pre_balances = pre_balances; - uint64_t post_balances[acct_cnt]; - txn_status.post_balances = post_balances; - - for (ulong idx = 0; idx < acct_cnt; idx++) { - fd_txn_account_t const * acct = &txn_ctx->accounts[idx]; - ulong pre = ( acct->starting_lamports == ULONG_MAX ? 0UL : acct->starting_lamports ); - - pre_balances[idx] = pre; - post_balances[idx] = ( acct->vt->is_mutable( acct ) || acct->vt->is_readonly( acct ) ? - acct->vt->get_lamports( acct ) : pre ); - } - - if( txn_ctx->return_data.len ) { - txn_status.has_return_data = 1; - txn_status.return_data.has_program_id = 1; - fd_memcpy( txn_status.return_data.program_id, txn_ctx->return_data.program_id.uc, 32U ); - pb_callback_t data = { .funcs.encode = encode_return_data, .arg = txn_ctx }; - txn_status.return_data.data = data; - } - - union { - pb_bytes_array_t arr; - uchar space[64]; - } errarr; - pb_byte_t * errptr = errarr.arr.bytes; - if( txn_ctx->custom_err != UINT_MAX ) { - *(uint*)errptr = 8 /* Instruction error */; - errptr += sizeof(uint); - *errptr = (uchar)txn_ctx->instr_err_idx; - errptr += 1; - *(int*)errptr = FD_EXECUTOR_INSTR_ERR_CUSTOM_ERR; - errptr += sizeof(int); - *(uint*)errptr = txn_ctx->custom_err; - errptr += sizeof(uint); - errarr.arr.size = (uint)(errptr - errarr.arr.bytes); - txn_status.has_err = 1; - txn_status.err.err = &errarr.arr; - } else if( txn_ctx->exec_err ) { - switch( txn_ctx->exec_err_kind ) { - case FD_EXECUTOR_ERR_KIND_SYSCALL: - break; - case FD_EXECUTOR_ERR_KIND_INSTR: - *(uint*)errptr = 8 /* Instruction error */; - errptr += sizeof(uint); - *errptr = (uchar)txn_ctx->instr_err_idx; - errptr += 1; - *(int*)errptr = txn_ctx->exec_err; - errptr += sizeof(int); - errarr.arr.size = (uint)(errptr - errarr.arr.bytes); - txn_status.has_err = 1; - txn_status.err.err = &errarr.arr; - break; - case FD_EXECUTOR_ERR_KIND_EBPF: - break; - } - } - - if( dest == NULL ) { - size_t sz = 0; - bool r = pb_get_encoded_size( &sz, fd_solblock_TransactionStatusMeta_fields, &txn_status ); - if( !r ) { - FD_LOG_WARNING(( "pb_get_encoded_size failed" )); - return 0; - } - return sz + txn_ctx->log_collector.buf_sz; - } - - pb_ostream_t stream = pb_ostream_from_buffer( dest, dest_sz ); - bool r = pb_encode( &stream, fd_solblock_TransactionStatusMeta_fields, &txn_status ); - if( !r ) { - FD_LOG_WARNING(( "pb_encode failed" )); - return 0; - } - pb_write( &stream, txn_ctx->log_collector.buf, txn_ctx->log_collector.buf_sz ); - return stream.bytes_written; -} - -/* fd_runtime_finalize_txns_update_blockstore_meta() updates transaction metadata - after execution. - - Execution recording is controlled by slot_ctx->enable_exec_recording, and this - function does nothing if execution recording is off. The following comments - only apply when execution recording is on. - - Transaction metadata includes execution result (success/error), balance changes, - transaction logs, ... All this info is not part of consensus but can be retrieved, - for instance, via RPC getTransaction. Firedancer stores txn meta in the blockstore, - in the same binary format as Agave, protobuf TransactionStatusMeta. */ -FD_FN_UNUSED static void -fd_runtime_finalize_txns_update_blockstore_meta( fd_exec_slot_ctx_t * slot_ctx, - fd_execute_txn_task_info_t * task_info, - ulong txn_cnt ) { - /* Nothing to do if execution recording is off */ - if( !slot_ctx->enable_exec_recording ) { - return; - } - - fd_blockstore_t * blockstore = slot_ctx->blockstore; - fd_wksp_t * blockstore_wksp = fd_blockstore_wksp( blockstore ); - fd_alloc_t * blockstore_alloc = fd_blockstore_alloc( blockstore ); - fd_txn_map_t * txn_map = fd_blockstore_txn_map( blockstore ); - - /* Get the total size of all logs */ - ulong tot_meta_sz = 2*sizeof(ulong); - for( ulong txn_idx = 0; txn_idx < txn_cnt; txn_idx++ ) { - /* Prebalance compensation */ - fd_exec_txn_ctx_t * txn_ctx = task_info[txn_idx].txn_ctx; - txn_ctx->accounts[0].starting_lamports += (txn_ctx->execution_fee + txn_ctx->priority_fee); - /* Get the size without the copy */ - tot_meta_sz += fd_txn_copy_meta( txn_ctx, NULL, 0 ); - } - uchar * cur_laddr = fd_alloc_malloc( blockstore_alloc, 1, tot_meta_sz ); - if( cur_laddr == NULL ) { - return; - } - uchar * const end_laddr = cur_laddr + tot_meta_sz; - - /* Link to previous allocation */ - ((ulong*)cur_laddr)[0] = slot_ctx->txns_meta_gaddr; - ((ulong*)cur_laddr)[1] = slot_ctx->txns_meta_sz; - slot_ctx->txns_meta_gaddr = fd_wksp_gaddr_fast( blockstore_wksp, cur_laddr ); - slot_ctx->txns_meta_sz = tot_meta_sz; - cur_laddr += 2*sizeof(ulong); - - for( ulong txn_idx = 0; txn_idx < txn_cnt; txn_idx++ ) { - fd_exec_txn_ctx_t * txn_ctx = task_info[txn_idx].txn_ctx; - ulong meta_sz = fd_txn_copy_meta( txn_ctx, cur_laddr, (size_t)(end_laddr - cur_laddr) ); - if( meta_sz ) { - ulong meta_gaddr = fd_wksp_gaddr_fast( blockstore_wksp, cur_laddr ); - - /* Update all the signatures */ - char const * sig_p = (char const *)txn_ctx->_txn_raw->raw + txn_ctx->txn_descriptor->signature_off; - fd_txn_key_t sig; - for( uchar i=0U; itxn_descriptor->signature_cnt; i++ ) { - fd_memcpy( &sig, sig_p, sizeof(fd_txn_key_t) ); - fd_txn_map_t * txn_map_entry = fd_txn_map_query( txn_map, &sig, NULL ); - if( FD_LIKELY( txn_map_entry ) ) { - txn_map_entry->meta_gaddr = meta_gaddr; - txn_map_entry->meta_sz = meta_sz; - } - sig_p += FD_ED25519_SIG_SZ; - } - - cur_laddr += meta_sz; - } - fd_log_collector_delete( &txn_ctx->log_collector ); - } - - FD_TEST( cur_laddr == end_laddr ); -} - /******************************************************************************/ /* Block-Level Execution Preparation/Finalization */ /******************************************************************************/ @@ -1817,7 +1544,6 @@ fd_runtime_pre_execute_check( fd_execute_txn_task_info_t * task_info, void fd_runtime_finalize_txn( fd_exec_slot_ctx_t * slot_ctx, - fd_capture_ctx_t * capture_ctx, fd_execute_txn_task_info_t * task_info, fd_spad_t * finalize_spad ) { @@ -1832,10 +1558,6 @@ fd_runtime_finalize_txn( fd_exec_slot_ctx_t * slot_ctx, fd_exec_txn_ctx_t * txn_ctx = task_info->txn_ctx; int exec_txn_err = task_info->exec_res; - /* For ledgers that contain txn status, decode and write out for solcap */ - if( capture_ctx != NULL && capture_ctx->capture && capture_ctx->capture_txns && slot_ctx->slot_bank.slot>=capture_ctx->solcap_start_slot ) { - fd_runtime_write_transaction_status( capture_ctx, slot_ctx, txn_ctx, exec_txn_err ); - } FD_ATOMIC_FETCH_AND_ADD( &slot_ctx->signature_cnt, txn_ctx->txn_descriptor->signature_cnt ); // if( slot_ctx->status_cache ) { @@ -2042,7 +1764,7 @@ fd_runtime_prepare_execute_finalize_txn_task( void * tpool, return; } - fd_runtime_finalize_txn( slot_ctx, capture_ctx, task_info, task_info->txn_ctx->spad ); + fd_runtime_finalize_txn( slot_ctx, task_info, task_info->txn_ctx->spad ); } /* fd_executor_txn_verify and fd_runtime_pre_execute_check are responisble diff --git a/src/flamenco/runtime/fd_runtime.h b/src/flamenco/runtime/fd_runtime.h index 57688751ae..66b33579e8 100644 --- a/src/flamenco/runtime/fd_runtime.h +++ b/src/flamenco/runtime/fd_runtime.h @@ -574,7 +574,6 @@ fd_runtime_process_txns_in_microblock_stream( fd_exec_slot_ctx_t * slot_ctx, void fd_runtime_finalize_txn( fd_exec_slot_ctx_t * slot_ctx, - fd_capture_ctx_t * capture_ctx, fd_execute_txn_task_info_t * task_info, fd_spad_t * finalize_spad ); diff --git a/src/flamenco/runtime/test_blockstore.c b/src/flamenco/runtime/test_blockstore.c index b40b2ce3f8..9bd06fb90c 100644 --- a/src/flamenco/runtime/test_blockstore.c +++ b/src/flamenco/runtime/test_blockstore.c @@ -88,9 +88,9 @@ main( int argc, char ** argv ) { ulong shred_max = 1 << 24; - void * mem = fd_wksp_alloc_laddr( wksp, fd_blockstore_align(), fd_blockstore_footprint( shred_max, 4096, 4096, shred_max ), 1UL ); + void * mem = fd_wksp_alloc_laddr( wksp, fd_blockstore_align(), fd_blockstore_footprint( shred_max, 4096, 4096 ), 1UL ); FD_TEST( mem ); - void * shblockstore = fd_blockstore_new( mem, 1UL, 42UL, shred_max, 4096, 4096, shred_max ); + void * shblockstore = fd_blockstore_new( mem, 1UL, 42UL, shred_max, 4096, 4096 ); FD_TEST( shblockstore ); fd_blockstore_t blockstore_ljoin; fd_blockstore_t * blockstore = fd_blockstore_join( &blockstore_ljoin, shblockstore );