Parallelization tune up and other cleanups

Polished up some thread parallelism rough edges noted in the recent
parallel thread sorting API installation.  Significant performance
improvement for thread parallelism strong scaling (not that it wasn't
already very fast) and some low impact tpool API changes from user point
of view for existing usages (that make it easier to use and more
flexible to deploy):

- Improved cache traffic between dispatch and worker threads in tpool.
  Thread dispatch / sync is low tens of percent faster (from ~400 ns per
  level to ~300 ns per level by reducing cache contention).  A side
  effect of this is that the tpool state variable was split into two
  sequence number variables.  This eliminated the
  FD_TPOOL_WORKER_STATE_* defines and corresponding fd_tpool_state_cstr
  APIs.  It also replaced the fd_tpool_worker_state with a simpler to
  use fd_tpool_worker_idle API.

- Added an run-time option for tpool threads to sleep when idle instead
  of spinning when idle.  This option slows down thread dispatch by over
  an order of magnitude (e.g.  ~28 us vs ~1.8 us to dispatch and
  synchronize 64 threads instead of ~1.8 us on typical x86 server ...
  the OS does a _terrible_ job at parallelism, especially at high core
  count).  But this is helpful for various kinds of testing and for
  situations where low latency / strong scaling is not a concern
  (especially with floating threads, low priority background processing,
  etc).  This added a new opt argument to fd_tool_init.

- Polished up FD_MAP_REDUCE / FD_FOR_ALL.  These now support passing
  more arguments, more flexibily and more simply than previously.  This
  tweaked how arguments are presented to implementation (simplifying
  practically, no more LOC than required for writing a normal function
  input argument list).  The dispatch logic was similarly tuned to take
  advantage of the above.

- Removed dependence on fd_scratch APIs from tpool.  This simplifies
  API usage and removes the scratch / scratch_sz argument from
  fd_tpool_worker_push.  At this point, suspect the fd_scratch API can
  be fully eliminated in favor of spad.

- Gave test_tpool the ability to select spin or sleeping style from
  command line and made minor improvement to the accuracy of the
  dispatch performance model in the various benchmarks.

- Did a linting pass on fd_util.h and related includes to reflect
  current util API organization.

Author: kbowers-jump

src/app/ledger/main.c

@@ -221,12 +221,12 @@ init_tpool( fd_ledger_args_t * ledger_args ) {
 
   ulong start_idx = 1UL;
   if( tcnt>=1UL ) {
-    tpool = fd_tpool_init( ledger_args->tpool_mem, tcnt );
+    tpool = fd_tpool_init( ledger_args->tpool_mem, tcnt, 0UL );
     if( tpool == NULL ) {
       FD_LOG_ERR(( "failed to create thread pool" ));
     }
     for( ulong i=1UL; i<tcnt; ++i ) {
-      if( fd_tpool_worker_push( tpool, start_idx++, NULL, 0UL ) == NULL ) {
+      if( fd_tpool_worker_push( tpool, start_idx++ ) == NULL ) {
         FD_LOG_ERR(( "failed to launch worker" ));
       }
       else {
@@ -249,25 +249,24 @@ init_tpool( fd_ledger_args_t * ledger_args ) {
     FD_LOG_ERR(( "This is an invalid value for the number of threads to use for snapshot creation" ));
   }
 
-  fd_tpool_t * snapshot_bg_tpool = fd_tpool_init( ledger_args->tpool_mem_snapshot_bg, snapshot_tcnt );
-  if( FD_UNLIKELY( !fd_tpool_worker_push( snapshot_bg_tpool, start_idx++, NULL, 0UL ) ) ) {
+  fd_tpool_t * snapshot_bg_tpool = fd_tpool_init( ledger_args->tpool_mem_snapshot_bg, snapshot_tcnt, 0UL );
+  if( FD_UNLIKELY( !fd_tpool_worker_push( snapshot_bg_tpool, start_idx++ ) ) ) {
       FD_LOG_ERR(( "failed to launch worker" ));
   } else {
     FD_LOG_NOTICE(( "launched snapshot bg worker %lu", start_idx - 1UL ));
   }
 
   ledger_args->snapshot_bg_tpool = snapshot_bg_tpool;
 
-
   if( snapshot_tcnt==2UL ) {
     return 0;
   }
 
   /* If a snapshot is being created, setup its own tpool. */
 
-  fd_tpool_t * snapshot_tpool = fd_tpool_init( ledger_args->tpool_mem_snapshot, snapshot_tcnt - 1UL );
+  fd_tpool_t * snapshot_tpool = fd_tpool_init( ledger_args->tpool_mem_snapshot, snapshot_tcnt - 1UL, 0UL );
   for( ulong i=1UL; i<snapshot_tcnt - 1UL; ++i ) {
-    if( FD_UNLIKELY( !fd_tpool_worker_push( snapshot_tpool, start_idx++, NULL, 0UL ) ) ) {
+    if( FD_UNLIKELY( !fd_tpool_worker_push( snapshot_tpool, start_idx++ ) ) ) {
       FD_LOG_ERR(( "failed to launch worker" ));
     } else {
       FD_LOG_NOTICE(( "launched snapshot hash worker %lu", start_idx - 1UL ));
@@ -1251,7 +1250,6 @@ replay( fd_ledger_args_t * args ) {
 
   FD_SPAD_FRAME_BEGIN( spad ) {
 
-
   /* Setup slot_ctx */
   fd_funk_t * funk = args->funk;
 

src/discof/consensus/test_consensus.c

@@ -892,7 +892,7 @@ main( void ) {
 //   uchar tpool_mem[FD_TPOOL_FOOTPRINT( FD_TILE_MAX )]__attribute__((aligned(FD_TPOOL_ALIGN))) = { 0 };
 //   /* clang-format on */
 //   if( tile_cnt > 4 ) {
-//     tpool = fd_tpool_init( tpool_mem, tile_cnt );
+//     tpool = fd_tpool_init( tpool_mem, tile_cnt, 0UL );
 //     FD_TEST( tpool );
 //     if( tpool == NULL ) FD_LOG_ERR( ( "failed to create thread pool" ) );
 //     ulong   scratch_sz = fd_scratch_smem_footprint( 256 << 20 );

src/flamenco/runtime/fd_runtime.c

@@ -281,7 +281,6 @@ fd_runtime_validate_fee_collector( fd_exec_slot_ctx_t const * slot_ctx,
   return 0UL;
 }
 
-
 static int
 fd_runtime_run_incinerator( fd_exec_slot_ctx_t * slot_ctx ) {
   FD_TXN_ACCOUNT_DECL( rec );
@@ -658,7 +657,6 @@ fd_runtime_collect_from_existing_account( ulong                       slot,
   #undef COLLECT_RENT
 }
 
-
 /* fd_runtime_collect_rent_from_account performs rent collection duties.
    Although the Solana runtime prevents the creation of new accounts
    that are subject to rent, some older accounts are still undergo the
@@ -1391,7 +1389,6 @@ fd_runtime_prepare_txns_start( fd_exec_slot_ctx_t *         slot_ctx,
 
     fd_rawtxn_b_t raw_txn = { .raw = txn->payload, .txn_sz = (ushort)txn->payload_sz };
 
-
     task_info[txn_idx].txn_ctx->spad      = runtime_spad;
     task_info[txn_idx].txn_ctx->spad_wksp = fd_wksp_containing( runtime_spad );
     int err = fd_execute_txn_prepare_start( slot_ctx,
@@ -1837,8 +1834,7 @@ fd_runtime_process_txns_in_microblock_stream( fd_exec_slot_ctx_t * slot_ctx,
       if( curr_exec_idx>=txn_cnt ) {
         break;
       }
-      int state = fd_tpool_worker_state( tpool, worker_idx );
-      if( state!=FD_TPOOL_WORKER_STATE_IDLE ) {
+      if( !fd_tpool_worker_idle( tpool, worker_idx ) ) {
         continue;
       }
 

src/flamenco/runtime/tests/harness/fd_block_harness.c

@@ -374,8 +374,8 @@ fd_runtime_fuzz_block_ctx_exec( fd_runtime_fuzz_runner_t * runner,
   /* Initialize tpool and spad(s) */
   ulong        worker_max = FD_BLOCK_HARNESS_TPOOL_WORKER_CNT;
   void *       tpool_mem  = fd_spad_alloc( runner->spad, FD_TPOOL_ALIGN, FD_TPOOL_FOOTPRINT( worker_max ) );
-  fd_tpool_t * tpool      = fd_tpool_init( tpool_mem, worker_max );
-  fd_tpool_worker_push( tpool, 1UL, NULL, 0UL );
+  fd_tpool_t * tpool      = fd_tpool_init( tpool_mem, worker_max, 0UL );
+  fd_tpool_worker_push( tpool, 1UL );
 
   fd_spad_t * runtime_spad = runner->spad;
 

src/util/alloc/fd_alloc.h

@@ -120,7 +120,6 @@
    time to try to bound the amount of pre-allocation for small requests. */
 
 #include "../wksp/fd_wksp.h"
-#include "../valloc/fd_valloc.h"
 
 /* FD_ALLOC_{ALIGN,FOOTPRINT} give the required alignment and footprint
    needed for a wksp allocation to be suitable as a fd_alloc.  ALIGN is
@@ -633,4 +632,3 @@ fd_alloc_virtual( fd_alloc_t * alloc ) {
 FD_PROTOTYPES_END
 
 #endif /* HEADER_fd_src_util_alloc_fd_alloc_h */
-

src/util/archive/fd_tar.h

@@ -5,7 +5,6 @@
    format. This is not a general-purpose TAR implementation.  It is
    currently only intended for loading and writing Solana snapshots. */
 
-#include "../fd_util_base.h"
 #include "../io/fd_io.h"
 
 /* File Format ********************************************************/
@@ -84,7 +83,7 @@ fd_tar_set_octal( char  buf[ static 12 ],
                   ulong val );
 
 /* fd_tar_meta_set_size sets the size field.  Returns 1 on success, 0
-   if sz is too large to be represented in TAR header. Set size using the 
+   if sz is too large to be represented in TAR header. Set size using the
    OLDGNU size extension to allow for unlimited file sizes. The first byte
    must be 0x80 followed by 0s and then the size in binary. */
 
@@ -240,30 +239,30 @@ fd_tar_read( void *        reader,
     2. Write out file data with fd_tar_writer_write_file_data( writer, data, data_sz ).
        This can be done as many times as you want.
     3. Finish the current file with fd_tar_writer_fini_file( writer ).
-  
-   When you are done, call fd_tar_writer_delete( writer ) to write out the 
+
+   When you are done, call fd_tar_writer_delete( writer ) to write out the
    tar archive trailer and close otu the file descriptor.
 
-   If you want to reserve space for an existing file and write back to it 
+   If you want to reserve space for an existing file and write back to it
    at some point in the future see the below comments for
    fd_tar_writer_{make,fill}_space().
-   
+
    */
 
 struct fd_tar_writer {
   int                      fd;         /* The file descriptor for the tar archive. */
   ulong                    header_pos; /* The position in the file for the current files header.
-                                          If there is no current file that is being streamed out, 
+                                          If there is no current file that is being streamed out,
                                           the header_pos will be equal to ULONG_MAX. */
   ulong                    data_sz;    /* The size of the current files data. If there is no
                                           current file that is being streamed out, the data_sz
                                           will be equal to ULONG_MAX. */
   ulong                    wb_pos;     /* If this value is not equal to ULONG_MAX that means that
-                                          this is the position at which to write back to with a 
+                                          this is the position at which to write back to with a
                                           call to fd_tar_writer_fill_space. */
-  /* TODO: Right now, the stream to the tar writer just uses fd_io_write. 
+  /* TODO: Right now, the stream to the tar writer just uses fd_io_write.
      This can eventually be abstracted to use write callbacks that use
-     fd_io streaming under the hood. This adds some additional complexity 
+     fd_io streaming under the hood. This adds some additional complexity
      that's related to writing back into the header: if the header is still
      in the ostream buf, modify the buffer. Otherwise, read the header
      directly from the file. */
@@ -333,15 +332,15 @@ fd_tar_writer_fini_file( fd_tar_writer_t * writer );
 /* fd_tar_writer_make_space and fd_tar_writer_fill_space, allow for writing
    back to a specific place in the tar stream. This can be used by first
    making a call to fd_tar_write_new_file, fd_tar_writer_make_space, and
-   fd_tar_writer_fini_file. This will populate the header and write out 
+   fd_tar_writer_fini_file. This will populate the header and write out
    random bytes. The start of this data file will be saved by the tar writer.
-   Up to n data files can be appended to the tar archive before a call to 
+   Up to n data files can be appended to the tar archive before a call to
    fd_tar_writer_fill_space. fd_tar_writer_fill_space should only be called
    after an unpaired call to fd_tar_writer_make_space and it requires a valid
    fd_tar_writer_t handle. It allows the user to write back to the point at
    which they made space. _make_space and _fill_space should be paired together.
    There can only be one oustanding call to make_space at a time.
-   
+
    TODO: This can be extended to support multiple write backs. */
 
 int

src/util/bits/fd_bits.h

@@ -3,7 +3,7 @@
 
 /* Bit manipulation APIs */
 
-#include "../sanitize/fd_msan.h"
+#include "../sanitize/fd_sanitize.h"
 
 FD_PROTOTYPES_BEGIN
 

src/util/fd_util.h

@@ -1,50 +1,50 @@
 #ifndef HEADER_fd_src_util_fd_util_h
 #define HEADER_fd_src_util_fd_util_h
 
-//#include "fd_util_base.h"         /* includes stdalign.h, string.h, limits.h, float.h */
-//#include "sanitize/fd_msan.h"     /* includes fd_util_base.h */
-//#include "bits/fd_bits.h"         /* includes sanitize/fd_msan.h */
-//#include "sanitize/fd_asan.h"     /* includes fd_util_base.h" */
-//#include "sanitize/fd_msan.h"     /* includes fd_util_base.h" */
-//#include "sanitize/fd_sanitize.h" /* includes sanitize/fd_asan.h sanitize/fd_msan.h */
-//#include "io/fd_io.h"             /* includes bits/fd_bits.h */
-#include "spad/fd_spad.h"           /* includes bits/fd_bits.h */
-//#include "cstr/fd_cstr.h"         /* includes bits/fd_bits.h */
-//#include "pod/fd_pod.h"           /* includes cstr/fd_cstr.h */
-//#include "env/fd_env.h"           /* includes cstr/fd_cstr.h */
-//#include "log/fd_log.h"           /* includes env/fd_env.h io/fd_io.h */
-//#include "checkpt/fd_checkpt.h"   /* includes log/fd_log.h */
-//#include "shmem/fd_shmem.h"       /* includes log/fd_log.h */
-//#include "tile/fd_tile.h"         /* includes shmem/fd_shmem.h */
-//#include "scratch/fd_scratch.h"   /* includes tile/fd_tile.h sanitize/fd_sanitize.h valloc/fd_valloc.h */
-//#include "tpool/fd_tpool.h"       /* includes tile/fd_tile.h scratch/fd_scratch.h */
-//#include "wksp/fd_wksp.h"         /* tpool/fd_tpool.h checkpt/fd_checkpt.h sanitize/fd_sanitize.h */
-#include "alloc/fd_alloc.h"         /* includes wksp/fd_wksp.h valloc/fd_valloc.h */
-#include "rng/fd_rng.h"             /* includes bits/fd_bits.h */
-
-/* FIXME: Should these be optional APIs? */
-#include "sandbox/fd_sandbox.h"     /* includes fd_util_base.h */
-//#include "math/fd_stat.h"         /* includes bits/fd_bits.h */
-#include "bits/fd_sat.h"            /* includes bits/fd_bits.h */
-//#include "hist/fd_histf.h"        /* includes log/fd_log.h */
+//#include "fd_util_base.h"          /* includes stdalign.h string.h limits.h float.h */
+//#include "sanitize/fd_sanitize.h"  /* includes fd_util_base.h (fd_asan.h fd_msan.h) */
+//#include "valloc/fd_valloc.h"      /* includes fd_util_base.h */ /* FIXME: deprecate? */
+//#include "bits/fd_bits.h"          /* includes sanitize/fd_sanitize.h (fd_bits_find_lsb.h fd_bits_find_msb.h fd_bits_tg.h) */
+//#include "io/fd_io.h"              /* includes bits/fd_bits.h */
+//#include "cstr/fd_cstr.h"          /* includes bits/fd_bits.h */
+#include "rng/fd_rng.h"              /* includes bits/fd_bits.h */
+#include "spad/fd_spad.h"            /* includes bits/fd_bits.h valloc/fd_valloc.h */
+//#include "env/fd_env.h"            /* includes cstr/fd_cstr.h */
+//#include "log/fd_log.h"            /* includes env/fd_env.h io/fd_io.h */
+//#include "checkpt/fd_checkpt.h"    /* includes log/fd_log.h */
+//#include "shmem/fd_shmem.h"        /* includes log/fd_log.h */
+//#include "tile/fd_tile.h"          /* includes shmem/fd_shmem.h */
+//#include "scratch/fd_scratch.h"    /* includes tile/fd_tile.h valloc/fd_valloc.h */ /* FIXME: deprecate non alloca parts? */
+//#include "tpool/fd_tpool.h"        /* includes scratch/fd_scratch.h */
+//#include "wksp/fd_wksp.h"          /* includes tpool/fd_tpool.h checkpt/fd_checkpt.h */
+#include "alloc/fd_alloc.h"          /* includes wksp/fd_wksp.h */
+
+#include "sandbox/fd_sandbox.h"      /* includes fd_util_base.h */ /* FIXME: should this be included by default? */
+#include "bits/fd_sat.h"             /* includes bits/fd_bits.h */ /* FIXME: should this be incldued by default? */
 
 /* Additional fd_util APIs that are not included by default */
 
-//#include "archive/fd_ar.h"        /* includes fd_util_base.h */
-//#include "net/fd_pcapng.h"        /* includes fd_util_base.h */
-//#include "net/fd_eth.h"           /* includes bits/fd_bits.h */
-//#include "net/fd_ip4.h"           /* includes bits/fd_bits.h */
-//#include "net/fd_igmp.h"          /* includes net/fd_ip4.h */
-//#include "net/fd_udp.h"           /* includes net/fd_ip4.h */
-//#include "net/fd_net_headers.h */ /* includes net/fd_udp.h net/fd_eth.h */
-//#include "net/fd_pcap.h"          /* includes net/fd_eth.h log/fd_log.h */
-//#include "bits/fd_float.h"        /* includes bits/fd_bits.h */
-//#include "bits/fd_uwide.h"        /* includes bits/fd_bits.h */
-//#include "math/fd_sqrt.h"         /* includes bits/fd_bits.h */
-//#include "math/fd_fxp.h"          /* includes math/fd_sqrt.h, (!FD_HAS_INT128) bits/fd_uwide.h */
-//#include "simd/fd_sse.h"          /* includes bits/fd_bits.h, requires FD_HAS_SSE */
-//#include "simd/fd_avx.h"          /* includes bits/fd_bits.h, requires FD_HAS_AVX */
-//#include "simd/fd_avx512.h"       /* includes bits/fd_bits.h, requires FD_HAS_AVX512 */
+//#include "archive/fd_ar.h"         /* includes fd_util_base.h */
+//#include "archive/fd_tar.h"        /* includes fd_io.h */
+//#include "bits/fd_float.h"         /* includes bits/fd_bits.h */
+//#include "bits/fd_uwide.h"         /* includes bits/fd_bits.h */
+//#include "hist/fd_histf.h"         /* includes log/fd_log.h math.h (FD_HAS_AVX) fd_avx.h */
+//#include "math/fd_stat.h"          /* includes bits/fd_bits.h */
+//#include "math/fd_sqrt.h"          /* includes bits/fd_bits.h */
+//#include "math/fd_fxp.h"           /* includes math/fd_sqrt.h, (!FD_HAS_INT128) bits/fd_uwide.h */
+//#include "net/fd_pcapng.h"         /* includes fd_util_base.h */
+//#include "net/fd_eth.h"            /* includes bits/fd_bits.h */
+//#include "net/fd_ip4.h"            /* includes bits/fd_bits.h */
+//#include "net/fd_igmp.h"           /* includes net/fd_ip4.h */
+//#include "net/fd_udp.h"            /* includes net/fd_ip4.h */
+//#include "net/fd_net_headers.h */  /* includes net/fd_udp.h net/fd_eth.h */
+//#include "net/fd_pcap.h"           /* includes net/fd_eth.h log/fd_log.h */
+//#include "pod/fd_pod.h"            /* includes cstr/fd_cstr.h */
+//#include "sanitize/fd_fuzz.h"      /* includes fd_util_base.h */
+//#include "sanitize/fd_backtrace.h" /* FIXME: this probably should be merged with another header */
+//#include "simd/fd_sse.h"           /* includes bits/fd_bits.h, requires FD_HAS_SSE */
+//#include "simd/fd_avx.h"           /* includes bits/fd_bits.h, requires FD_HAS_AVX */
+//#include "simd/fd_avx512.h"        /* includes bits/fd_bits.h, requires FD_HAS_AVX512 */
 
 FD_PROTOTYPES_BEGIN
 

src/util/scratch/fd_scratch.h

@@ -9,7 +9,6 @@
    It is meant for use in situations that have very complex and large
    temporary memory usage. */
 
-#include "../sanitize/fd_sanitize.h"
 #include "../tile/fd_tile.h"
 #include "../valloc/fd_valloc.h"
 

src/util/spad/fd_spad.h

@@ -1,8 +1,6 @@
 #ifndef HEADER_fd_src_util_spad_fd_spad_h
 #define HEADER_fd_src_util_spad_fd_spad_h
 
-#include "../sanitize/fd_sanitize.h"
-
 /* APIs for high performance persistent inter-process shared scratch pad
    memories.  A spad as a scratch pad that behaves very much like a
    thread's stack:
@@ -296,7 +294,6 @@ FD_FN_PURE static inline ulong fd_spad_mem_wmark( fd_spad_t const * spad ) { ret
 
 FD_FN_PURE static inline int fd_spad_in_frame( fd_spad_t const * spad ) { return spad->frame_free<FD_SPAD_FRAME_MAX; }
 
-
 /* operations */
 /* fd_spad_alloc_max returns the maximum number of bytes with initial
    byte alignment of align that can currently be allocated / prepared
@@ -532,7 +529,6 @@ fd_spad_virtual( fd_spad_t * spad ) {
   return valloc;
 }
 
-
 /* fn implementations */
 static inline void
 fd_spad_reset_impl( fd_spad_t * spad ) {