Bug#36750146 Ndb : Graceful node shutdown does not avoid event duplicates

Graceful node stop is intended to carefully handover
responsibility for forwarding event streams from the stopping
node to still-running nodes, so that there are no duplicate
events sent to event subscribers as a result of an unexpected
node failure.

However, this was not working correctly which could lead
to issues with event merge and other event consumers even
when performing a graceful node stop.

The problem is fixed and two new tests covering the problem
with Graceful stop are added :

  test_event -n EventConsumer_Graceful
  test_event -n MergeEventConsumer_Graceful

Change-Id: I2effa2e0908c920e48ac235a5e0f3a1fce10413a

Author: frazerclement

storage/ndb/src/kernel/blocks/suma/Suma.cpp

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2003, 2023, Oracle and/or its affiliates.
+   Copyright (c) 2003, 2024, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License, version 2.0,
@@ -5255,6 +5255,11 @@ Suma::execFIRE_TRIG_ORD(Signal* signal)
 
   Uint32 bucket= hashValue % c_no_of_buckets;
   m_max_seen_gci = (gci > m_max_seen_gci ? gci : m_max_seen_gci);
+  /**
+   * Normally a bucket is either in the active or switchover set, or neither.
+   * Exception is during add/drop NG when an active bucket may be in the
+   * switchover set for notifying subscribers of change.
+   */
   if(m_active_buckets.get(bucket) || 
      (m_switchover_buckets.get(bucket) && (check_switchover(bucket, gci))))
   {
@@ -6871,9 +6876,10 @@ Suma::execSUMA_HANDOVER_REQ(Signal* signal)
       if(get_responsible_node(i) == nodeId &&
          get_responsible_node(i, nodegroup) == getOwnNodeId())
       {
-        // I'm will be running this bucket when nodeId shutdown
+        // I will be running this bucket when nodeId shutdown
         jam();
         tmp.set(i);
+        ndbassert(!m_active_buckets.get(i));
         m_switchover_buckets.set(i);
         c_buckets[i].m_switchover_gci = (Uint64(start_gci) << 32) - 1;
         c_buckets[i].m_state |= Bucket::BUCKET_SHUTDOWN_TO;
@@ -6962,6 +6968,7 @@ Suma::execSUMA_HANDOVER_CONF(Signal* signal) {
               nodeId, gci, buf, c_no_of_buckets);
     g_eventLogger->info("Suma: handover from node %u gci: %u buckets: %s (%u)",
                         nodeId, gci, buf, c_no_of_buckets);
+    ndbassert(!m_active_buckets.overlaps(tmp));
     m_switchover_buckets.bitOR(tmp);
     c_startup.m_handover_nodes.clear(nodeId);
     DBUG_VOID_RETURN;
@@ -6987,6 +6994,7 @@ Suma::execSUMA_HANDOVER_CONF(Signal* signal) {
               nodeId, gci, buf, c_no_of_buckets);
     g_eventLogger->info("Suma: handover to node %u gci: %u buckets: %s (%u)",
                         nodeId, gci, buf, c_no_of_buckets);
+    m_active_buckets.bitANDC(tmp);
     m_switchover_buckets.bitOR(tmp);
     c_startup.m_handover_nodes.clear(nodeId);
     DBUG_VOID_RETURN;
@@ -7417,6 +7425,7 @@ Suma::start_resend(Signal* signal, Uint32 buck)
   bucket->m_switchover_node = get_responsible_node(buck);
   bucket->m_switchover_gci = max;
 
+  ndbassert(!m_active_buckets.get(buck));
   m_switchover_buckets.set(buck);
 
   signal->theData[0] = SumaContinueB::RESEND_BUCKET;