Bug#36886242 Node failure late in Index Statistics Update schema

 transaction causes president failure

Problem

The Index Statistics UPDATE_STAT Schema Transaction can be
generated automatically by Ndb data nodes in response to
monitoring data change rates, or triggered as a part of
schema changes or manually by ANALYZE_TABLE et al.

Most Schema Transactions (ST) involve all data nodes equally
and so the survivable loss of one data node does not affect
the outcome, and the surviving nodes carry on.

The UPDATE_STAT ST has phases operating only on one node,
so that failure of that node does affect the overall outcome.

This is handled by requiring that any node failure during
the UPDATE_STAT ST results in the transaction being rolled
back.

However this mechanism did not handle the case where the
UPDATE_STAT ST was already in the short final commit or
complete phases.

In that case it caused an assertion failure at the
Master/President node.

That behaves as a cascading node failure - one node
failure leading to another, which endangers availability,
especially if the nodes involved are in the same nodegroup.

Solution

The UPDATE_STAT ST rollback on node failure behaviour is
limited to the pre-commit ST states.

This avoids the assertion failure and cascading node
failures.

Testing

A new testcase is added :

  testDict -n IndexStatNodeFailures T1

Change-Id: I5915087fa5d24b50e11bb5697e93400443a367d3

Author: frazerclement

storage/ndb/src/kernel/blocks/ERROR_codes.txt

@@ -27,7 +27,7 @@ Next NDBFS 2003
 Next DBACC 3006
 Next DBTUP 4039
 Next DBLQH 5112
-Next DBDICT 6223
+Next DBDICT 6227
 Next DBDIH 7251
 Next DBTC 8124
 Next TRPMAN 9007

storage/ndb/src/kernel/blocks/dbdict/Dbdict.cpp

@@ -16839,6 +16839,8 @@ Dbdict::indexStat_prepare(Signal* signal, SchemaOpPtr op_ptr)
 
   D("indexStat_prepare" << V(*op_ptr.p));
 
+  CRASH_INSERTION(6224);
+
   if (impl_req->requestType == IndexStatReq::RT_UPDATE_STAT ||
       impl_req->requestType == IndexStatReq::RT_DELETE_STAT) {
     // the main op of stat update or delete does nothing
@@ -16895,7 +16897,11 @@ Dbdict::indexStat_toLocalStat(Signal* signal, SchemaOpPtr op_ptr)
 
   switch (impl_req->requestType) {
   case IndexStatReq::RT_SCAN_FRAG:
-    trans_ptr.p->m_abort_on_node_fail = true;
+    /**
+     * Node failure during prepare phase should result in ST rollback
+     * to handle the case where the single scanning node has failed
+     */
+    trans_ptr.p->m_abort_on_node_fail_pre_commit = true;
     req->fragId = indexPtr.p->indexStatFragId;
     if (!do_action(trans_ptr.p->m_nodes, indexPtr.p->indexStatNodes,
                    getOwnNodeId()))
@@ -16990,6 +16996,7 @@ Dbdict::indexStat_commit(Signal* signal, SchemaOpPtr op_ptr)
   IndexStatRecPtr indexStatPtr;
   getOpRec(op_ptr, indexStatPtr);
   D("indexStat_commit" << *op_ptr.p);
+  CRASH_INSERTION(6225);
   sendTransConf(signal, op_ptr);
 }
 
@@ -17002,6 +17009,7 @@ Dbdict::indexStat_complete(Signal* signal, SchemaOpPtr op_ptr)
   IndexStatRecPtr indexStatPtr;
   getOpRec(op_ptr, indexStatPtr);
   D("indexStat_complete" << *op_ptr.p);
+  CRASH_INSERTION(6226);
   sendTransConf(signal, op_ptr);
 }
 
@@ -30383,15 +30391,37 @@ Dbdict::execSCHEMA_TRANS_IMPL_REF(Signal* signal)
     jam();
     // trans_ptr.p->m_nodes.clear(nodeId);
     // No need to clear, will be cleared when next REQ is set
-    if (!trans_ptr.p->m_abort_on_node_fail)
+    if (!trans_ptr.p->m_abort_on_node_fail_pre_commit)
     {
       jam();
       ref->errorCode = 0;
     }
     else
     {
       jam();
-      ref->errorCode = SchemaTransBeginRef::Nodefailure;
+      /* Abort on node fail if in pre-commit phase */
+      switch (trans_ptr.p->m_state) {
+        case SchemaTrans::TS_FLUSH_COMMIT:
+          jam();
+          /* fallthrough */;
+        case SchemaTrans::TS_COMMITTING:
+          jam();
+          /* fallthrough */;
+        case SchemaTrans::TS_FLUSH_COMPLETE:
+          jam();
+          /* fallthrough */;
+        case SchemaTrans::TS_COMPLETING:
+          jam();
+          /* fallthrough */;
+        case SchemaTrans::TS_ENDING:
+          jam();
+          /* Ignore */
+          ref->errorCode = 0;
+          break;
+        default:
+          ref->errorCode = SchemaTransBeginRef::Nodefailure;
+          break;
+      }
     }
   }
 

storage/ndb/src/kernel/blocks/dbdict/Dbdict.hpp

@@ -2380,7 +2380,7 @@ class Dbdict: public SimulatedBlock {
     bool m_flush_complete;
     bool m_flush_end;
     bool m_wait_gcp_on_commit;
-    bool m_abort_on_node_fail;
+    bool m_abort_on_node_fail_pre_commit;
 
     // magic is on when record is seized
     enum { DICT_MAGIC = ~RT_DBDICT_SCHEMA_TRANSACTION };
@@ -2406,7 +2406,7 @@ class Dbdict: public SimulatedBlock {
       m_flush_complete = false;
       m_flush_end = false;
       m_wait_gcp_on_commit = true;
-      m_abort_on_node_fail = false;
+      m_abort_on_node_fail_pre_commit = false;
     }
 
     SchemaTrans(Uint32 the_trans_key) {

storage/ndb/test/ndbapi/testDict.cpp

@@ -11745,6 +11745,89 @@ runCreateManyDataFiles(NDBT_Context* ctx, NDBT_Step* step)
   return result;
 }
 
+int runIndexStatNF(NDBT_Context *ctx, NDBT_Step *step) {
+  NdbRestarter restarter;
+  Ndb *pNdb = GETNDB(step);
+  NdbDictionary::Dictionary *pDict = pNdb->getDictionary();
+  const NdbDictionary::Table *pTab = ctx->getTab();
+  if (pTab == NULL) {
+    ndbout << "Failed to get table " << endl;
+    return NDBT_FAILED;
+  }
+
+  char idxname[20];
+  sprintf(idxname, "%s_idx", pTab->getName());
+  const NdbDictionary::Index *pIdx = pDict->getIndex(idxname, pTab->getName());
+  if (pIdx == NULL) {
+    ndbout << "Failed to get index" << idxname << " error "
+           << pDict->getNdbError() << endl;
+    return NDBT_FAILED;
+  }
+
+  /**
+   * Test node failures while update index stats schema transaction
+   * is in various phases (prepare, commit, complete)
+   * Test node failures of :
+   *   Master node (requiring DICT SchemaTrans takeover)
+   *   Non master node (requiring node failure handling)
+   */
+  /* Fail in index stat ST prepare, commit, complete */
+  int nfcases[] = {6224, 6225, 6226};
+  const int numCases = sizeof(nfcases) / sizeof(int);
+  for (int s = 0; s < 2; s++) {
+    const char *sname = (s == 0) ? "Master" : "Non-master";
+
+    ndbout_c("Scenario %d, inserting errors in %s node", s, sname);
+
+    for (int c = 0; c < numCases; c++) {
+      int errorCode = nfcases[c];
+      int errorNodeId = restarter.getNode(
+          (s == 0) ? NdbRestarter::NS_MASTER : NdbRestarter::NS_NON_MASTER);
+      ndbout_c("Inserting error %u in %s node %u", errorCode, sname,
+               errorNodeId);
+
+      if (restarter.insertErrorInNode(errorNodeId, errorCode) != 0) {
+        ndbout_c("Failed to insert error");
+        return NDBT_FAILED;
+      }
+
+      ndbout_c("Requesting updated index stats");
+      do {
+        if (pDict->updateIndexStat(*pIdx, *pTab) == 0) {
+          ndbout_c("Success generating new index stats");
+          break;
+        } else {
+          const NdbError err = pDict->getNdbError();
+
+          ndbout_c("Index stat update failed with error %u %s", err.code,
+                   err.message);
+
+          if (err.code == 781 || /* Invalid schema transaction key */
+              err.code == 286 || /* Node failure caused abort */
+              err.code == 787)   /* Schema transaction aborted */
+          {
+            ndbout_c("Retrying");
+            continue;
+          }
+
+          return NDBT_FAILED;
+        }
+      } while (1);
+
+      ndbout_c("Wait for all data nodes to be running");
+
+      if (restarter.waitClusterStarted() != 0) {
+        ndbout_c("Timed out waiting for nodes to start");
+        return NDBT_FAILED;
+      }
+    }
+  }
+
+  ndbout_c("Success");
+
+  return NDBT_OK;
+}
+
 NDBT_TESTSUITE(testDict);
 TESTCASE("testDropDDObjects",
          "* 1. start cluster\n"
@@ -12179,6 +12262,14 @@ TESTCASE("CreateManyDataFiles", "Test lack of DiskPageBufferMemory "
   FINALIZER(runDropTableSpaceLG);
   FINALIZER(changeStartDiskPageBufMem);
 }
+TESTCASE("IndexStatNodeFailures",
+         "Test node failures in various phases of index stat updates") {
+  INITIALIZER(runCreateTheTable);
+  INITIALIZER(runCreateTheIndex);
+  STEP(runIndexStatNF);
+  FINALIZER(runDropTheIndex);
+  FINALIZER(runDropTheTable);
+}
 
 NDBT_TESTSUITE_END(testDict);
 

storage/ndb/test/run-test/daily-devel--07-tests.txt

@@ -289,3 +289,7 @@ cmd: test_event
 args: -n MergeEventConsumer_Graceful T1 --loops=3
 max-time: 480
 
+cmd: testDict
+args: -n IndexStatNodeFailures T1
+max-time: 360
+