Steal-able tasks/Task Queue + undefined behavior handler

examples/CMakeLists.txt

@@ -15,3 +15,4 @@ add_subdirectory(reduction_node)
 add_subdirectory(cth)
 add_subdirectory(self_send)
 add_subdirectory(ctv)
+add_subdirectory(taskqueue)

examples/reduction_dual/CMakeLists.txt

@@ -1,3 +1,2 @@
 add_reconverse_executable(reduction_dual reduction_dual.cpp)
 add_test(NAME reduction_dual COMMAND reduction_dual +pe 7)
-add_test(NAME reduction_dual-multinode COMMAND ${RECONVERSE_TEST_LAUNCHER} -n 2 $<TARGET_FILE:reduction_dual> +pe 4)

examples/taskqueue/CMakeLists.txt

@@ -0,0 +1 @@
+add_reconverse_executable(taskqueue taskqueue.cpp)

examples/taskqueue/Makefile

@@ -0,0 +1,23 @@
+# Compiler and flags
+CXX := g++
+CXXFLAGS := -std=c++11 -pthread -I ../..
+
+# Source files
+SRCS := taskqueue.cpp ../../convcore.cpp ../../scheduler.cpp ../../queue.cpp ../../conv-conds.cpp
+HDRS := ../../converse_internal.h ../../scheduler.h ../../queue.h ../../barrier.h
+
+# Output executable
+TARGET := test
+
+# Default target
+all: $(TARGET)
+
+# Link the object files to create the executable
+$(TARGET): $(SRCS)
+	$(CXX) -o $@ $^ $(CXXFLAGS)
+
+# Clean up build files
+clean:
+	rm -f $(TARGET)
+
+.PHONY: all clean

examples/taskqueue/taskqueue.cpp

@@ -0,0 +1,69 @@
+#include "converse.h"
+#include <pthread.h>
+#include <stdio.h>
+#include <atomic>
+
+#define K 2000 // will tend to abort if your k value is too high (higher than the max task queue size)
+#define X 10000
+
+thread_local int tasksExecuted = 0;
+static std::atomic<int> globalCounter{K};
+
+int handlerID;
+int print_handlerID;
+
+struct Message {
+  CmiMessageHeader header;
+  int data[1];
+};
+
+void print_results_handler_func(void* vmsg) {
+    printf("PE %d executed %d tasks.\n", CmiMyRank(), tasksExecuted);
+    CmiNodeBarrier();
+    if (CmiMyRank() == 0) {
+        CmiExit(0);
+    }
+}
+
+void handler_func(void *vmsg) {
+  Message* incoming_msg = (Message*)vmsg;  
+  //printf("PE %d pinged this function with data index: %d.\n", CmiMyRank(), incoming_msg->data[0]);
+
+  // do dummy work 
+  for (int i = 0; i < X; i++) {
+    CmiWallTimer();
+  }
+
+  tasksExecuted++; 
+
+  int prev = globalCounter.fetch_sub(1, std::memory_order_acq_rel);
+  //CmiPrintf("Current globalCounter: %d\n", prev - 1);
+  if (prev == 1) {
+    Message* msg = new Message; 
+    msg->header.handlerId = print_handlerID; 
+    msg->header.messageSize = sizeof(Message);
+    printf("All tasks have been executed.\n");
+    CmiSyncBroadcastAllAndFree(sizeof(Message), msg);
+  }
+}
+
+CmiStartFn mymain(int argc, char **argv) { 
+    handlerID = CmiRegisterHandler(handler_func);
+    print_handlerID = CmiRegisterHandler(print_results_handler_func);
+
+    if (CmiMyPe() == 0) {
+        for (int i = 0; i < K; i++) {
+            Message* newmsg = new Message; 
+            newmsg->data[0] = i; 
+            newmsg->header.messageSize = sizeof(Message);
+            newmsg->header.handlerId = handlerID;
+            CmiSyncTaskQSendAndFree(CmiMyRank(), sizeof(Message), newmsg);
+        }
+    }
+    return 0;
+}
+
+int main(int argc, char **argv) {
+  ConverseInit(argc, argv, (CmiStartFn)mymain);
+  return 0;
+}

include/converse.h

@@ -337,6 +337,9 @@ void CmiFreeSendFn(int destPE, int messageSize, char *msg);
 void CmiSyncListSendFn(int npes, const int *pes, int len, char *msg);
 void CmiFreeListSendFn(int npes, const int *pes, int len, char *msg);
 
+void CmiSyncTaskQSend(int destPE, int messageSize, void *msg);
+void CmiSyncTaskQSendAndFree(int destPE, int messageSize, void *msg);
+
 // broadcasts
 void CmiSyncBroadcast(int size, void *msg);
 void CmiSyncBroadcastAndFree(int size, void *msg);