Support fp32 bp

Author: RaymondWang0

TinyEngine/include/nnfunctions.h

@@ -1,23 +1,27 @@
 /* ----------------------------------------------------------------------
- * Project: TinyEngine
+ * Project: Tiny Training Engine, MCUNetV3
  * Title:   tinyengine_function.h
  *
  * Reference papers:
  *  - MCUNet: Tiny Deep Learning on IoT Device, NeurIPS 2020
  *  - MCUNetV2: Memory-Efficient Patch-based Inference for Tiny Deep Learning, NeurIPS 2021
- *  - MCUNetV3: On-Device Training Under 256KB Memory, arXiv:2206.15472
+ *  - MCUNetV3: On-Device Training Under 256KB Memory, NeurIPS 2022
  * Contact authors:
- *  - Wei-Ming Chen, [email protected]
  *  - Wei-Chen Wang, [email protected]
+ *  - Wei-Ming Chen, [email protected]
  *  - Ji Lin, [email protected]
  *  - Ligeng Zhu, [email protected]
  *  - Song Han, [email protected]
+ *  - Chuang Gan, [email protected]
  *
  * Target ISA:  ARMv7E-M
  * -------------------------------------------------------------------- */
 
 #include <stdint.h>
 #include <stdbool.h>
+#include <stdlib.h>
+#include <math.h>
+
 typedef int8_t q7_t;
 typedef uint8_t q8_t;
 typedef int16_t q15_t;
@@ -146,9 +150,6 @@ tinyengine_status add_fpreq_bitmask(int size, const int8_t* input1_data, const f
 			const int8_t* input2_data, const float input2_scale, const float input2_zero, const float output_scale,
 			const float zero_y, int8_t* output_data, int8_t* output_mask);
 
-tinyengine_status where_int8(const bool* inMask, const uint16_t size, signed char* input1_data,
-	    const char* input2_data, char* output_data);
-
 tinyengine_status convolve_1x1_s8_fpreq_mask_partialCH(const q7_t *input,
 		const uint16_t input_x, const uint16_t input_y, const uint16_t input_ch,
 		const q7_t *kernel_sram, const q7_t *kernel_flash, const uint16_t first_k_channel, const int32_t *bias, const float *scales,
@@ -157,5 +158,7 @@ tinyengine_status convolve_1x1_s8_fpreq_mask_partialCH(const q7_t *input,
 		q7_t *output, q7_t *mask, const uint16_t output_x, const uint16_t output_y,
 		const uint16_t output_ch, q15_t *runtime_buf);
 
+
 #include "genInclude.h"
 #include "fp_requantize_op.h"
+//#include "int8_bp_op.h"

TinyEngine/include/nnfunctions_fp.h

@@ -0,0 +1,32 @@
+/* ----------------------------------------------------------------------
+ * Project: Tiny Training Engine, MCUNetV3
+ * Title:   add_fp.c
+ *
+ * Reference papers:
+ *  - MCUNet: Tiny Deep Learning on IoT Device, NeurIPS 2020
+ *  - MCUNetV2: Memory-Efficient Patch-based Inference for Tiny Deep Learning, NeurIPS 2021
+ *  - MCUNetV3: On-Device Training Under 256KB Memory, NeurIPS 2022
+ * Contact authors:
+ *  - Wei-Chen Wang, [email protected]
+ *  - Wei-Ming Chen, [email protected]
+ *  - Ji Lin, [email protected]
+ *  - Ligeng Zhu, [email protected]
+ *  - Song Han, [email protected]
+ *  - Chuang Gan, [email protected]
+ *
+ * Target ISA:  ARMv7E-M
+ * -------------------------------------------------------------------- */
+
+#include "tinyengine_function_fp.h"
+
+tinyengine_status_fp add_fp(const uint16_t size, const float* input1_data,
+			               const float* input2_data, float* output_data) {
+  int i;
+
+  for (i = 0; i < size; ++i) {
+    output_data[i] = input1_data[i] + input2_data[i];
+  }
+  
+  /* Return to application */
+  return STATE_SUCCESS_fp;
+}

TinyEngine/include/tinyengine_function.h

@@ -0,0 +1,32 @@
+/* ----------------------------------------------------------------------
+ * Project: Tiny Training Engine, MCUNetV3
+ * Title:   div_fp.c
+ *
+ * Reference papers:
+ *  - MCUNet: Tiny Deep Learning on IoT Device, NeurIPS 2020
+ *  - MCUNetV2: Memory-Efficient Patch-based Inference for Tiny Deep Learning, NeurIPS 2021
+ *  - MCUNetV3: On-Device Training Under 256KB Memory, NeurIPS 2022
+ * Contact authors:
+ *  - Wei-Chen Wang, [email protected]
+ *  - Wei-Ming Chen, [email protected]
+ *  - Ji Lin, [email protected]
+ *  - Ligeng Zhu, [email protected]
+ *  - Song Han, [email protected]
+ *  - Chuang Gan, [email protected]
+ *
+ * Target ISA:  ARMv7E-M
+ * -------------------------------------------------------------------- */
+
+#include "tinyengine_function_fp.h"
+
+tinyengine_status_fp div_fp(const uint16_t size, const float* input1_data,
+			               const float* input2_data, float* output_data) {
+  int i;
+  
+  for (i = 0; i < size; ++i) {
+    output_data[i] = input1_data[i] / input2_data[i];
+  }
+  
+  /* Return to application */
+  return STATE_SUCCESS_fp;
+}

TinyEngine/include/tinyengine_function_fp.h

@@ -0,0 +1,138 @@
+/* ----------------------------------------------------------------------
+ * Project: Tiny Training Engine, MCUNetV3
+ * Title:   group_pointwise_conv_fp.c
+ *
+ * Reference papers:
+ *  - MCUNet: Tiny Deep Learning on IoT Device, NeurIPS 2020
+ *  - MCUNetV2: Memory-Efficient Patch-based Inference for Tiny Deep Learning, NeurIPS 2021
+ *  - MCUNetV3: On-Device Training Under 256KB Memory, NeurIPS 2022
+ * Contact authors:
+ *  - Wei-Chen Wang, [email protected]
+ *  - Wei-Ming Chen, [email protected]
+ *  - Ji Lin, [email protected]
+ *  - Ligeng Zhu, [email protected]
+ *  - Song Han, [email protected]
+ *  - Chuang Gan, [email protected]
+ *
+ * Target ISA:  ARMv7E-M
+ * -------------------------------------------------------------------- */
+
+#include "tinyengine_function_fp.h"
+#include "tinyengine_function.h"
+#include "nnfunctions_fp.h"
+#define DIM_KER_X (1U)
+#define DIM_KER_Y (1U)
+
+tinyengine_status_fp group_pointwise_conv_fp_in1x1_out1x1_1row10col_uniweight_int8input_inplace(const int8_t* input_data, 
+                 const uint16_t input_height, const uint16_t input_width, const uint16_t input_depth, 
+                 const float* filter_data, const float* bias_data, 
+                 int8_t* output_weight_data, const uint16_t output_height, const uint16_t output_width, const uint16_t output_depth, 
+                 const float output_activation_min, const float output_activation_max,
+                 float* im2col_data, const uint16_t batches, const uint16_t groups,
+                 const float* scales, const float learning_rate) {
+  (void) input_height;
+  (void) input_width;
+
+  int group;
+  int output_depth_per_group = output_depth / groups;
+
+  for (group = 0; group < groups; group++) {
+    int i_ch_out;
+
+    for (i_ch_out = 0; i_ch_out < output_depth_per_group; i_ch_out+=10) {
+      /* Point to the beginning of the im2col buffer where the input is available as a rearranged column */
+      const float input_0 = (float)input_data[group];
+      const float filter[10] = {filter_data[i_ch_out], filter_data[i_ch_out + 1], filter_data[i_ch_out + 2], filter_data[i_ch_out + 3], filter_data[i_ch_out + 4], 
+                      filter_data[i_ch_out + 5], filter_data[i_ch_out + 6], filter_data[i_ch_out + 7], filter_data[i_ch_out + 8], filter_data[i_ch_out + 9]};
+
+      uint16_t col_count_div10 = (output_depth_per_group * DIM_KER_X * DIM_KER_Y) / 10;
+
+      while (col_count_div10--) {
+        // Assume bias_data as NULL
+        float sum[10] = {};
+
+        sum[0] += input_0 * filter[0];
+        sum[1] += input_0 * filter[1];
+        sum[2] += input_0 * filter[2];
+        sum[3] += input_0 * filter[3];
+        sum[4] += input_0 * filter[4];
+        sum[5] += input_0 * filter[5];
+        sum[6] += input_0 * filter[6];
+        sum[7] += input_0 * filter[7];
+        sum[8] += input_0 * filter[8];
+        sum[9] += input_0 * filter[9];
+
+        output_weight_data[i_ch_out + group] -= TN_MIN(TN_MAX(sum[0], output_activation_min), output_activation_max) * scales[i_ch_out] * learning_rate;
+        output_weight_data[(i_ch_out + 1) * groups + group] -= TN_MIN(TN_MAX(sum[1], output_activation_min), output_activation_max) * scales[i_ch_out + 1] * learning_rate;
+        output_weight_data[(i_ch_out + 2) * groups + group] -= TN_MIN(TN_MAX(sum[2], output_activation_min), output_activation_max) * scales[i_ch_out + 2] * learning_rate;
+        output_weight_data[(i_ch_out + 3) * groups + group] -= TN_MIN(TN_MAX(sum[3], output_activation_min), output_activation_max) * scales[i_ch_out + 3] * learning_rate;
+        output_weight_data[(i_ch_out + 4) * groups + group] -= TN_MIN(TN_MAX(sum[4], output_activation_min), output_activation_max) * scales[i_ch_out + 4] * learning_rate;
+        output_weight_data[(i_ch_out + 5) * groups + group] -= TN_MIN(TN_MAX(sum[5], output_activation_min), output_activation_max) * scales[i_ch_out + 5] * learning_rate;
+        output_weight_data[(i_ch_out + 6) * groups + group] -= TN_MIN(TN_MAX(sum[6], output_activation_min), output_activation_max) * scales[i_ch_out + 6] * learning_rate;
+        output_weight_data[(i_ch_out + 7) * groups + group] -= TN_MIN(TN_MAX(sum[7], output_activation_min), output_activation_max) * scales[i_ch_out + 7] * learning_rate;
+        output_weight_data[(i_ch_out + 8) * groups + group] -= TN_MIN(TN_MAX(sum[8], output_activation_min), output_activation_max) * scales[i_ch_out + 8] * learning_rate;
+        output_weight_data[(i_ch_out + 9) * groups + group] -= TN_MIN(TN_MAX(sum[9], output_activation_min), output_activation_max) * scales[i_ch_out + 9] * learning_rate;
+      }
+    }
+  }
+
+  /* Return to application */
+  return STATE_SUCCESS_fp;
+}
+
+tinyengine_status_fp group_pointwise_conv_fp_in1x1_out1x1_1row10col_uniweight_inplace(const float* input_data, 
+                 const uint16_t input_height, const uint16_t input_width, const uint16_t input_depth, 
+                 const float* filter_data, const float* bias_data, 
+                 int8_t* output_weight_data, const uint16_t output_height, const uint16_t output_width, const uint16_t output_depth, 
+                 const float output_activation_min, const float output_activation_max,
+                 float* im2col_data, const uint16_t batches, const uint16_t groups,
+                 const float* scales, const float learning_rate) {
+  (void) input_height;
+  (void) input_width;
+
+  int group;
+  int output_depth_per_group = output_depth / groups;
+
+  for(group = 0; group < groups; group++) {
+    int i_ch_out;
+
+    for (i_ch_out = 0; i_ch_out < output_depth_per_group; i_ch_out+=10) {
+      /* Point to the beginning of the im2col buffer where the input is available as a rearranged column */
+      const float input_0 = input_data[group];
+      const float filter[10] = {filter_data[i_ch_out], filter_data[i_ch_out + 1], filter_data[i_ch_out + 2], filter_data[i_ch_out + 3], filter_data[i_ch_out + 4], 
+                      filter_data[i_ch_out + 5], filter_data[i_ch_out + 6], filter_data[i_ch_out + 7], filter_data[i_ch_out + 8], filter_data[i_ch_out + 9]};
+
+      uint16_t col_count_div10 = (output_depth_per_group * DIM_KER_X * DIM_KER_Y) / 10;
+
+      while (col_count_div10--) {
+        // Assume bias_data as NULL
+        float sum[10] = {};
+
+        sum[0] += input_0 * filter[0];
+        sum[1] += input_0 * filter[1];
+        sum[2] += input_0 * filter[2];
+        sum[3] += input_0 * filter[3];
+        sum[4] += input_0 * filter[4];
+        sum[5] += input_0 * filter[5];
+        sum[6] += input_0 * filter[6];
+        sum[7] += input_0 * filter[7];
+        sum[8] += input_0 * filter[8];
+        sum[9] += input_0 * filter[9];
+
+        output_weight_data[i_ch_out + group] -= TN_MIN(TN_MAX(sum[0], output_activation_min), output_activation_max) * scales[i_ch_out] * learning_rate;
+        output_weight_data[(i_ch_out + 1) * groups + group] -= TN_MIN(TN_MAX(sum[1], output_activation_min), output_activation_max) * scales[i_ch_out + 1] * learning_rate;
+        output_weight_data[(i_ch_out + 2) * groups + group] -= TN_MIN(TN_MAX(sum[2], output_activation_min), output_activation_max) * scales[i_ch_out + 2] * learning_rate;
+        output_weight_data[(i_ch_out + 3) * groups + group] -= TN_MIN(TN_MAX(sum[3], output_activation_min), output_activation_max) * scales[i_ch_out + 3] * learning_rate;
+        output_weight_data[(i_ch_out + 4) * groups + group] -= TN_MIN(TN_MAX(sum[4], output_activation_min), output_activation_max) * scales[i_ch_out + 4] * learning_rate;
+        output_weight_data[(i_ch_out + 5) * groups + group] -= TN_MIN(TN_MAX(sum[5], output_activation_min), output_activation_max) * scales[i_ch_out + 5] * learning_rate;
+        output_weight_data[(i_ch_out + 6) * groups + group] -= TN_MIN(TN_MAX(sum[6], output_activation_min), output_activation_max) * scales[i_ch_out + 6] * learning_rate;
+        output_weight_data[(i_ch_out + 7) * groups + group] -= TN_MIN(TN_MAX(sum[7], output_activation_min), output_activation_max) * scales[i_ch_out + 7] * learning_rate;
+        output_weight_data[(i_ch_out + 8) * groups + group] -= TN_MIN(TN_MAX(sum[8], output_activation_min), output_activation_max) * scales[i_ch_out + 8] * learning_rate;
+        output_weight_data[(i_ch_out + 9) * groups + group] -= TN_MIN(TN_MAX(sum[9], output_activation_min), output_activation_max) * scales[i_ch_out + 9] * learning_rate;
+      }
+    }
+  }
+  
+  /* Return to application */
+  return STATE_SUCCESS_fp;
+}

TinyEngine/src/kernels/fp_backward_op/add_fp.c

@@ -0,0 +1,32 @@
+/* ----------------------------------------------------------------------
+ * Project: Tiny Training Engine, MCUNetV3
+ * Title:   less_fp.c
+ *
+ * Reference papers:
+ *  - MCUNet: Tiny Deep Learning on IoT Device, NeurIPS 2020
+ *  - MCUNetV2: Memory-Efficient Patch-based Inference for Tiny Deep Learning, NeurIPS 2021
+ *  - MCUNetV3: On-Device Training Under 256KB Memory, NeurIPS 2022
+ * Contact authors:
+ *  - Wei-Chen Wang, [email protected]
+ *  - Wei-Ming Chen, [email protected]
+ *  - Ji Lin, [email protected]
+ *  - Ligeng Zhu, [email protected]
+ *  - Song Han, [email protected]
+ *  - Chuang Gan, [email protected]
+ *
+ * Target ISA:  ARMv7E-M
+ * -------------------------------------------------------------------- */
+
+#include "tinyengine_function_fp.h"
+
+tinyengine_status_fp less(const uint16_t size, const float* input1_data,
+			               const float* input2_data, bool* output_data) {
+  int i;
+
+  for (i = 0; i < size; ++i) {
+    output_data[i] = input1_data[i] < input2_data[i];
+  }
+  
+  /* Return to application */
+  return STATE_SUCCESS_fp;
+}

TinyEngine/src/kernels/fp_backward_op/div_fp.c

@@ -0,0 +1,34 @@
+/* ----------------------------------------------------------------------
+ * Name: log_softmax_fp.c
+ * Project: TinyEngine, MCUNetV3
+ * Contact author: Wei-Chen Wang, [email protected]
+ * -------------------------------------------------------------------- */
+
+#include "tinyengine_function_fp.h"
+#include "tinyengine_function.h"
+
+tinyengine_status_fp LogSoftmax(const float* input_data, const uint16_t input_height, const uint16_t input_width, const uint16_t input_depth, 
+                       float* output_data, const uint16_t output_height, const uint16_t output_width, const uint16_t output_depth) {
+  const int outer_size = input_height * input_width;
+  const int depth = TN_MIN(input_depth, output_depth);
+
+  for (int i = 0; i < outer_size; ++i) {
+    float max = FLT_MIN;
+    for (int c = 0; c < depth; ++c) {
+      max = TN_MAX(max, input_data[i * depth + c]);
+    }
+
+    float sum = 0.f;
+    for (int c = 0; c < depth; ++c) {
+      sum += exp(input_data[i * depth + c] - max);
+    }
+
+    const float log_sum = log(sum);
+    for (int c = 0; c < depth; ++c) {
+      output_data[i * depth + c] = input_data[i * depth + c] - max - log_sum;
+    }
+  }
+  
+  /* Return to application */
+  return STATE_SUCCESS_fp;
+}

TinyEngine/src/kernels/fp_backward_op/group_conv_fp_kernel4_stride1_pad0.c

@@ -0,0 +1,32 @@
+/* ----------------------------------------------------------------------
+ * Project: Tiny Training Engine, MCUNetV3
+ * Title:   mul_fp.c
+ *
+ * Reference papers:
+ *  - MCUNet: Tiny Deep Learning on IoT Device, NeurIPS 2020
+ *  - MCUNetV2: Memory-Efficient Patch-based Inference for Tiny Deep Learning, NeurIPS 2021
+ *  - MCUNetV3: On-Device Training Under 256KB Memory, NeurIPS 2022
+ * Contact authors:
+ *  - Wei-Chen Wang, [email protected]
+ *  - Wei-Ming Chen, [email protected]
+ *  - Ji Lin, [email protected]
+ *  - Ligeng Zhu, [email protected]
+ *  - Song Han, [email protected]
+ *  - Chuang Gan, [email protected]
+ *
+ * Target ISA:  ARMv7E-M
+ * -------------------------------------------------------------------- */
+
+#include "tinyengine_function_fp.h"
+
+tinyengine_status_fp mul(const uint16_t size, const float* input1_data,
+			               const float* input2_data, float* output_data) {
+  int i;
+  
+  for (i = 0; i < size; ++i) {
+    output_data[i] = input1_data[i] * input2_data[i];
+  }
+  
+  /* Return to application */
+  return STATE_SUCCESS_fp;
+}

TinyEngine/src/kernels/fp_backward_op/group_conv_fp_kernel8_stride1_pad0.c

@@ -0,0 +1,31 @@
+/* ----------------------------------------------------------------------
+ * Project: Tiny Training Engine, MCUNetV3
+ * Title:   negative_fp.c
+ *
+ * Reference papers:
+ *  - MCUNet: Tiny Deep Learning on IoT Device, NeurIPS 2020
+ *  - MCUNetV2: Memory-Efficient Patch-based Inference for Tiny Deep Learning, NeurIPS 2021
+ *  - MCUNetV3: On-Device Training Under 256KB Memory, NeurIPS 2022
+ * Contact authors:
+ *  - Wei-Chen Wang, [email protected]
+ *  - Wei-Ming Chen, [email protected]
+ *  - Ji Lin, [email protected]
+ *  - Ligeng Zhu, [email protected]
+ *  - Song Han, [email protected]
+ *  - Chuang Gan, [email protected]
+ *
+ * Target ISA:  ARMv7E-M
+ * -------------------------------------------------------------------- */
+
+#include "tinyengine_function_fp.h"
+
+tinyengine_status_fp negative(const uint16_t size, const float* input1_data, bool* output_data) {
+  int i;
+
+  for (i = 0; i < size; ++i) {
+    output_data[i] = input1_data[i] < 0;
+  }
+  
+  /* Return to application */
+  return STATE_SUCCESS_fp;
+}