Merge pull request #37 from emlearn/user-module-build

Extmod: Merge .py module code for trees, setup Unix build and tests in CI

Author: jonnor

.github/workflows/build.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-24.04
     env:
       MPY_DIR: ./micropython
-      MICROPYTHON_BIN: ./micropython/ports/unix/build-standard/micropython
+      MICROPYTHON_BIN: ./micropython/ports/unix/build-nomodules/micropython
     steps:
     - uses: actions/checkout@v4
       with:
@@ -29,8 +29,12 @@ jobs:
       run: pip install -r requirements.txt
     - name: Setup MicroPython X86
       working-directory: micropython
-      run: source tools/ci.sh && ci_unix_32bit_setup && ci_unix_standard_build
-    - name: Run test and build module x64
+      run: |
+        source tools/ci.sh && ci_unix_32bit_setup && ci_unix_standard_build
+        mv ./ports/unix/build-standard/ ./ports/unix/build-nomodules/
+    - name: Build custom firmware with user modules, and tests. Unix/x64
+      run: make check_unix V=1
+    - name: Build .mpy modules and run tests Unix/x64
       run: make check ARCH=x64 V=1
     - name: Setup MicroPython ARM
       working-directory: micropython

Makefile

@@ -8,7 +8,13 @@ VERSION := $(shell git describe --tags --always)
 
 MPY_DIR_ABS = $(abspath $(MPY_DIR)) 
 
+C_MODULES_SRC_PATH = $(abspath ./src)
+MANIFEST_PATH = $(abspath ./src/manifest.py)
+
+PORT=unix
 MODULES_PATH = ./dist/$(ARCH)_$(MPY_ABI_VERSION)
+PORT_DIR = ./dist/ports/$(PORT)
+UNIX_MICROPYTHON = ./dist/ports/unix/micropython
 
 $(MODULES_PATH)/emlearn_trees.mpy:
 	make -C src/emlearn_trees/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) V=1 clean dist
@@ -61,7 +67,20 @@ emlearn_iir_q15.results: $(MODULES_PATH)/emlearn_iir_q15.mpy
 emlearn_arrayutils.results: $(MODULES_PATH)/emlearn_arrayutils.mpy
 	MICROPYPATH=$(MODULES_PATH) $(MICROPYTHON_BIN) tests/test_arrayutils.py
 
-.PHONY: clean
+$(PORT_DIR):
+	mkdir -p $@
+
+$(UNIX_MICROPYTHON): $(PORT_DIR)
+	make -C $(MPY_DIR)/ports/unix V=1 USER_C_MODULES=$(C_MODULES_SRC_PATH) FROZEN_MANIFEST=$(MANIFEST_PATH) CFLAGS_EXTRA='-Wno-unused-function -Wno-unused-function' -j4
+	cp $(MPY_DIR)/ports/unix/build-standard/micropython $@
+
+unix: $(UNIX_MICROPYTHON)
+
+check_unix: $(UNIX_MICROPYTHON)
+	$(UNIX_MICROPYTHON) tests/test_trees.py
+	$(UNIX_MICROPYTHON) tests/test_iir.py
+
+.PHONY: clean unix
 
 clean:
 	make -C src/emlearn_trees/ ARCH=$(ARCH) MPY_DIR=$(MPY_DIR_ABS) V=1 clean

requirements.txt

@@ -1,4 +1,4 @@
-emlearn>=0.21.1
+emlearn>=0.21.2
 scikit-learn>=1.0.0
 ar>=1.0.0
 pyelftools>=0.31

src/emlearn_fft/Makefile

@@ -33,6 +33,6 @@ $(DIST_FILE): $(MOD).mpy $(DIST_DIR)
 include $(MPY_DIR)/py/dynruntime.mk
 
 
-CFLAGS += -I$(EMLEARN_DIR)
+CFLAGS += -I$(EMLEARN_DIR) -Wno-unused-function
 
 dist: $(DIST_FILE)

src/emlearn_iir/Makefile

@@ -32,6 +32,6 @@ $(DIST_FILE): $(MOD).mpy $(DIST_DIR)
 # Include to get the rules for compiling and linking the module
 include $(MPY_DIR)/py/dynruntime.mk
 
-CFLAGS += -I$(EMLEARN_DIR)
+CFLAGS += -I$(EMLEARN_DIR) -Wno-unused-function
 
 dist: $(DIST_FILE)

src/emlearn_iir_q15/Makefile

@@ -41,6 +41,7 @@ LIBGCC_FILENAME = $(shell $(CROSS)gcc $(CFLAGS) -print-libgcc-file-name)
 $(info $(LIBGCC_FILENAME))
 
 CFLAGS += -I$(CMSIS_DSP_DIR)/Include
+CFLAGS += -Wno-unused-function
 
 $(CMSIS_DSP_DIR)/iir_q15.patched:
 	cd $(CMSIS_DSP_DIR) && git apply -v ../df1_q15_disable_dsp.patch

src/emlearn_neighbors/Makefile

@@ -33,6 +33,6 @@ $(DIST_DIR):
 $(DIST_FILE): $(MOD).mpy $(DIST_DIR)
 	cp $< $@
 
-CFLAGS += -I$(EMLEARN_DIR)
+CFLAGS += -I$(EMLEARN_DIR) -Wno-unused-function
 
 dist: $(DIST_FILE)

src/emlearn_trees/Makefile

@@ -19,7 +19,7 @@ DIST_DIR := ../../dist/$(ARCH)_$(MPY_ABI_VERSION)
 MOD = emlearn_trees
 
 # Source files (.c or .py)
-SRC = trees.c trees.py
+SRC = trees.c emlearn_trees.py
 
 # Releases
 DIST_FILE = $(DIST_DIR)/$(MOD).mpy
@@ -32,6 +32,6 @@ $(DIST_FILE): $(MOD).mpy $(DIST_DIR)
 # Include to get the rules for compiling and linking the module
 include $(MPY_DIR)/py/dynruntime.mk
 
-CFLAGS += -I$(EMLEARN_DIR) -DDYNAMIC_RUNTIME=1
+CFLAGS += -I$(EMLEARN_DIR) -Wno-unused-function
 
 dist: $(DIST_FILE)

src/emlearn_trees/trees.py renamed to src/emlearn_trees/emlearn_trees.py

@@ -1,4 +1,11 @@
 
+# When used as external C module, the .py is the top-level import,
+# and we need to merge the native module symbols at import time
+# When used as dynamic native modules (.mpy), .py and native code is merged at build time
+try:
+    from emlearn_trees_c import *
+except ImportError as e:
+    pass
 
 def load_model(builder, f):
 

src/emlearn_trees/trees.c

@@ -332,7 +332,8 @@ const mp_obj_module_t emlearn_trees_cmodule = {
     .globals = (mp_obj_dict_t *)&emlearn_trees_globals,
 };
 
-MP_REGISTER_MODULE(MP_QSTR_emlearn_trees, emlearn_trees_cmodule);
+// External module name is XXX_c to allow .py file to be the entrypoint
+MP_REGISTER_MODULE(MP_QSTR_emlearn_trees_c, emlearn_trees_cmodule);
 
 #endif
 

src/manifest.py

@@ -0,0 +1,5 @@
+
+# Manifest is used to include .py files for external C module build
+# NOTE: this is a different mechanism than
+# Ref https://docs.micropython.org/en/latest/reference/manifest.html
+module("emlearn_trees.py", base_path='./emlearn_trees')