ImageManipV2 optimization and refactor

bindings/python/src/pipeline/node/ImageManipV2Bindings.cpp

@@ -23,13 +23,20 @@ void bind_imagemanipv2(pybind11::module& m, void* pCallstack) {
     ///////////////////////////////////////////////////////////////////////
     ///////////////////////////////////////////////////////////////////////
     ///////////////////////////////////////////////////////////////////////
+    py::enum_<ImageManipV2::PerformanceMode> perfMode(imageManip, "PerformanceMode");
+    perfMode.value("BALANCED", ImageManipV2::PerformanceMode::BALANCED)
+        .value("PERFORMANCE", ImageManipV2::PerformanceMode::PERFORMANCE)
+        .value("LOW_POWER", ImageManipV2::PerformanceMode::LOW_POWER);
 
     // ImageManip Node
     imageManip.def_readonly("inputConfig", &ImageManipV2::inputConfig, DOC(dai, node, ImageManipV2, inputConfig))
         .def_readonly("inputImage", &ImageManipV2::inputImage, DOC(dai, node, ImageManipV2, inputImage))
         .def_readonly("out", &ImageManipV2::out, DOC(dai, node, ImageManipV2, out))
         .def_readonly("initialConfig", &ImageManipV2::initialConfig, DOC(dai, node, ImageManipV2, initialConfig))
         .def("setRunOnHost", &ImageManipV2::setRunOnHost, DOC(dai, node, ImageManipV2, setRunOnHost))
+        .def("setBackendCPU", &ImageManipV2::setBackendCPU, DOC(dai, node, ImageManipV2, setBackendCPU))
+        .def("setBackendHW", &ImageManipV2::setBackendHW, DOC(dai, node, ImageManipV2, setBackendHW))
+        .def("setPerformanceMode", &ImageManipV2::setPerformanceMode, DOC(dai, node, ImageManipV2, setPerformanceMode))
         .def("setNumFramesPool", &ImageManipV2::setNumFramesPool, DOC(dai, node, ImageManipV2, setNumFramesPool))
         .def("setMaxOutputFrameSize", &ImageManipV2::setMaxOutputFrameSize, DOC(dai, node, ImageManipV2, setMaxOutputFrameSize));
 }

cmake/Depthai/DepthaiDeviceRVC4Config.cmake

@@ -4,4 +4,4 @@ set(DEPTHAI_DEVICE_RVC4_MATURITY "snapshot")
 
 # "version if applicable"
 # set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+93f7b75a885aa32f44c5e9f53b74470c49d2b1af")
-set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+e0e0596fead6a181e5dea4628058199c79ca892a")
+set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+94ad947b519d98aa005acb95def4e9d476bab2cc")

cmake/Depthai/DepthaiDeviceSideConfig.cmake

@@ -2,7 +2,7 @@
 set(DEPTHAI_DEVICE_SIDE_MATURITY "snapshot")
 
 # "full commit hash of device side binary"
-set(DEPTHAI_DEVICE_SIDE_COMMIT "4f0b0f3219b701b8e5408eea19648a8d0ec06fa7")
+set(DEPTHAI_DEVICE_SIDE_COMMIT "5c2d9f46aa754bed02044e359469e46571724cf1")
 
 # "version if applicable"
 set(DEPTHAI_DEVICE_SIDE_VERSION "")

examples/python/AprilTags/april_tags_12mp.py

@@ -18,7 +18,7 @@
     # ATM, AprilTags don't contain timestamps, so we can't sync them with frames
     manip = pipeline.create(dai.node.ImageManipV2)
     manip.initialConfig.setOutputSize(PREVIEW_SIZE[0], PREVIEW_SIZE[1], dai.ImageManipConfigV2.ResizeMode.STRETCH)
-    manip.setMaxOutputFrameSize(2004096)
+    manip.setMaxOutputFrameSize(2162688)
     outputCam.link(manip.inputImage)
     frameQ = manip.out.createOutputQueue()
 

include/depthai/pipeline/node/ImageManipV2.hpp

@@ -25,6 +25,9 @@ class ImageManipV2 : public DeviceNodeCRTP<DeviceNode, ImageManipV2, ImageManipP
    public:
     constexpr static const char* NAME = "ImageManipV2";
     using DeviceNodeCRTP::DeviceNodeCRTP;
+    using Backend = ImageManipPropertiesV2::Backend;
+    using PerformanceMode = ImageManipPropertiesV2::PerformanceMode;
+
     ImageManipV2() = default;
     ImageManipV2(std::unique_ptr<Properties> props);
 
@@ -70,6 +73,22 @@ class ImageManipV2 : public DeviceNodeCRTP<DeviceNode, ImageManipV2, ImageManipP
      */
     ImageManipV2& setRunOnHost(bool runOnHost = true);
 
+    /**
+     * Set CPU as backend preference
+     */
+    ImageManipV2& setBackendCPU();
+
+    /**
+     * Set HW as backend preference
+     */
+    ImageManipV2& setBackendHW();
+
+    /**
+     * Set performance mode
+     * @param performanceMode Performance mode
+     */
+    ImageManipV2& setPerformanceMode(PerformanceMode performanceMode);
+
     /**
      * Check if the node is set to run on host
      */
@@ -82,8 +101,8 @@ class ImageManipV2 : public DeviceNodeCRTP<DeviceNode, ImageManipV2, ImageManipP
                      const ImageManipConfigV2& initialConfig,
                      std::shared_ptr<spdlog::async_logger> logger,
                      std::function<size_t(const ImageManipConfigV2&, const ImgFrame&)> build,
-                     std::function<bool(std::shared_ptr<Memory>&, span<uint8_t>)> apply,
-                     std::function<void(const ImageManipConfigV2&, const ImgFrame&, ImgFrame&)> getFrame);
+                     std::function<bool(std::shared_ptr<Memory>&, std::shared_ptr<ImageManipData>)> apply,
+                     std::function<void(const ImgFrame&, ImgFrame&)> getFrame);
 };
 
 }  // namespace node
@@ -97,8 +116,8 @@ void ImageManipV2::loop(N& node,
                         const ImageManipConfigV2& initialConfig,
                         std::shared_ptr<spdlog::async_logger> logger,
                         std::function<size_t(const ImageManipConfigV2&, const ImgFrame&)> build,
-                        std::function<bool(std::shared_ptr<Memory>&, span<uint8_t>)> apply,
-                        std::function<void(const ImageManipConfigV2&, const ImgFrame&, ImgFrame&)> getFrame) {
+                        std::function<bool(std::shared_ptr<Memory>&, std::shared_ptr<ImageManipData>)> apply,
+                        std::function<void(const ImgFrame&, ImgFrame&)> getFrame) {
     using namespace std::chrono;
     auto config = initialConfig;
 
@@ -166,10 +185,10 @@ void ImageManipV2::loop(N& node,
             bool success = true;
             {
                 auto t3 = steady_clock::now();
-                success = apply(inImage->data, outImageData->getData());
+                success = apply(inImage->data, outImageData);
                 auto t4 = steady_clock::now();
 
-                getFrame(config, *inImage, *outImage);
+                getFrame(*inImage, *outImage);
 
                 logger->trace("Build time: {}us, Process time: {}us, Total time: {}us, image manip id: {}",
                               duration_cast<microseconds>(t2 - t1).count(),

include/depthai/properties/ImageManipPropertiesV2.hpp

@@ -10,6 +10,9 @@ namespace dai {
  * Specify properties for ImageManip
  */
 struct ImageManipPropertiesV2 : PropertiesSerializable<Properties, ImageManipPropertiesV2> {
+    enum class Backend : uint8_t { CPU, HW };
+    enum class PerformanceMode : uint8_t { PERFORMANCE, BALANCED, LOW_POWER };
+
     /// Initial configuration for ImageManip node
     ImageManipConfigV2 initialConfig;
 
@@ -18,8 +21,12 @@ struct ImageManipPropertiesV2 : PropertiesSerializable<Properties, ImageManipPro
 
     /// Num frames in output pool
     int numFramesPool = 4;
+
+    /// Using HW backend can cause some unexpected behavior when using multiple ImageManipV2 nodes in series
+    Backend backend = Backend::CPU;
+    PerformanceMode performanceMode = PerformanceMode::PERFORMANCE;
 };
 
-DEPTHAI_SERIALIZE_EXT(ImageManipPropertiesV2, initialConfig, outputFrameSize, numFramesPool);
+DEPTHAI_SERIALIZE_EXT(ImageManipPropertiesV2, initialConfig, outputFrameSize, numFramesPool, backend, performanceMode);
 
 }  // namespace dai