feat(clip): guess if warping has been failed by using similarity measure

pyproject.toml

@@ -34,6 +34,7 @@ torch = "^2.1.2"
 torchvision = "^0.16.2"
 flask-babel = "^4.0.0"
 my-ultralytics-4bands = { git = "https://github.com/itisacloud/ultralytics_multiband_support.git", rev = "ef61cf9870755ae8b21b03253237d15f5856e1a6" }
+scikit-image = "^0.24.0"
 
 [tool.poetry.group.dev.dependencies]
 # Versions are fixed to match versions used by pre-commit

sketch_map_tool/upload_processing/clip.py

@@ -3,9 +3,12 @@
 based on a matching template
 """
 
+import logging
+
 import cv2
 import numpy as np
 from numpy.typing import NDArray
+from skimage.metrics import structural_similarity as ssim
 
 
 def clip(photo: NDArray, template: NDArray) -> NDArray:
@@ -65,12 +68,14 @@ def clip(photo: NDArray, template: NDArray) -> NDArray:
     # Get dimensions of template
     height, width, _ = template.shape
 
-    # TODO: fix issue demonstrated by the test case
-    # `test_failed_georeferencing` in `test_clip.py`,
-    # then check success before return clipped img.
-    # succeed = filter_matrix(homography_matrix)
+    warped = cv2.warpPerspective(photo, homography_matrix, (width, height))
 
-    return cv2.warpPerspective(photo, homography_matrix, (width, height))
+    # Check if wrapping lead to skewed results.
+    # Various approaches haven been tried (SSIM, pHash, DL embeddings &
+    # analyzing the homography matrix)
+    if ssim(cv2.cvtColor(warped, cv2.COLOR_BGR2GRAY), template_gray) < 0.3:
+        logging.warning("Georeferencing might have failed.")
+    return warped
 
 
 def limit_keypoints(

sketch_map_tool/upload_processing/detect_markings.py

@@ -1,3 +1,5 @@
+import logging
+
 import cv2
 import numpy as np
 from numpy.typing import NDArray
@@ -41,6 +43,8 @@ def detect_markings(
         yolo_cls,
         sam_predictor,
     )
+    if len(colors) == 0:
+        logging.warning("No markings detected.")
     colors = [int(c) + 1 for c in colors]  # +1 because 0 is background
     processed_markings = post_process(masks, bboxes, colors)
     return processed_markings

tests/integration/upload_processing/test_detect_markings.py

@@ -1,3 +1,5 @@
+import logging
+
 import numpy as np
 import pytest
 from PIL import Image, ImageEnhance
@@ -77,3 +79,19 @@ def test_detect_markings(
     for m in markings:
         img = Image.fromarray(m)
         ImageEnhance.Contrast(img).enhance(10).show()
+
+
+def test_detectec_markings_failure(yolo_osm_cls, yolo_osm_obj, sam_predictor, caplog):
+    # Empty map and template should not contain any markings
+    empty_map = np.zeros((1024, 1024, 3), dtype=np.uint8)
+    empty_template = np.zeros((1024, 1024, 3), dtype=np.uint8)
+
+    with caplog.at_level(logging.WARNING):
+        detect_markings(
+            empty_map,
+            empty_template,
+            yolo_osm_obj,
+            yolo_osm_cls,
+            sam_predictor,
+        )
+        assert "No markings detected." in caplog.text

tests/unit/upload_processing/test_clip.py

@@ -108,7 +108,7 @@ def test_clip_failure(
     """Clip produces a distorted image."""
     with caplog.at_level(logging.WARNING):
         clip(photo_of_sketch_map, map_frame_2)
-        assert "Something bad happened!" in caplog.text
+        assert "Georeferencing might have failed." in caplog.text
     # cv2.imshow("image", result)
     # cv2.waitKey(0)
     # cv2.destroyAllWindows()