From 2ddcf0003831fd42784b4d1ccf88383d239bb664 Mon Sep 17 00:00:00 2001 From: rodolpheh Date: Wed, 14 Feb 2024 20:33:30 +0000 Subject: [PATCH 1/2] Add images in visualiser --- examples/visualiser.py | 49 ++++++++++++++++++++++++++- leapc-python-api/src/leap/__init__.py | 2 +- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/examples/visualiser.py b/examples/visualiser.py index 53d31d18..36a11db5 100644 --- a/examples/visualiser.py +++ b/examples/visualiser.py @@ -2,12 +2,23 @@ import numpy as np import cv2 +from cffi import FFI +from enum import Enum + +class Side(Enum): + Left = 0 + Right = 1 + _TRACKING_MODES = { leap.TrackingMode.Desktop: "Desktop", leap.TrackingMode.HMD: "HMD", leap.TrackingMode.ScreenTop: "ScreenTop", } +_CAMERA_SIDE = { + Side.Left: "Left", + Side.Right: "Right" +} class Canvas: def __init__(self): @@ -19,6 +30,10 @@ def __init__(self): self.output_image = np.zeros((self.screen_size[0], self.screen_size[1], 3), np.uint8) self.tracking_mode = None + # For images management + self.image = np.zeros((self.screen_size[0], self.screen_size[1], 3), np.uint8) + self.displayed_side = Side.Left + def set_tracking_mode(self, tracking_mode): self.tracking_mode = tracking_mode @@ -26,6 +41,9 @@ def toggle_hands_format(self): self.hands_format = "Dots" if self.hands_format == "Skeleton" else "Skeleton" print(f"Set hands format to {self.hands_format}") + def switch_view(self): + self.displayed_side = Side.Right if self.displayed_side is Side.Left else Side.Left + def get_joint_position(self, bone): if bone: return int(bone.x + (self.screen_size[1] / 2)), int(bone.z + (self.screen_size[0] / 2)) @@ -34,7 +52,7 @@ def get_joint_position(self, bone): def render_hands(self, event): # Clear the previous image - self.output_image[:, :] = 0 + self.output_image[:, :] = cv2.resize(self.image, dsize=(self.screen_size[1], self.screen_size[0]), interpolation=cv2.INTER_CUBIC) cv2.putText( self.output_image, @@ -46,6 +64,16 @@ def render_hands(self, event): 1, ) + cv2.putText( + self.output_image, + f"Camera: {_CAMERA_SIDE[self.displayed_side]}", + (10, self.screen_size[0] - 30), + cv2.FONT_HERSHEY_SIMPLEX, + 0.5, + self.font_colour, + 1 + ) + if len(event.hands) == 0: return @@ -111,6 +139,7 @@ def render_hands(self, event): class TrackingListener(leap.Listener): def __init__(self, canvas): self.canvas = canvas + self.ffi = FFI() def on_connection_event(self, event): pass @@ -131,6 +160,20 @@ def on_device_event(self, event): def on_tracking_event(self, event): self.canvas.render_hands(event) + def on_image_event(self, event): + index = 0 if self.canvas.displayed_side is Side.Left else 1 + + properties = event.image[index].c_data.properties + offset = event.image[index].c_data.offset + height = properties.height + width = properties.width + + # I'm sure it can be improved. Not feeling comfy managing pointers in Python + image_ptr = self.ffi.cast("int *", event.image[index].c_data.data + offset) + ffi_buffer = self.ffi.buffer(image_ptr, height * width) + np_buffer = np.flip(np.frombuffer(ffi_buffer, dtype=np.uint8).reshape((height, width)), 1) + self.canvas.image = np.repeat(np_buffer[:, :, np.newaxis], 3, axis=2) + def main(): canvas = Canvas() @@ -143,6 +186,7 @@ def main(): print(" s: Select ScreenTop tracking mode") print(" d: Select Desktop tracking mode") print(" f: Toggle hands format between Skeleton/Dots") + print(" g: Switch between right and left view") tracking_listener = TrackingListener(canvas) @@ -154,6 +198,7 @@ def main(): with connection.open(): connection.set_tracking_mode(leap.TrackingMode.Desktop) canvas.set_tracking_mode(leap.TrackingMode.Desktop) + connection.set_policy_flags([leap.PolicyFlag.Images], []) while running: cv2.imshow(canvas.name, canvas.output_image) @@ -170,6 +215,8 @@ def main(): connection.set_tracking_mode(leap.TrackingMode.Desktop) elif key == ord("f"): canvas.toggle_hands_format() + elif key == ord("g"): + canvas.switch_view() if __name__ == "__main__": diff --git a/leapc-python-api/src/leap/__init__.py b/leapc-python-api/src/leap/__init__.py index 58031cd5..a550d60a 100644 --- a/leapc-python-api/src/leap/__init__.py +++ b/leapc-python-api/src/leap/__init__.py @@ -92,7 +92,7 @@ def check_required_files(cffi_dir): get_extrinsic_matrix, ) from .connection import Connection -from .enums import EventType, TrackingMode, HandType +from .enums import EventType, TrackingMode, HandType, FlagPolicy from .event_listener import Listener from .exceptions import LeapError from .recording import Recording, Recorder From 102ecae8f490e32aa41f9d3a840ccd739478d9a9 Mon Sep 17 00:00:00 2001 From: rodolpheh Date: Thu, 15 Feb 2024 18:11:41 +0000 Subject: [PATCH 2/2] Fix typo --- leapc-python-api/src/leap/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/leapc-python-api/src/leap/__init__.py b/leapc-python-api/src/leap/__init__.py index a550d60a..ce9a26d6 100644 --- a/leapc-python-api/src/leap/__init__.py +++ b/leapc-python-api/src/leap/__init__.py @@ -92,7 +92,7 @@ def check_required_files(cffi_dir): get_extrinsic_matrix, ) from .connection import Connection -from .enums import EventType, TrackingMode, HandType, FlagPolicy +from .enums import EventType, TrackingMode, HandType, PolicyFlag from .event_listener import Listener from .exceptions import LeapError from .recording import Recording, Recorder