Merge pull request #17 from arekmula/devel

V1.0.0

.gitignore

@@ -127,3 +127,16 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+*idea
+
+# Data stuff
+models
+*dataset
+*.png
+*.zip
+*.avi
+*.mp4
+*.mkv
+*.rar
+*.jpg

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Arkadiusz Mula, Jakub Bielawski, Krzysztof Czerwiński, Maciej Skwara
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1 +1,93 @@
-# tello_drawer
+# tello_drawer
+
+Draw a shape in space with your hand and make the drone replicate this shape!
+The drone stays in the air and watches your hand by the camera.
+The images from the camera are being sent to the PC, where your hand and its pose are detected.
+The detected hand movement is then converted to drone steering commands which makes the drone replicate your movement.
+
+## Steering
+There are two methods to draw the drawing. The method can be chosen
+by providing `finish_drawing` argument while running the script.:  
+- First allows the user to draw by **any hand gesture**. The drawing is finished by showing **two hands at once**.
+- The second allows the user to draw by the **palm gesture**. The drawing is finished by showing a **fist gesture**.
+  Note, that if more than one hand is being shown, the drawing will be made by a right hand.
+
+![alt text](pictures/palm.png "PALM GESTURE")
+![alt text](pictures/fist.png  "FIST GESTURE")
+
+## Performance
+![Alt Text](pictures/performance.gif)
+## Prerequisites
+- Python 3.8
+
+
+## Cloning the repository
+To clone the repository use the following lines
+```
+git clone https://github.com/arekmula/tello_drawer
+cd tello_drawer
+```
+
+Create and activate virtual environment
+```
+python3 -m venv venv
+source venv/bin/activate
+```
+
+Install requirements
+```
+pip install -r requirements.txt
+```
+
+Create `models` directory
+```
+cd src
+mkdir models
+```
+
+Download detector model and its weights along classification model from the **Releases** page and add it to the `models`
+directory.
+
+## Using the repository
+### Tello Drawer
+To run the Tello Drawer use following commands:
+- To run with the the Tello drone:
+```
+python3 main.py 
+```
+While running up the script you can set additional parameters:
+```
+--finish_drawing - Finish drawing sign
+--max_area - The max area [cm] that drone can use to perform the drawing
+--min_length - Minimum length between points, to reduce number of points from detection
+--takeoff_offset - Takeoff move up offset in cm
+```
+
+- You can also run the test drawing with your built-in PC camera or video that you recorded earlier.
+```
+python3 main.py --image_source "built_camera" --camera_index 0
+python3 main.py --image_source "saved_file" --filepath "path/to/file"
+```
+
+
+### Dataset saver
+The dataset saver helps in gathering the data using the Tello drone for further processing.
+It connects to the Tello drone, activates the video stream, and saves each received frame.
+```
+python3 dataset_saver.py --save_img True 
+```
+- Set fps with `--fps` flag
+- Set dataset saving directory with `--save_dir`
+
+
+### Hand detection
+To detect hands on the image we utilized cansik's YOLO hand detector which is available
+[here](https://github.com/cansik/yolo-hand-detection).
+We haven't made any changes to the detector. 
+
+### Hand classification
+We have to split the hand detections into 2 separate classes.
+The fist is responsible for the start/stop signal while the palm is responsible for drawing. To do so we created 
+classifier based on pretrained EfficientNetB0. Date base is available [here](https://www.gti.ssr.upm.es/data/HandGesture_database.html)
+
+TODO: Improve accuraccy of hand classification in real environment.

requirements.txt

@@ -0,0 +1,4 @@
+opencv-python~=4.5.1.48
+tensorflow==2.4.2
+numpy~=1.19.5
+djitellopy2

src/dataset_saver.py

@@ -0,0 +1,51 @@
+import cv2
+import time
+from argparse import ArgumentParser
+from pathlib import Path
+
+from djitellopy import Tello
+
+
+def main(args):
+    tello = Tello()
+    tello.connect()
+    tello.streamon()
+    # Create directory to save images if it doesn't exists
+    if args.save_img:
+        timestamp = str(time.time())
+        save_dir = Path(f"{args.save_dir}") / Path(timestamp)
+        save_dir.mkdir(parents=True, exist_ok=True)
+
+    fps_delay_ms = int((1 / args.fps) * 1000)
+
+    save_frame_count = 0
+    cv2.namedWindow("tello")
+    while True:
+
+        key = cv2.waitKey(fps_delay_ms)
+        if key & 0xFF == ord("q"):
+            # Exit if q pressed
+            cv2.destroyAllWindows()
+            break
+
+        img = tello.get_frame_read().frame
+        if img is not None:
+
+            # Show the image
+            cv2.imshow("tello", img)
+
+            # Save the images
+            if args.save_img:
+                cv2.imwrite(f"{str(save_dir)}/{save_frame_count:07d}.png", img)
+                save_frame_count += 1
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+
+    parser.add_argument("--save_img", metavar="save_img", type=bool, default=False)
+    parser.add_argument("--save_dir", metavar="save_dir", type=str, default="dataset")
+    parser.add_argument("--fps", metavar="fps", type=int, default=30)
+
+    args, _ = parser.parse_known_args()
+    main(args)

src/drone_processing/__init__.py

@@ -0,0 +1,2 @@
+from .processing import DroneProcessor
+from .helpers import distance, convert_to_distance_in_xy

src/drone_processing/helpers.py

@@ -0,0 +1,34 @@
+import numpy as np
+import time
+
+
+def convert_to_distance_in_xy(point_list):
+    """
+    Converts list of points in space to list of distances between previous point in list
+
+    :param point_list:
+    :return:
+    """
+    temp_list = []
+    for i in range(1, len(point_list)):
+        temp_list.append([point_list[i][0] - point_list[i - 1][0], point_list[i][1] - point_list[i - 1][1]])
+    return temp_list
+
+
+def convert_to_euclidean_distance(point_list):
+    temp = []
+    for i in range(1, len(point_list)):
+        temp.append(distance(point_list[i], point_list[i - 1]))
+    return temp
+
+
+def distance(point1, point2):
+    """
+    Calculates distance between 2 points
+    :param point1:
+    :param point2:
+    :return:
+    """
+    return np.sqrt((point2[1] - point1[1]) ** 2 + (point2[0] - point1[0]) ** 2)
+
+

src/drone_processing/processing.py

@@ -0,0 +1,141 @@
+import time
+
+import numpy as np
+from threading import Thread
+
+from djitellopy import Tello
+from .helpers import distance
+
+
+class DroneProcessor:
+    FINISH_DRAWING_HOLD_TIME_S = 2
+
+    def __init__(self, max_area_cm=100, starting_move_up_cm=50, min_length_between_points_cm=5,
+                 max_speed=30):
+        """
+
+        :param max_area_cm: Maximum length that drone can move from starting point in both axes.
+        :param starting_move_up_cm: How many cms should drone go up after the takeoff
+        :param min_length_between_points_cm: Minimum length between points, to reduce number of points from detection.
+        """
+        self.max_area = max_area_cm
+        self.min_length_between_points_cm = min_length_between_points_cm
+        self.max_speed = max_speed
+
+        self.tello = Tello()
+        self.tello.connect()
+        self.tello.streamon()
+        self.tello.takeoff()
+        self.tello.move_up(starting_move_up_cm)
+
+        self.tello_ping_thread = Thread(target=self.ping_tello)
+        self.should_stop_pinging_tello = False
+
+    def get_last_frame(self):
+        return self.tello.get_frame_read().frame
+
+    def finish_drawing(self):
+        """
+        Finish drawing, by stopping drone in air for a while and then force it to land. Disable video streaming.
+
+        :return:
+        """
+        self.tello.send_rc_control(0, 0, 0, 0)
+        time.sleep(self.FINISH_DRAWING_HOLD_TIME_S)
+        self.tello.land()
+        self.tello.streamoff()
+
+    def ping_tello(self):
+        """
+        Ping tello to prevent it from landing while drawing.
+
+        :return:
+        """
+        while True:
+            time.sleep(1)
+            self.tello.send_command_with_return("command")
+            print(f"Battery level: {self.tello.get_battery()}")
+            if self.should_stop_pinging_tello:
+                break
+
+    def start_pinging_tello(self):
+        """
+        Starts thread that pings Tello drone, to prevent it from landing while drawing
+
+        :return:
+        """
+        self.tello_ping_thread.start()
+
+    def stop_pinging_tello(self):
+        """
+        Stop pinging tello to make it available to control
+
+        :return:
+        """
+        self.should_stop_pinging_tello = True
+        self.tello_ping_thread.join()
+
+    def rescale_points(self, point_list, is_int=False):
+        """
+        Rescale points from 0-1 range to range defined by max_area.
+
+        :param point_list:
+        :param is_int:
+        :return: Points rescaled to max_area
+        """
+        temp_list = []
+        for point in point_list:
+            temp_point = []
+            for coordinate in point:
+                coordinate = coordinate * self.max_area
+                if is_int:
+                    temp_point.append(int(coordinate))
+                else:
+                    temp_point.append(coordinate)
+            temp_list.append(temp_point)
+        return temp_list
+
+    def discrete_path(self, rescaled_points):
+        """
+        Reduce number of points in list, so the difference between next points needs to be at least
+        min_length_between_points_cm
+
+        :param rescaled_points:
+        :return:
+        """
+        last_index = -1
+        length = 0
+        while length < self.min_length_between_points_cm:
+            last_index -= 1
+            length = distance(rescaled_points[-1], rescaled_points[last_index])
+
+        last_index = len(rescaled_points) + last_index
+        discrete_path = [rescaled_points[0]]
+        actual_point = 0
+        for ind, point in enumerate(rescaled_points):
+            if ind > last_index:
+                discrete_path.append(rescaled_points[-1])
+                break
+            if distance(rescaled_points[actual_point], point) > 5:
+                discrete_path.append(point)
+                actual_point = ind
+
+        return discrete_path
+
+    def reproduce_discrete_path_by_drone(self, discrete_path):
+        """
+        Converts discrete path to velocity commands and sends them to drone, so the drone reproduce the path
+
+        :param discrete_path: list of [x, y] points, which represents distance in each axis between previous point in
+         list
+        :return:
+        """
+        for current_point in discrete_path:
+            ang = np.arctan2(current_point[0], current_point[1])
+            x_speed = int(np.sin(ang) * self.max_speed)
+            y_speed = -int(np.cos(ang) * self.max_speed)
+
+            euclidean_distance = (current_point[0] ** 2 + current_point[1] ** 2) ** 0.5
+            move_time = euclidean_distance / self.max_speed
+            self.tello.send_rc_control(x_speed, 0, y_speed, 0)
+            time.sleep(move_time)

src/image_processing/__init__.py

@@ -0,0 +1 @@
+from .processing import ImageProcessor