import cv2 import mediapipe as mp import time import serial import numpy as np import handTrackingModule as htm #needs to be in same folder as project import threading arduino = serial.Serial(port='COM7', baudrate=9600, timeout=.1) #create serial connection to arduino hand_cmd = int(3) #byte array to hold hand position information array_buffer = int(3) def read(): try: byte_array = [None] *3 byte_array = arduino.readline() except:pass return byte_array def send_data(id, val): val = int(val) id = int(id) if((id!=1)): val = abs(val-255) data_temp = [255, id, val] if((val <=256 and val>0)): data=bytes(data_temp) # Convert the data array to bytes byte_data = bytes(data) # Send the byte data to Arduino arduino.write(byte_data) # Print the values being sent on the terminal print(f"id: {id} val: {val}") # Wait for Arduino to process the data time.sleep(0.01) def translation(id,max,min,val): """translates the node data into values between 0-255 Args: max (int): max value recorded at that node min (_type_): min value at that node val (_type_): current value at that node Returns: int: translated value """ translated = str("9999") try: translated= str(int(((255)/(max-min))*val -(((255)/(max-min))*min))) if id == 1: translated = 255-translated if translated < 100: translated = "0" + translated except: pass return translated def main(): #variables min_length_thumb, min_length_index, min_length_middle, min_length_ring, min_length_pinky = 999,999,999,999,999 max_length_thumb, max_length_index, max_length_middle, max_length_ring, max_length_pinky = 0,0,0,0,0 translated_length_thumb, translated_length_index, translated_length_middle, translated_length_ring, translated_length_pinky = 0,0,0,0,0 pTime, cTime = 0,0 WRIST_NODE,THUMB_NODE,INDEX_NODE,MIDDLE_NODE,RING_NODE,PINKY_NODE = 0,4,8,12,16,20 THUMB,INDEX,MIDDLE,RING,PINKY = 1,2,3,4,5 cap = cv2.VideoCapture(0) #0 for built in camera, 1 for external camera detector = htm.handDetector() #detects hand while True: while True: try: success,img = cap.read() #reads image data. img = detector.findHands(img) #detects if there are hands present lmList = detector.getPositions(img) #gets the postions and nodes of hand pTime = detector.showFPS(img,pTime = pTime) #shows the fps on screen break except: pass #gets the length between point of finger and WRIST_NODE length_thumb = detector.get_length(img, WRIST_NODE,THUMB_NODE,False) length_index = detector.get_length(img, WRIST_NODE,INDEX_NODE,False) length_middle = detector.get_length(img, WRIST_NODE,MIDDLE_NODE,False) length_ring = detector.get_length(img, WRIST_NODE,RING_NODE,False) length_pinky = detector.get_length(img, WRIST_NODE,PINKY_NODE,False) #for calibration if length_thumb > max_length_thumb: max_length_thumb = length_thumb if length_index > max_length_index: max_length_index = length_index if length_middle > max_length_middle: max_length_middle = length_middle if length_ring > max_length_ring: max_length_ring = length_ring if length_pinky > max_length_pinky: max_length_pinky = length_pinky if length_thumb < min_length_thumb and length_thumb!=0: min_length_thumb = length_thumb if length_index < min_length_index and length_index!=0: min_length_index = length_index if length_middle < min_length_middle and length_middle!=0: min_length_middle = length_middle if length_ring < min_length_ring and length_ring!=0: min_length_ring = length_ring if length_pinky < min_length_pinky and length_pinky!=0: min_length_pinky = length_pinky #translates given image input into usable data for external device translated_length_thumb = translation(THUMB,max_length_thumb, min_length_thumb, length_thumb) translated_length_index = translation(INDEX,max_length_index, min_length_index, length_index) translated_length_middle = translation(MIDDLE,max_length_middle, min_length_middle, length_middle) translated_length_ring = translation(RING,max_length_ring, min_length_ring, length_ring) translated_length_pinky = translation(PINKY,max_length_pinky, min_length_pinky, length_pinky) #print for debugging #print(F"THUMB_NODE: {length_thumb} , INDEX_NODE: {length_index} , MIDDLE_NODE: {length_middle} , RING_NODE: {length_ring} , PINKY_NODE: {length_pinky}") #print(F"MAX THUMB_NODE: {max_length_thumb} , MAX INDEX_NODE: {max_length_index} , MAX MIDDLE_NODE: {max_length_middle} , MAX RING_NODE: {max_length_ring} , MAX PINKY_NODE: {max_length_pinky}") #print(F"min THUMB_NODE: {min_length_thumb} , min INDEX_NODE: {min_length_index} , min MIDDLE_NODE: {min_length_middle} , min RING_NODE: {min_length_ring} , min PINKY_NODE: {min_length_pinky}") #print(F"Translated THUMB_NODE: {translated_length_thumb} , Translated INDEX_NODE: {translated_length_index} , Translated MIDDLE_NODE: {translated_length_middle} , Translated RING_NODE: {translated_length_ring} , Translated PINKY_NODE: {translated_length_pinky}") #print() #for sending data to arduino send_data(THUMB, translated_length_thumb) #for reading arduino information send_data(INDEX, translated_length_index) #for reading arduino information send_data(MIDDLE, translated_length_middle) #for reading arduino information send_data(RING, translated_length_ring) #for sending data to arduino send_data(PINKY, translated_length_pinky) cv2.imshow("Video",img) if cv2.waitKey(1) == ord('q'): break cap.release() cv2.destroyAllWindows() if __name__ == "__main__": main()