распознавание пальцев рук

Co-authored-by: Cursor <cursoragent@cursor.com>

hand_landmarker.ipynb

@@ -0,0 +1,288 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "h2q27gKz1H20"
+      },
+      "source": [
+        "##### Copyright 2023 The MediaPipe Authors. All Rights Reserved."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TUfAcER1oUS6"
+      },
+      "outputs": [],
+      "source": [
+        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "# https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "L_cQX8dWu4Dv"
+      },
+      "source": [
+        "# Hand Landmarks Detection with MediaPipe Tasks\n",
+        "\n",
+        "This notebook shows you how to use MediaPipe Tasks Python API to detect hand landmarks from images."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "O6PN9FvIx614"
+      },
+      "source": [
+        "## Preparation\n",
+        "\n",
+        "Let's start with installing MediaPipe."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "gxbHBsF-8Y_l"
+      },
+      "outputs": [],
+      "source": [
+        "!pip install -q mediapipe"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "a49D7h4TVmru"
+      },
+      "source": [
+        "Then download an off-the-shelf model bundle. Check out the [MediaPipe documentation](https://developers.google.com/mediapipe/solutions/vision/hand_landmarker#models) for more information about this model bundle."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "OMjuVQiDYJKF"
+      },
+      "outputs": [],
+      "source": [
+        "!wget -q https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "YYKAJ5nDU8-I"
+      },
+      "source": [
+        "## Visualization utilities"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "s3E6NFV-00Qt"
+      },
+      "outputs": [],
+      "source": [
+        "#@markdown We implemented some functions to visualize the hand landmark detection results. <br/> Run the following cell to activate the functions.\n",
+        "import mediapipe as mp\n",
+        "import numpy as np\n",
+        "\n",
+        "mp_hands = mp.tasks.vision.HandLandmarksConnections\n",
+        "mp_drawing = mp.tasks.vision.drawing_utils\n",
+        "mp_drawing_styles = mp.tasks.vision.drawing_styles\n",
+        "\n",
+        "MARGIN = 10  # pixels\n",
+        "FONT_SIZE = 1\n",
+        "FONT_THICKNESS = 1\n",
+        "HANDEDNESS_TEXT_COLOR = (88, 205, 54) # vibrant green\n",
+        "\n",
+        "def draw_landmarks_on_image(rgb_image, detection_result):\n",
+        "  hand_landmarks_list = detection_result.hand_landmarks\n",
+        "  handedness_list = detection_result.handedness\n",
+        "  annotated_image = np.copy(rgb_image)\n",
+        "\n",
+        "  # Loop through the detected hands to visualize.\n",
+        "  for idx in range(len(hand_landmarks_list)):\n",
+        "    hand_landmarks = hand_landmarks_list[idx]\n",
+        "    handedness = handedness_list[idx]\n",
+        "\n",
+        "    # Draw the hand landmarks.\n",
+        "    mp_drawing.draw_landmarks(\n",
+        "      annotated_image,\n",
+        "      hand_landmarks,\n",
+        "      mp_hands.HAND_CONNECTIONS,\n",
+        "      mp_drawing_styles.get_default_hand_landmarks_style(),\n",
+        "      mp_drawing_styles.get_default_hand_connections_style())\n",
+        "\n",
+        "    # Get the top left corner of the detected hand's bounding box.\n",
+        "    height, width, _ = annotated_image.shape\n",
+        "    x_coordinates = [landmark.x for landmark in hand_landmarks]\n",
+        "    y_coordinates = [landmark.y for landmark in hand_landmarks]\n",
+        "    text_x = int(min(x_coordinates) * width)\n",
+        "    text_y = int(min(y_coordinates) * height) - MARGIN\n",
+        "\n",
+        "    # Draw handedness (left or right hand) on the image.\n",
+        "    cv2.putText(annotated_image, f\"{handedness[0].category_name}\",\n",
+        "                (text_x, text_y), cv2.FONT_HERSHEY_DUPLEX,\n",
+        "                FONT_SIZE, HANDEDNESS_TEXT_COLOR, FONT_THICKNESS, cv2.LINE_AA)\n",
+        "\n",
+        "  return annotated_image"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "83PEJNp9yPBU"
+      },
+      "source": [
+        "## Download test image\n",
+        "\n",
+        "Let's grab a test image that we'll use later. The image is from [Unsplash](https://unsplash.com/photos/mt2fyrdXxzk)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tzXuqyIBlXer"
+      },
+      "outputs": [],
+      "source": [
+        "!wget -q -O image.jpg https://storage.googleapis.com/mediapipe-tasks/hand_landmarker/woman_hands.jpg\n",
+        "\n",
+        "import cv2\n",
+        "from google.colab.patches import cv2_imshow\n",
+        "\n",
+        "img = cv2.imread(\"image.jpg\")\n",
+        "cv2_imshow(img)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "u-skLwMBmMN_"
+      },
+      "source": [
+        "Optionally, you can upload your own image. If you want to do so, uncomment and run the cell below."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "etBjSdwImQPw"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import files\n",
+        "# uploaded = files.upload()\n",
+        "\n",
+        "# for filename in uploaded:\n",
+        "#   content = uploaded[filename]\n",
+        "#   with open(filename, 'wb') as f:\n",
+        "#     f.write(content)\n",
+        "\n",
+        "# if len(uploaded.keys()):\n",
+        "#   IMAGE_FILE = next(iter(uploaded))\n",
+        "#   print('Uploaded file:', IMAGE_FILE)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "Iy4r2_ePylIa"
+      },
+      "source": [
+        "## Running inference and visualizing the results\n",
+        "\n",
+        "Here are the steps to run hand landmark detection using MediaPipe.\n",
+        "\n",
+        "Check out the [MediaPipe documentation](https://developers.google.com/mediapipe/solutions/vision/hand_landmarker/python) to learn more about configuration options that this solution supports.\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_JVO3rvPD4RN"
+      },
+      "outputs": [],
+      "source": [
+        "# STEP 1: Import the necessary modules.\n",
+        "import mediapipe as mp\n",
+        "from mediapipe.tasks import python\n",
+        "from mediapipe.tasks.python import vision\n",
+        "\n",
+        "# STEP 2: Create an HandLandmarker object.\n",
+        "base_options = python.BaseOptions(model_asset_path='hand_landmarker.task')\n",
+        "options = vision.HandLandmarkerOptions(base_options=base_options,\n",
+        "                                       num_hands=2)\n",
+        "detector = vision.HandLandmarker.create_from_options(options)\n",
+        "\n",
+        "# STEP 3: Load the input image.\n",
+        "image = mp.Image.create_from_file(\"image.jpg\")\n",
+        "\n",
+        "# STEP 4: Detect hand landmarks from the input image.\n",
+        "detection_result = detector.detect(image)\n",
+        "\n",
+        "# STEP 5: Process the classification result. In this case, visualize it.\n",
+        "annotated_image = draw_landmarks_on_image(image.numpy_view(), detection_result)\n",
+        "cv2_imshow(cv2.cvtColor(annotated_image, cv2.COLOR_RGB2BGR))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SE6_sPCXaX3g"
+      },
+      "outputs": [],
+      "source": []
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [
+        "h2q27gKz1H20"
+      ],
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3 (ipykernel)",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.8.10"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

handle_faces.py

@@ -0,0 +1,328 @@
+import cv2
+import mediapipe as mp
+import math
+
+print("Запуск MediaPipe детектора рук (новый API)...")
+print(f"Версия MediaPipe: {mp.__version__}")
+
+if not hasattr(mp, 'tasks'):
+    print("ОШИБКА: mediapipe.tasks не найден!")
+    exit()
+
+BaseOptions = mp.tasks.BaseOptions
+HandLandmarker = mp.tasks.vision.HandLandmarker
+HandLandmarkerOptions = mp.tasks.vision.HandLandmarkerOptions
+VisionRunningMode = mp.tasks.vision.RunningMode
+
+HAND_CONNECTIONS = [
+    (0, 1), (1, 2), (2, 3), (3, 4),
+    (0, 5), (5, 6), (6, 7), (7, 8),
+    (0, 9), (9, 10), (10, 11), (11, 12),
+    (0, 13), (13, 14), (14, 15), (15, 16),
+    (0, 17), (17, 18), (18, 19), (19, 20),
+    (5, 9), (9, 13), (13, 17)
+]
+
+options = HandLandmarkerOptions(
+    base_options=BaseOptions(model_asset_path='hand_landmarker.task'),
+    running_mode=VisionRunningMode.IMAGE,
+    num_hands=2,
+    min_hand_detection_confidence=0.7,
+    min_hand_presence_confidence=0.7,
+    min_tracking_confidence=0.5
+)
+
+hand_landmarker = HandLandmarker.create_from_options(options)
+
+cap = cv2.VideoCapture(0)
+if not cap.isOpened():
+    print("Ошибка: не удалось открыть камеру")
+    exit()
+
+print("Нажмите 'q' для выхода")
+
+hand_colors = [
+    (0, 255, 0),
+    (255, 0, 0),
+    (0, 255, 255),
+    (255, 0, 255),
+]
+
+def recognize_gesture(hand_landmarks, handedness, fingers_up):
+    """
+    Распознаёт жесты на основе положения пальцев.
+    
+    hand_landmarks: список из 21 точки (x, y)
+    handedness: 'Left' или 'Right'
+    fingers_up: список названий поднятых пальцев
+    
+    Возвращает: название жеста или None
+    """
+    
+    # === Жест "ОК" ===
+    # Кончики большого (4) и указательного (8) пальцев соединены
+    thumb_tip = hand_landmarks[4]
+    index_tip = hand_landmarks[8]
+    
+    distance_thumb_index = math.sqrt(
+        (thumb_tip[0] - index_tip[0])**2 + 
+        (thumb_tip[1] - index_tip[1])**2
+    )
+    
+    # Порог расстояния (в пикселях) - зависит от размера руки в кадре
+    pinch_threshold = 50
+    
+    if distance_thumb_index < pinch_threshold:
+        # Проверяем, что остальные пальцы не обязательно подняты
+        return "OK"
+    
+    # === Жест "Большой палец вверх" ===
+    # Только большой палец поднят, остальные согнуты
+    if fingers_up == ['Большой']:
+        # Дополнительно проверяем, что большой палец направлен вверх
+        thumb_tip = hand_landmarks[4]
+        thumb_ip = hand_landmarks[3]
+        if thumb_tip[1] > thumb_ip[1]:  # Кончик выше сустава
+            return "THUMBS_UP"
+    
+    # === Жест "Победа/Мир" (V sign) ===
+    # Указательный и средний подняты, остальные согнуты
+    if set(fingers_up) == {'Указательный', 'Средний'}:
+        return "PEACE"
+    
+    # === Жест "Стоп/Пять" ===
+    # Все 5 пальцев подняты
+    if len(fingers_up) == 5:
+        return "STOP"
+    
+    # === Жест "Рок/Коза" ===
+    # Указательный и мизинец подняты, остальные согнуты
+    if set(fingers_up) == {'Указательный', 'Мизинец'}:
+        return "ROCK"
+    
+    # === Жест "Указание" ===
+    # Только указательный палец поднят
+    if fingers_up == ['Указательный']:
+        return "POINT"
+    
+    # === Жест "Кулак" ===
+    # Ни один палец не поднят
+    if len(fingers_up) == 0:
+        return "FIST"
+    
+    # === Жест "Три" ===
+    # Указательный, средний и безымянный подняты
+    if set(fingers_up) == {'Указательный', 'Средний', 'Безымянный'}:
+        return "THREE"
+    
+    # === Жест "Четыре" ===
+    # Четыре пальца подняты (все кроме большого)
+    if set(fingers_up) == {'Указательный', 'Средний', 'Безымянный', 'Мизинец'}:
+        return "FOUR"
+    
+    # Жест не распознан
+    return None
+
+def count_fingers(hand_landmarks, handedness):
+    """Подсчитывает количество поднятых пальцев"""
+    fingers_up = []
+    
+    # Большой палец
+    thumb_tip = hand_landmarks[4]
+    thumb_ip = hand_landmarks[3]
+    if handedness == 'Right':
+        if  thumb_ip[0] > thumb_tip[0] :
+            fingers_up.append('Большой')
+    else:
+        if thumb_ip[0] > thumb_tip[0] :
+            fingers_up.append('Большой')
+    
+    # Указательный
+    if hand_landmarks[8][1] < hand_landmarks[6][1]:
+        fingers_up.append('Указательный')
+    
+    # Средний
+    if hand_landmarks[12][1] < hand_landmarks[10][1]:
+        fingers_up.append('Средний')
+    
+    # Безымянный
+    if hand_landmarks[16][1] < hand_landmarks[14][1]:
+        fingers_up.append('Безымянный')
+    
+    # Мизинец
+    if hand_landmarks[20][1] < hand_landmarks[18][1]:
+        fingers_up.append('Мизинец')
+    
+    return len(fingers_up), fingers_up
+
+
+def detect_palm_orientation(hand_landmarks, handedness):
+    """
+    Определяет ориентацию ладони: к камере или тыльной стороной.
+    
+    Логика метода:
+    - Берём большой палец (точка 4) и мизинец (точка 17)
+    - Сравниваем их x-координаты
+    - Для ПРАВОЙ руки: если большой палец ЛЕВЕЕ мизинца (x меньше) → ладонь к камере
+    - Для ЛЕВОЙ руки: если большой палец ПРАВЕЕ мизинца (x больше) → ладонь к камере
+    
+    Почему так:
+    - Когда правая рука показывает ладонью к камере, большой палец оказывается слева
+    - Когда правая рука показывает тыльной стороной, большой палец оказывается справа
+    - Для левой руки всё зеркально
+    
+    Возвращает: 'palm' (ладонь к камере) или 'back' (тыльная сторона)
+    """
+    thumb_tip = hand_landmarks[4]   # Кончик большого пальца
+    pinky_mcp = hand_landmarks[17]  # Основание мизинца
+    
+    thumb_x = thumb_tip[0]
+    pinky_x = pinky_mcp[0]
+    
+    if handedness == 'Right':
+        # Правая рука: большой палец слева от мизинца → ладонь к камере
+        if thumb_x < pinky_x:
+            return 'back'
+        else:
+            return 'palm'
+    else:
+        # Левая рука: большой палец справа от мизинца → ладонь к камере
+        if thumb_x > pinky_x:
+            return 'back'
+        else:
+            return 'palm'
+
+
+frame_counter = 0
+
+while True:
+    ret, frame = cap.read()
+    if not ret:
+        break
+    
+    rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+    mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb_frame)
+    
+    result = hand_landmarker.detect(mp_image)
+    
+    h, w, _ = frame.shape
+    total_fingers = 0
+    
+    if result.hand_landmarks:
+        for hand_idx, hand_landmarks in enumerate(result.hand_landmarks):
+            color = hand_colors[hand_idx % len(hand_colors)]
+            handedness = result.handedness[hand_idx][0].category_name
+            
+            points = []
+            for lm in hand_landmarks:
+                px = int(lm.x * w)
+                py = int(lm.y * h)
+                points.append((px, py))
+            
+            # Подсчёт пальцев
+            fingers_count, fingers_names = count_fingers(points, handedness)
+            total_fingers += fingers_count
+            
+            # Определение ориентации ладони
+            orientation = detect_palm_orientation(points, handedness)
+
+            # === РАСПОЗНАВАНИЕ ЖЕСТА ===
+            gesture = recognize_gesture(points, handedness, fingers_names)
+
+            # Рисуем соединения
+            for connection in HAND_CONNECTIONS:
+                start_idx, end_idx = connection
+                if start_idx < len(points) and end_idx < len(points):
+                    pt1 = points[start_idx]
+                    pt2 = points[end_idx]
+                    # Если ладонь к камере - рисуем жирнее и ярче
+                    thickness = 3 if orientation == 'palm' else 2
+                    cv2.line(frame, pt1, pt2, color, thickness)
+            
+            # Рисуем точки
+            for i, (px, py) in enumerate(points):
+                if i == 0:
+                    cv2.circle(frame, (px, py), 8, color, -1)
+                    cv2.circle(frame, (px, py), 8, (255, 255, 255), 2)
+                elif i in [4, 8, 12, 16, 20]:
+                    cv2.circle(frame, (px, py), 7, color, -1)
+                    cv2.circle(frame, (px, py), 7, (255, 255, 255), 2)
+                else:
+                    cv2.circle(frame, (px, py), 5, color, -1)
+            
+            # Подписи
+            if points:
+                wrist_x, wrist_y = points[0]
+                
+                # Название руки
+                cv2.putText(frame, f"Hand {hand_idx+1}: {handedness}", 
+                           (wrist_x - 50, wrist_y - 40),
+                           cv2.FONT_HERSHEY_SIMPLEX, 0.7, color, 2)
+                
+                # Ориентация ладони (с цветовой индикацией)
+                if orientation == 'palm':
+                    orient_text = "LADON (palm)"
+                    orient_color = (0, 255, 0)  # Зелёный
+                else:
+                    orient_text = "TYLNAYA (back)"
+                    orient_color = (0, 0, 255)  # Красный
+                
+                cv2.putText(frame, orient_text, 
+                           (wrist_x - 50, wrist_y - 10),
+                           cv2.FONT_HERSHEY_SIMPLEX, 0.6, orient_color, 2)
+                
+                # === ОТОБРАЖЕНИЕ ЖЕСТА ===
+                if gesture:
+                    gesture_color = (0, 255, 255)  # Жёлтый для распознанного жеста
+                    cv2.putText(frame, f"GESTURE: {gesture}", 
+                            (wrist_x - 50, wrist_y),
+                            cv2.FONT_HERSHEY_SIMPLEX, 0.9, gesture_color, 3)
+                else:
+                    cv2.putText(frame, "No gesture", 
+                            (wrist_x - 50, wrist_y),
+                            cv2.FONT_HERSHEY_SIMPLEX, 0.6, (128, 128, 128), 2)
+        
+                # Количество пальцев
+                cv2.putText(frame, f"Fingers: {fingers_count}", 
+                           (wrist_x - 50, wrist_y + 25),
+                           cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 255, 255), 3)
+                
+                # Список пальцев
+                if fingers_names:
+                    fingers_text = ', '.join(fingers_names)
+                    cv2.putText(frame, fingers_text, 
+                               (wrist_x - 50, wrist_y + 55),
+                               cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 0), 2)
+    
+    # Большая панель с общей информацией
+    cv2.rectangle(frame, (10, 10), (350, 120), (0, 0, 0), -1)
+    cv2.putText(frame, f"Total fingers: {total_fingers}", 
+               (20, 50), cv2.FONT_HERSHEY_SIMPLEX, 1.5, (0, 255, 255), 3)
+    
+    hands_count = len(result.hand_landmarks) if result.hand_landmarks else 0
+    cv2.putText(frame, f"Hands detected: {hands_count}", 
+               (20, 90), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 255, 255), 2)
+    
+    # Вывод в консоль (раз в 30 кадров)
+    frame_counter += 1
+    if result.hand_landmarks and frame_counter % 30 == 0:
+        print(f"\n=== Кадр {frame_counter} ===")
+        print(f"Рук обнаружено: {hands_count}, всего пальцев: {total_fingers}")
+        for hand_idx, hand_landmarks in enumerate(result.hand_landmarks):
+            handedness = result.handedness[hand_idx][0].category_name
+            points = [(int(lm.x * w), int(lm.y * h)) for lm in hand_landmarks]
+            fingers_count, fingers_names = count_fingers(points, handedness)
+            orientation = detect_palm_orientation(points, handedness)
+            orient_str = "ЛАДОНЬ" if orientation == 'palm' else "ТЫЛЬНАЯ"
+            fingers_str = ', '.join(fingers_names) if fingers_names else 'нет'
+            print(f"  Рука {hand_idx+1} ({handedness}): ориентация={orient_str}, "
+                  f"пальцев={fingers_count} [{fingers_str}]")
+    
+    cv2.imshow("Hand Detection - Palm Orientation", frame)
+    
+    if cv2.waitKey(1) & 0xFF == ord('q'):
+        break
+
+cap.release()
+cv2.destroyAllWindows()
+print("Завершено.")