From 77d95377373b55131bab0c5138b0b9cb161f0acb Mon Sep 17 00:00:00 2001 From: springstart11111 <1312982847@qq.com> Date: Fri, 10 Apr 2026 10:32:05 +0800 Subject: [PATCH 01/11] =?UTF-8?q?feat(drone=5Fhand=5Fgesture):=20=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=20AirSim=20=E7=9C=9F=E5=AE=9E=E6=A8=A1=E6=8B=9F?= =?UTF-8?q?=E5=99=A8=E9=9B=86=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增功能: - 添加 airsim_controller.py: AirSim 无人机控制器 - 添加 main_airsim.py: AirSim 真实模拟器版本 - 更新 README.md: 添加 AirSim 使用说明 功能特性: - 支持连接 Microsoft AirSim 模拟器 - 实现起飞、降落、悬停、移动控制 - 保持原有手势识别功能 - 支持键盘控制（空格/T/L/H/Q） 使用说明: - 本地仿真：python main.py - AirSim 模式：python main_airsim.py - 需要先运行 AirSim 模拟器 --- src/drone_hand_gesture/README.md | 48 +++-- src/drone_hand_gesture/airsim_controller.py | 195 ++++++++++++++++++ src/drone_hand_gesture/main_airsim.py | 212 ++++++++++++++++++++ 3 files changed, 436 insertions(+), 19 deletions(-) create mode 100644 src/drone_hand_gesture/airsim_controller.py create mode 100644 src/drone_hand_gesture/main_airsim.py diff --git a/src/drone_hand_gesture/README.md b/src/drone_hand_gesture/README.md index 6a7429a33..5951dada6 100644 --- a/src/drone_hand_gesture/README.md +++ b/src/drone_hand_gesture/README.md @@ -1,22 +1,32 @@ -# 通过手势来控制无人机行动 - -### 运行目录结构 - ```text - Drone_hand_gesture_project/ - ├── main.py # 主程序 - ├── drone_controller.py # 无人机控制 - ├── simulation_3d.py # 3D仿真 - ├── physics_engine.py # 物理仿真引擎 - ├── gesture_detector.py # 基础手势检测器 - ├── gesture_data_collector.py # 手势图像数据收集 - ├── gesture_detector_enhanced.py # 增强手势检测器 - ├── gesture_classifier.py # 手势识别分类器 - ├── train_gesture_model.py # 训练识别模型 - ├── dataset/ # 数据集目录 - │ ├── raw/ # 原始数据 - │ ├── processed/ # 处理后的数据 - │ └── models/ # 训练好的模型 - └── requirements.txt # 依赖列表 +# 手势控制无人机项目 + +基于计算机视觉的手势识别无人机控制系统，支持本地仿真和 AirSim 真实模拟器。 + +## 功能特性 + +- ✅ 实时手势识别（MediaPipe） +- ✅ 本地 3D 仿真模式 +- ✅ AirSim 真实模拟器集成 +- ✅ 机器学习手势分类 +- ✅ 数据收集与训练 + +## 项目结构 + +```text +drone_hand_gesture/ +├── main.py # 主程序（本地仿真模式） +├── main_airsim.py # AirSim 真实模拟器版本 +├── airsim_controller.py # AirSim 控制器（新增） +├── drone_controller.py # 无人机控制器 +├── simulation_3d.py # 3D 仿真 +├── physics_engine.py # 物理仿真引擎 +├── gesture_detector.py # 基础手势检测器 +├── gesture_detector_enhanced.py # 增强手势检测器 +├── gesture_classifier.py # 手势识别分类器 +├── gesture_data_collector.py # 手势图像数据收集 +├── train_gesture_model.py # 训练识别模型 +└── requirements.txt # 依赖列表 +``` ### 运行步骤(在ubuntu或是xubuntu下) 1. 进入项目目录： diff --git a/src/drone_hand_gesture/airsim_controller.py b/src/drone_hand_gesture/airsim_controller.py new file mode 100644 index 000000000..4c23c5280 --- /dev/null +++ b/src/drone_hand_gesture/airsim_controller.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- +""" +AirSim 无人机控制器 +连接到 Microsoft AirSim 模拟器，实现真实的无人机控制 +""" + +import time +import numpy as np +from typing import Optional, Dict, Any + + +class AirSimController: + """AirSim 无人机控制器""" + + def __init__(self, ip_address: str = "127.0.0.1", port: int = 41451): + """ + 初始化 AirSim 控制器 + + Args: + ip_address: AirSim 服务器地址 + port: AirSim RPC 端口 + """ + self.ip_address = ip_address + self.port = port + self.client = None + self.connected = False + self.vehicle_name = "" # 空字符串表示默认飞行器 + + # 无人机状态 + self.state = { + 'position': np.array([0.0, 0.0, 0.0]), + 'velocity': np.array([0.0, 0.0, 0.0]), + 'orientation': np.array([0.0, 0.0, 0.0]), + 'battery': 100.0, + 'armed': False, + 'flying': False + } + + def connect(self) -> bool: + """连接到 AirSim 模拟器""" + try: + import airsim + + print(f"正在连接到 AirSim ({self.ip_address}:{self.port})...") + self.client = airsim.MultirotorClient() + self.client.confirmConnection() + + self.client.enableApiControl(True) + self.client.armDisarm(True, vehicle_name=self.vehicle_name) + + self.connected = True + self.state['armed'] = True + + print("[OK] 成功连接到 AirSim 模拟器！") + print(f" 飞行器：{self.vehicle_name if self.vehicle_name else '默认'}") + print(f" 状态：已解锁") + + return True + + except ImportError: + print("[ERROR] 未找到 airsim 模块") + print(" 请运行：pip install airsim") + return False + + except Exception as e: + print(f"[ERROR] 连接失败 - {e}") + print(" 请确保 AirSim 模拟器正在运行") + return False + + def disconnect(self): + """断开连接""" + if self.client and self.connected: + try: + self.client.armDisarm(False, vehicle_name=self.vehicle_name) + self.client.enableApiControl(False) + self.connected = False + print("[OK] 已断开 AirSim 连接") + except Exception as e: + print(f"Warning: 断开连接时出错 - {e}") + + def takeoff(self, altitude: float = 2.0) -> bool: + """起飞""" + if not self.connected: + print("[ERROR] 未连接到 AirSim") + return False + + try: + print(f"[INFO] 正在起飞到 {altitude} 米高度...") + self.client.takeoffAsync(altitude=altitude, vehicle_name=self.vehicle_name).join() + self.state['flying'] = True + print("[OK] 起飞完成！") + return True + except Exception as e: + print(f"[ERROR] 起飞失败：{e}") + return False + + def land(self) -> bool: + """降落""" + if not self.connected: + print("[ERROR] 未连接到 AirSim") + return False + + try: + print("[INFO] 正在降落...") + self.client.landAsync(vehicle_name=self.vehicle_name).join() + self.state['flying'] = False + print("[OK] 降落完成！") + return True + except Exception as e: + print(f"[ERROR] 降落失败：{e}") + return False + + def hover(self): + """悬停""" + if not self.connected: + return + try: + self.client.moveToPositionAsync( + self.state['position'][0], + self.state['position'][1], + self.state['position'][2], + 1.0, + vehicle_name=self.vehicle_name + ) + except: + pass + + def move_by_velocity(self, vx: float, vy: float, vz: float, duration: float = 0.5): + """ + 按速度控制无人机 + + Args: + vx: X 轴速度 (m/s), 右移为正 + vy: Y 轴速度 (m/s), 前进为正 + vz: Z 轴速度 (m/s), 下降为正 + duration: 持续时间 (秒) + """ + if not self.connected: + return + + try: + import airsim + self.client.moveByVelocityZAsync( + vx, vy, -vz, duration, + drivetrain=airsim.DrivetrainType.ForwardOnly, + yaw_mode=airsim.YawMode(), + vehicle_name=self.vehicle_name + ) + except Exception as e: + print(f"Warning: 速度控制失败 - {e}") + + def get_state(self) -> Dict[str, Any]: + """获取无人机状态""" + if not self.connected: + return self.state + + try: + state = self.client.getMultirotorState(vehicle_name=self.vehicle_name) + self.state['position'] = np.array([ + state.position.x_val, + state.position.y_val, + -state.position.z_val + ]) + self.state['velocity'] = np.array([ + state.velocity.x_val, + state.velocity.y_val, + -state.velocity.z_val + ]) + return self.state + except Exception as e: + print(f"Warning: 获取状态失败 - {e}") + return self.state + + +def test_airsim_connection(): + """测试 AirSim 连接""" + print("=" * 60) + print("AirSim 连接测试") + print("=" * 60) + + controller = AirSimController() + + if controller.connect(): + print("\n[OK] AirSim 连接成功！") + state = controller.get_state() + print(f"位置：{state['position']}") + controller.disconnect() + return True + else: + print("\n[ERROR] AirSim 连接失败") + return False + + +if __name__ == "__main__": + test_airsim_connection() diff --git a/src/drone_hand_gesture/main_airsim.py b/src/drone_hand_gesture/main_airsim.py new file mode 100644 index 000000000..6c8ed6507 --- /dev/null +++ b/src/drone_hand_gesture/main_airsim.py @@ -0,0 +1,212 @@ +# -*- coding: utf-8 -*- +""" +手势控制无人机 - AirSim 真实模拟器版 +基于 drone_hand_gesture 项目，添加 AirSim 集成 +""" + +import cv2 +import numpy as np +import time +import sys +import os + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + +from airsim_controller import AirSimController +from gesture_detector_enhanced import EnhancedGestureDetector + + +def main(): + """主函数""" + print("=" * 70) + print("手势控制无人机 - AirSim 真实模拟器版") + print("=" * 70) + print() + + # 1. 连接 AirSim + print("[1/4] 正在连接 AirSim 模拟器...") + controller = AirSimController(ip_address="127.0.0.1", port=41451) + + if not controller.connect(): + print("\n[ERROR] AirSim 连接失败") + print("\n请检查:") + print(" 1. AirSim 模拟器是否运行") + print(" 2. 防火墙设置") + print("\n按回车键退出...") + input() + return + + # 2. 初始化手势检测器 + print("\n[2/4] 正在初始化手势检测器...") + detector = EnhancedGestureDetector(use_ml=False) + print("[OK] 手势检测器就绪") + + # 3. 初始化摄像头 + print("\n[3/4] 正在初始化摄像头...") + cap = cv2.VideoCapture(0) + + if not cap.isOpened(): + print("[ERROR] 摄像头不可用") + controller.disconnect() + return + + cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) + cap.set(cv2.CAP_PROP_FPS, 30) + print("[OK] 摄像头已就绪：640x480 @ 30fps") + + # 4. 系统就绪 + print("\n[4/4] 系统就绪！") + print("\n" + "=" * 70) + print("手势控制:") + print(" 张开手掌 - 悬停") + print(" 食指向上 - 上升") + print(" 食指向下 - 下降") + print(" 指向左侧 - 左移") + print(" 指向右侧 - 右移") + print(" 两指向前 - 前进") + print(" 握拳 - 降落") + print("\n键盘控制:") + print(" 空格键 - 起飞/降落") + print(" T - 手动起飞") + print(" L - 手动降落") + print(" H - 悬停") + print(" Q/ESC - 退出程序") + print("=" * 70) + print() + + # 主循环 + is_flying = False + last_gesture_time = 0 + current_gesture = "" + gesture_hold_duration = 1.0 + frame_count = 0 + start_time = time.time() + + print("[INFO] 按 空格键 或 T 键 起飞") + + try: + while True: + ret, frame = cap.read() + if not ret: + print("[WARNING] 无法读取摄像头画面") + break + + frame_count += 1 + + # 手势识别 + debug_frame, gesture, confidence, _ = detector.detect_gestures(frame) + + # 显示帧率 + elapsed = time.time() - start_time + fps = frame_count / elapsed if elapsed > 0 else 0 + + cv2.putText(debug_frame, f"FPS: {fps:.1f}", (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2) + cv2.putText(debug_frame, f"Gesture: {gesture} ({confidence:.2f})", + (10, 60), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2) + + # 处理手势 + if gesture and gesture != "none" and gesture != "no_hand": + current_time = time.time() + + if gesture == current_gesture: + if current_time - last_gesture_time > gesture_hold_duration: + print(f"[CMD] 手势：{gesture}") + + if gesture == 'fist': + if is_flying: + print("[INFO] 降落...") + controller.land() + is_flying = False + + elif gesture == 'point_up': + print("[INFO] 上升") + controller.move_by_velocity(0, 0, -1.0, duration=0.5) + + elif gesture == 'point_down': + print("[INFO] 下降") + controller.move_by_velocity(0, 0, 1.0, duration=0.5) + + elif gesture == 'point_left': + print("[INFO] 左移") + controller.move_by_velocity(-1.0, 0, 0, duration=0.5) + + elif gesture == 'point_right': + print("[INFO] 右移") + controller.move_by_velocity(1.0, 0, 0, duration=0.5) + + elif gesture == 'victory': + print("[INFO] 前进") + controller.move_by_velocity(0, 1.0, 0, duration=0.5) + + elif gesture == 'open_palm': + print("[INFO] 悬停") + controller.hover() + + last_gesture_time = current_time + else: + current_gesture = gesture + last_gesture_time = current_time + + # 显示画面 + cv2.imshow('Gesture Control - AirSim', debug_frame) + + # 键盘控制 + key = cv2.waitKey(10) & 0xFF + + if key == ord('q') or key == ord('Q') or key == 27: + print("\n[INFO] 退出程序...") + break + + elif key == ord(' '): + if is_flying: + print("[INFO] 降落...") + controller.land() + is_flying = False + else: + print("[INFO] 起飞...") + controller.takeoff() + is_flying = True + time.sleep(0.5) + + elif key == ord('t') or key == ord('T'): + if not is_flying: + print("[INFO] 手动起飞...") + controller.takeoff() + is_flying = True + + elif key == ord('l') or key == ord('L'): + if is_flying: + print("[INFO] 手动降落...") + controller.land() + is_flying = False + + elif key == ord('h') or key == ord('H'): + print("[INFO] 悬停") + controller.hover() + + # 显示状态 + if is_flying and frame_count % 30 == 0: + state = controller.get_state() + print(f"[状态] 高度：{state['position'][2]:.2f}m") + + except KeyboardInterrupt: + print("\n[INFO] 程序中断") + + finally: + print("\n[INFO] 清理资源...") + + if is_flying: + print("[INFO] 正在降落...") + controller.land() + + cap.release() + cv2.destroyAllWindows() + controller.disconnect() + + print("[OK] 程序安全退出") + + +if __name__ == "__main__": + main() From 3f1dd5a9018c42cfdc31a8578d3bf966e3cc1556 Mon Sep 17 00:00:00 2001 From: springstart11111 <1312982847@qq.com> Date: Fri, 10 Apr 2026 10:48:47 +0800 Subject: [PATCH 02/11] =?UTF-8?q?docs(drone=5Fhand=5Fgesture):=20=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E8=BF=90=E8=A1=8C=E6=8C=87=E5=8D=97=E5=92=8C=E5=90=AF?= =?UTF-8?q?=E5=8A=A8=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增文档: - 运行指南.md: 详细的中文运行说明 - 运行_本地仿真.bat: 本地仿真一键启动 - 运行_AirSim 版.bat: AirSim 一键启动（含检查） 使用说明: - 新手：双击 运行_本地仿真.bat - AirSim: 双击 运行_AirSim 版.bat - 包含完整的控制说明和故障排除 --- ...7\220\350\241\214_AirSim \347\211\210.bat" | 32 ++++ ...4\345\234\260\344\273\277\347\234\237.bat" | 16 ++ ...20\350\241\214\346\214\207\345\215\227.md" | 177 ++++++++++++++++++ 3 files changed, 225 insertions(+) create mode 100644 "src/drone_hand_gesture/\350\277\220\350\241\214_AirSim \347\211\210.bat" create mode 100644 "src/drone_hand_gesture/\350\277\220\350\241\214_\346\234\254\345\234\260\344\273\277\347\234\237.bat" create mode 100644 "src/drone_hand_gesture/\350\277\220\350\241\214\346\214\207\345\215\227.md" diff --git "a/src/drone_hand_gesture/\350\277\220\350\241\214_AirSim \347\211\210.bat" "b/src/drone_hand_gesture/\350\277\220\350\241\214_AirSim \347\211\210.bat" new file mode 100644 index 000000000..be7b964e7 --- /dev/null +++ "b/src/drone_hand_gesture/\350\277\220\350\241\214_AirSim \347\211\210.bat" @@ -0,0 +1,32 @@ +@echo off +chcp 65001 >nul +echo ================================================ +echo 手势控制无人机 - AirSim 真实模拟器版 +echo ================================================ +echo. +echo 检查 AirSim 是否运行... +echo. + +REM 检查 AirSim 是否运行 +tasklist /FI "WINDOWTITLE eq Blocks" 2>nul | find "Blocks.exe" >nul +if %ERRORLEVEL% NEQ 0 ( + echo [警告] AirSim 未运行！ + echo. + echo 请先启动 AirSim 模拟器: + echo 双击运行：d:\机械学习\air\Blocks\WindowsNoEditor\Blocks.exe + echo. + echo 按任意键继续（如果 AirSim 已启动）... + pause >nul +) + +echo. +echo 正在启动手势控制程序... +echo. + +cd /d "%~dp0" +python main_airsim.py + +echo. +echo ================================================ +echo 程序已退出 +pause diff --git "a/src/drone_hand_gesture/\350\277\220\350\241\214_\346\234\254\345\234\260\344\273\277\347\234\237.bat" "b/src/drone_hand_gesture/\350\277\220\350\241\214_\346\234\254\345\234\260\344\273\277\347\234\237.bat" new file mode 100644 index 000000000..a5393cb4c --- /dev/null +++ "b/src/drone_hand_gesture/\350\277\220\350\241\214_\346\234\254\345\234\260\344\273\277\347\234\237.bat" @@ -0,0 +1,16 @@ +@echo off +chcp 65001 >nul +echo ================================================ +echo 手势控制无人机 - 本地仿真版 +echo ================================================ +echo. +echo 正在启动... +echo. + +cd /d "%~dp0" +python main.py + +echo. +echo ================================================ +echo 程序已退出 +pause diff --git "a/src/drone_hand_gesture/\350\277\220\350\241\214\346\214\207\345\215\227.md" "b/src/drone_hand_gesture/\350\277\220\350\241\214\346\214\207\345\215\227.md" new file mode 100644 index 000000000..249027984 --- /dev/null +++ "b/src/drone_hand_gesture/\350\277\220\350\241\214\346\214\207\345\215\227.md" @@ -0,0 +1,177 @@ +# 手势控制无人机 - 运行指南 + +## 🚀 快速开始 + +### 方案 1：本地仿真模式（推荐新手）⭐ + +**特点**： +- ✅ 不需要安装 AirSim +- ✅ 不需要额外配置 +- ✅ 直接运行即可 +- ✅ 有 3D 仿真显示 + +**运行步骤**： + +1. **打开终端**（PowerShell 或 CMD） + +2. **进入项目目录**： + ```bash + cd d:\机械学习\nn\src\drone_hand_gesture + ``` + +3. **运行程序**： + ```bash + python main.py + ``` + +4. **控制无人机**： + - 对着摄像头做手势 + - 或按键盘控制： + - `T` - 起飞 + - `L` - 降落 + - `H` - 悬停 + - `空格` - 重置位置 + - `Q` - 退出 + +--- + +### 方案 2：AirSim 真实模拟器模式 + +**特点**： +- ✅ 连接真实的 AirSim 模拟器 +- ✅ 可以获取真实飞行数据 +- ⚠️ 需要安装 AirSim + +**前提条件**： + +1. **安装 AirSim Python 包**： + ```bash + pip install airsim + ``` + +2. **启动 AirSim 模拟器**： + - 双击运行：`d:\机械学习\air\Blocks\WindowsNoEditor\Blocks.exe` + - 等待看到无人机和场景加载完成 + +**运行步骤**： + +1. **打开新终端** + +2. **进入项目目录**： + ```bash + cd d:\机械学习\nn\src\drone_hand_gesture + ``` + +3. **运行 AirSim 版本**： + ```bash + python main_airsim.py + ``` + +4. **控制无人机**： + - **按空格键或 T 键** - 起飞 + - **手势控制**： + - 张开手掌 - 悬停 + - 食指向上 - 上升 + - 食指向下 - 下降 + - 指向左右 - 左右移动 + - 两指向前 - 前进 + - 握拳 - 降落 + - **键盘控制**： + - `L` - 降落 + - `H` - 悬停 + - `Q/ESC` - 退出 + +--- + +## 📹 摄像头使用提示 + +1. **光线充足** - 确保环境光线良好 +2. **手势清晰** - 手放在摄像头中心位置 +3. **保持距离** - 手距离摄像头 30-50cm 最佳 +4. **背景简单** - 避免复杂的背景干扰 + +--- + +## ⚠️ 常见问题 + +### Q: 摄像头打不开？ +**A**: 检查是否有其他程序占用摄像头（如微信、Zoom 等） + +### Q: 手势识别不准确？ +**A**: +- 调整光线 +- 手靠近摄像头中心 +- 保持手势稳定 1-2 秒 + +### Q: AirSim 连接失败？ +**A**: +- 确保 Blocks.exe 已启动 +- 等待 15 秒让 AirSim 完全启动 +- 检查防火墙设置 + +### Q: 程序运行报错？ +**A**: +```bash +# 安装所有依赖 +pip install -r requirements.txt +``` + +--- + +## 🎮 完整控制说明 + +### 本地仿真模式 +- `T` - 手动起飞 +- `L` - 手动降落 +- `H` - 悬停 +- `空格` - 重置位置 +- `Q` - 退出程序 + +### AirSim 模式 +**键盘**： +- `空格` - 起飞/降落 +- `T` - 手动起飞 +- `L` - 手动降落 +- `H` - 悬停 +- `Q/ESC` - 退出 + +**手势**： +- ✋ 张开手掌 - 悬停 +- 👆 食指向上 - 上升 +- 👇 食指向下 - 下降 +- 👈 指向左 - 左移 +- 👉 指向右 - 右移 +- ✌️ 两指向前 - 前进 +- 👍 大拇指 - 后退 +- 👊 握拳 - 降落 + +--- + +## 📊 查看运行效果 + +运行成功后，你应该看到： + +1. **摄像头窗口** - 显示你的手和识别结果 +2. **3D 仿真窗口**（本地模式） - 显示无人机飞行 +3. **AirSim 窗口**（AirSim 模式） - 真实模拟器画面 +4. **控制台输出** - 显示状态信息 + +--- + +## 💡 建议 + +**新手建议**： +1. 先运行**本地仿真模式**熟悉操作 +2. 确认手势识别正常 +3. 再尝试 AirSim 真实模拟器 + +**运行顺序**： +```bash +# 1. 本地仿真（简单） +python main.py + +# 2. AirSim 模式（需要 AirSim） +python main_airsim.py +``` + +祝你使用愉快！🚁 From e5d46ca53cc3a1564ecfc298d39a260f34c232ad Mon Sep 17 00:00:00 2001 From: springstart11111 <1312982847@qq.com> Date: Fri, 10 Apr 2026 11:42:41 +0800 Subject: [PATCH 03/11] =?UTF-8?q?fix(airsim):=20=E4=BF=AE=E5=A4=8D=20AirSi?= =?UTF-8?q?m=20API=20=E8=B0=83=E7=94=A8=E9=97=AE=E9=A2=98=E5=92=8C?= =?UTF-8?q?=E6=98=BE=E7=A4=BA=E7=AA=97=E5=8F=A3=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/drone_hand_gesture/README.md | 40 +++++---- src/drone_hand_gesture/airsim_controller.py | 18 ++-- src/drone_hand_gesture/main_airsim.py | 93 +++++++++++++-------- 3 files changed, 89 insertions(+), 62 deletions(-) diff --git a/src/drone_hand_gesture/README.md b/src/drone_hand_gesture/README.md index 5951dada6..4a909e759 100644 --- a/src/drone_hand_gesture/README.md +++ b/src/drone_hand_gesture/README.md @@ -28,31 +28,35 @@ drone_hand_gesture/ └── requirements.txt # 依赖列表 ``` -### 运行步骤(在ubuntu或是xubuntu下) -1. 进入项目目录： +## 快速开始 - ```bash - cd /mnt/hgfs/nn/src/Drone_hand_gesture_project +### 方案 1：本地仿真模式（不需要 AirSim） -2. 激活虚拟环境： +```bash +# 进入项目目录 +cd src/drone_hand_gesture - ```bash - source gesture_env/bin/activate +# 安装依赖 +pip install -r requirements.txt -3. 收集手势图像： - - ```bash - python gesture_data_collector.py - -4. 训练学习模型： +# 运行主程序 +python main.py +``` - ```bash - python train_gesture_model.py --model_type ensemble +### 方案 2：AirSim 真实模拟器模式 -5. 运行主程序： +**前提条件**： +1. 安装 AirSim：`pip install airsim` +2. 运行 AirSim 模拟器（如 Blocks.exe） + +```bash +# 运行 AirSim 版本 +python main_airsim.py +``` - ```bash - python main.py +**控制方式**： +- **手势控制**：张开手掌（悬停）、食指向上（上升）、握拳（降落）等 +- **键盘控制**：空格（起飞/降落）、T（起飞）、L（降落）、H（悬停）、Q/ESC（退出） ##参考项目 diff --git a/src/drone_hand_gesture/airsim_controller.py b/src/drone_hand_gesture/airsim_controller.py index 4c23c5280..99dcf42dc 100644 --- a/src/drone_hand_gesture/airsim_controller.py +++ b/src/drone_hand_gesture/airsim_controller.py @@ -86,7 +86,10 @@ def takeoff(self, altitude: float = 2.0) -> bool: try: print(f"[INFO] 正在起飞到 {altitude} 米高度...") - self.client.takeoffAsync(altitude=altitude, vehicle_name=self.vehicle_name).join() + self.client.takeoffAsync(vehicle_name=self.vehicle_name).join() + # 起飞后上升到指定高度 + import airsim + self.client.moveToZAsync(-altitude, 1.0, vehicle_name=self.vehicle_name).join() self.state['flying'] = True print("[OK] 起飞完成！") return True @@ -156,15 +159,16 @@ def get_state(self) -> Dict[str, Any]: try: state = self.client.getMultirotorState(vehicle_name=self.vehicle_name) + # 修复状态获取 - 使用正确的属性名称 self.state['position'] = np.array([ - state.position.x_val, - state.position.y_val, - -state.position.z_val + state.kinematics_estimated.position.x_val, + state.kinematics_estimated.position.y_val, + -state.kinematics_estimated.position.z_val ]) self.state['velocity'] = np.array([ - state.velocity.x_val, - state.velocity.y_val, - -state.velocity.z_val + state.kinematics_estimated.linear_velocity.x_val, + state.kinematics_estimated.linear_velocity.y_val, + -state.kinematics_estimated.linear_velocity.z_val ]) return self.state except Exception as e: diff --git a/src/drone_hand_gesture/main_airsim.py b/src/drone_hand_gesture/main_airsim.py index 6c8ed6507..209461a8e 100644 --- a/src/drone_hand_gesture/main_airsim.py +++ b/src/drone_hand_gesture/main_airsim.py @@ -16,7 +16,7 @@ from gesture_detector_enhanced import EnhancedGestureDetector -def main(): +def main(show_window=True): """主函数""" print("=" * 70) print("手势控制无人机 - AirSim 真实模拟器版") @@ -25,8 +25,11 @@ def main(): # 1. 连接 AirSim print("[1/4] 正在连接 AirSim 模拟器...") + print("[DEBUG] 正在创建 AirSimController 实例...") controller = AirSimController(ip_address="127.0.0.1", port=41451) + print("[DEBUG] AirSimController 实例创建成功") + print("[DEBUG] 正在连接 AirSim...") if not controller.connect(): print("\n[ERROR] AirSim 连接失败") print("\n请检查:") @@ -35,24 +38,31 @@ def main(): print("\n按回车键退出...") input() return + print("[DEBUG] AirSim 连接成功") # 2. 初始化手势检测器 print("\n[2/4] 正在初始化手势检测器...") + print("[DEBUG] 正在创建 EnhancedGestureDetector 实例...") detector = EnhancedGestureDetector(use_ml=False) + print("[DEBUG] EnhancedGestureDetector 实例创建成功") print("[OK] 手势检测器就绪") # 3. 初始化摄像头 print("\n[3/4] 正在初始化摄像头...") + print("[DEBUG] 正在打开摄像头...") cap = cv2.VideoCapture(0) + print("[DEBUG] 摄像头打开成功") if not cap.isOpened(): print("[ERROR] 摄像头不可用") controller.disconnect() return + print("[DEBUG] 正在设置摄像头参数...") cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) cap.set(cv2.CAP_PROP_FPS, 30) + print("[DEBUG] 摄像头参数设置成功") print("[OK] 摄像头已就绪：640x480 @ 30fps") # 4. 系统就绪 @@ -150,41 +160,49 @@ def main(): last_gesture_time = current_time # 显示画面 - cv2.imshow('Gesture Control - AirSim', debug_frame) - - # 键盘控制 - key = cv2.waitKey(10) & 0xFF - - if key == ord('q') or key == ord('Q') or key == 27: - print("\n[INFO] 退出程序...") - break - - elif key == ord(' '): - if is_flying: - print("[INFO] 降落...") - controller.land() - is_flying = False - else: - print("[INFO] 起飞...") - controller.takeoff() - is_flying = True - time.sleep(0.5) - - elif key == ord('t') or key == ord('T'): - if not is_flying: - print("[INFO] 手动起飞...") - controller.takeoff() - is_flying = True - - elif key == ord('l') or key == ord('L'): - if is_flying: - print("[INFO] 手动降落...") - controller.land() - is_flying = False - - elif key == ord('h') or key == ord('H'): - print("[INFO] 悬停") - controller.hover() + if show_window: + cv2.imshow('Gesture Control - AirSim', debug_frame) + + # 键盘控制 + key = cv2.waitKey(10) & 0xFF + + if key == ord('q') or key == ord('Q') or key == 27: + print("\n[INFO] 退出程序...") + break + + elif key == ord(' '): + if is_flying: + print("[INFO] 降落...") + controller.land() + is_flying = False + else: + print("[INFO] 起飞...") + controller.takeoff() + is_flying = True + time.sleep(0.5) + + elif key == ord('t') or key == ord('T'): + if not is_flying: + print("[INFO] 手动起飞...") + controller.takeoff() + is_flying = True + + elif key == ord('l') or key == ord('L'): + if is_flying: + print("[INFO] 手动降落...") + controller.land() + is_flying = False + + elif key == ord('h') or key == ord('H'): + print("[INFO] 悬停") + controller.hover() + else: + # 没有显示窗口时，使用一个简单的循环来模拟 + time.sleep(0.1) + # 检查是否需要退出 + if frame_count > 300: # 运行 5 秒后自动退出 + print("\n[INFO] 自动退出程序...") + break # 显示状态 if is_flying and frame_count % 30 == 0: @@ -202,7 +220,8 @@ def main(): controller.land() cap.release() - cv2.destroyAllWindows() + if show_window: + cv2.destroyAllWindows() controller.disconnect() print("[OK] 程序安全退出") From 0be7176315940ae056809be9e16a155133c09816 Mon Sep 17 00:00:00 2001 From: springstart11111 <1312982847@qq.com> Date: Fri, 10 Apr 2026 15:57:20 +0800 Subject: [PATCH 04/11] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E8=B7=AF?= =?UTF-8?q?=E5=BE=84=E9=97=AE=E9=A2=98=E5=92=8C=E6=96=87=E4=BB=B6=E5=90=8D?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=8C=E4=BD=BF=E7=94=A8=E7=9B=B8=E5=AF=B9?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E5=92=8C=E8=8B=B1=E6=96=87=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/drone_hand_gesture/run_airsim.bat | 33 +++++++++++++++++++ src/drone_hand_gesture/run_local.bat | 16 +++++++++ ...20\350\241\214\346\214\207\345\215\227.md" | 2 +- 3 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 src/drone_hand_gesture/run_airsim.bat create mode 100644 src/drone_hand_gesture/run_local.bat diff --git a/src/drone_hand_gesture/run_airsim.bat b/src/drone_hand_gesture/run_airsim.bat new file mode 100644 index 000000000..89c3db632 --- /dev/null +++ b/src/drone_hand_gesture/run_airsim.bat @@ -0,0 +1,33 @@ +@echo off +chcp 65001 >nul +echo ================================================ +echo Drone Gesture Control - AirSim Version +echo ================================================ +echo. +echo Checking if AirSim is running... +echo. + +REM Check if AirSim is running +tasklist /FI "WINDOWTITLE eq Blocks" 2>nul | find "Blocks.exe" >nul +if %ERRORLEVEL% NEQ 0 ( + echo [WARNING] AirSim is not running! + echo. + echo Please start AirSim simulator first: + echo Double click: Blocks.exe + echo Note: Blocks.exe is usually in the AirSim installation directory + echo. + echo Press any key to continue (if AirSim is already running)... + pause >nul +) + +echo. +echo Starting gesture control program... +echo. + +cd /d "%~dp0" +python main_airsim.py + +echo. +echo ================================================ +echo Program exited +pause diff --git a/src/drone_hand_gesture/run_local.bat b/src/drone_hand_gesture/run_local.bat new file mode 100644 index 000000000..6b5b96629 --- /dev/null +++ b/src/drone_hand_gesture/run_local.bat @@ -0,0 +1,16 @@ +@echo off +chcp 65001 >nul +echo ================================================ +echo Drone Gesture Control - Local Simulation +echo ================================================ +echo. +echo Starting... +echo. + +cd /d "%~dp0" +python main.py + +echo. +echo ================================================ +echo Program exited +pause diff --git "a/src/drone_hand_gesture/\350\277\220\350\241\214\346\214\207\345\215\227.md" "b/src/drone_hand_gesture/\350\277\220\350\241\214\346\214\207\345\215\227.md" index 249027984..86d461764 100644 --- "a/src/drone_hand_gesture/\350\277\220\350\241\214\346\214\207\345\215\227.md" +++ "b/src/drone_hand_gesture/\350\277\220\350\241\214\346\214\207\345\215\227.md" @@ -16,7 +16,7 @@ 2. **进入项目目录**： ```bash - cd d:\机械学习\nn\src\drone_hand_gesture + cd path/to/nn/src/drone_hand_gesture ``` 3. **运行程序**： From 1303290a6e3930feb3043b9383b90c5f46abce2a Mon Sep 17 00:00:00 2001 From: springstart11111 <1312982847@qq.com> Date: Fri, 10 Apr 2026 16:22:56 +0800 Subject: [PATCH 05/11] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20main.py=20?= =?UTF-8?q?=E4=B8=AD=E7=9A=84=20emoji=20=E5=AD=97=E7=AC=A6=EF=BC=8C?= =?UTF-8?q?=E8=A7=A3=E5=86=B3=20Windows=20=E7=BB=88=E7=AB=AF=E7=BC=96?= =?UTF-8?q?=E7=A0=81=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/drone_hand_gesture/main.py | 62 +++++++++---------- ...20\350\241\214\346\214\207\345\215\227.md" | 2 +- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/drone_hand_gesture/main.py b/src/drone_hand_gesture/main.py index 3bfbea354..df16c242e 100644 --- a/src/drone_hand_gesture/main.py +++ b/src/drone_hand_gesture/main.py @@ -13,10 +13,10 @@ try: from gesture_detector_enhanced import EnhancedGestureDetector - print("✅ 导入增强版手势检测器 (机器学习)") + print("[OK] 导入增强版手势检测器 (机器学习)") HAS_ENHANCED_DETECTOR = True except ImportError: - print("⚠️ 未找到增强版检测器，使用原始手势检测器") + print("[WARNING] 未找到增强版检测器，使用原始手势检测器") from gesture_detector import GestureDetector HAS_ENHANCED_DETECTOR = False @@ -61,21 +61,21 @@ def __init__(self, config=None): for model_path, model_name in model_candidates: if os.path.exists(model_path): file_size = os.path.getsize(model_path) - print(f"📁 找到 {model_name}: {file_size / 1024:.1f} KB") + print(f"[INFO] 找到 {model_name}: {file_size / 1024:.1f} KB") # 检查文件大小是否合理 if file_size > 10 * 1024: selected_model = model_path selected_model_name = model_name - print(f"✅ 选择: {model_name}") + print(f"[OK] 选择: {model_name}") break if selected_model: - print(f"🎯 使用模型: {selected_model_name}") + print(f"[INFO] 使用模型: {selected_model_name}") try: from gesture_detector_enhanced import EnhancedGestureDetector - print("✅ 导入增强版手势检测器") + print("[OK] 导入增强版手势检测器") # 使用实际的模型文件 self.gesture_detector = EnhancedGestureDetector( @@ -85,21 +85,21 @@ def __init__(self, config=None): # 验证模型是否真正加载成功 if hasattr(self.gesture_detector, 'ml_classifier') and self.gesture_detector.ml_classifier: - print(f"✅ 机器学习模型加载成功 ({selected_model_name})") + print(f"[OK] 机器学习模型加载成功 ({selected_model_name})") print(f" 可识别手势: {self.gesture_detector.ml_classifier.gesture_classes}") else: - print("⚠️ 机器学习模型未加载，回退到规则检测") + print("[WARNING] 机器学习模型未加载，回退到规则检测") self.gesture_detector = EnhancedGestureDetector(use_ml=False) except ImportError as e: - print(f"⚠️ 无法导入增强版检测器: {e}") - print("✅ 使用原始手势检测器") + print(f"[WARNING] 无法导入增强版检测器: {e}") + print("[OK] 使用原始手势检测器") from gesture_detector import GestureDetector self.gesture_detector = GestureDetector() else: - print("⚠️ 未找到可用的机器学习模型文件") - print("✅ 使用原始手势检测器") + print("[WARNING] 未找到可用的机器学习模型文件") + print("[OK] 使用原始手势检测器") from gesture_detector import GestureDetector self.gesture_detector = GestureDetector() @@ -140,7 +140,7 @@ def __init__(self, config=None): # 手势识别阈值（降低以提高灵敏度） # 如果是机器学习模式，阈值可以进一步降低 if HAS_ENHANCED_DETECTOR and hasattr(self.gesture_detector, 'use_ml') and self.gesture_detector.use_ml: - print("✅ 使用机器学习模式，置信度阈值更低") + print("[OK] 使用机器学习模式，置信度阈值更低") base_threshold = 0.55 # 机器学习可以更低 else: base_threshold = 0.6 # 规则检测需要高一点 @@ -166,13 +166,13 @@ def __init__(self, config=None): print("无人机初始化完成，等待手势指令...") - print("无人机仿真系统初始化完成 ✓") + print("无人机仿真系统初始化完成 [OK]") if HAS_ENHANCED_DETECTOR and hasattr(self.gesture_detector, 'use_ml'): if self.gesture_detector.use_ml: - print("📊 当前模式: 机器学习手势识别") + print("[INFO] 当前模式: 机器学习手势识别") else: - print("📊 当前模式: 规则手势识别") + print("[INFO] 当前模式: 规则手势识别") def _initialize_camera(self): """初始化摄像头""" @@ -197,7 +197,7 @@ def _initialize_camera(self): height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) fps = cap.get(cv2.CAP_PROP_FPS) - print(f"✅ 摄像头 {camera_id} 初始化成功: {width}x{height} @ {fps:.1f}fps") + print(f"[OK] 摄像头 {camera_id} 初始化成功: {width}x{height} @ {fps:.1f}fps") return cap else: cap.release() @@ -205,7 +205,7 @@ def _initialize_camera(self): else: print(f"摄像头 {camera_id} 无法打开") - print("❌ 所有摄像头尝试失败，使用虚拟模式") + print("[ERROR] 所有摄像头尝试失败，使用虚拟模式") return None def _gesture_recognition_loop(self): @@ -223,7 +223,7 @@ def _gesture_recognition_loop(self): # 显示虚拟模式提示（如果摄像头未连接） if self.cap is None: - print("⚠️ 使用虚拟摄像头模式，请连接摄像头进行真实手势识别") + print("[WARNING] 使用虚拟摄像头模式，请连接摄像头进行真实手势识别") while self.running: if self.paused: @@ -410,7 +410,7 @@ def _process_gesture_command(self, gesture, confidence): # 添加调试信息 print( - f"🎯 检测到手势: {gesture} (置信度: {confidence:.2f}, 阈值: {threshold}) -> 执行: {command} (强度: {intensity:.2f})") + f"[INFO] 检测到手势: {gesture} (置信度: {confidence:.2f}, 阈值: {threshold}) -> 执行: {command} (强度: {intensity:.2f})") # 发送命令到控制器 self.drone_controller.send_command(command, intensity) @@ -446,7 +446,7 @@ def _simulation_loop(self): target_fps = 60 frame_delay = 1.0 / target_fps - print("\n🎮 键盘提示：按 'R' 键重置无人机位置到原点") + print("\n[INFO] 键盘提示：按 'R' 键重置无人机位置到原点") print(" 按 'T' 键手动起飞") print(" 按 'L' 键手动降落") print(" 按 'H' 键悬停") @@ -485,7 +485,7 @@ def _simulation_loop(self): if keys[pygame.K_r]: if ('r' not in self._last_key_press or current_time - self._last_key_press['r'] > 1.0): - print("🎮 键盘：重置无人机位置") + print("[INFO] 键盘：重置无人机位置") self.drone_controller.reset() print(" 无人机已重置到原点位置") self._last_key_press['r'] = current_time @@ -494,7 +494,7 @@ def _simulation_loop(self): if keys[pygame.K_t]: if ('t' not in self._last_key_press or current_time - self._last_key_press['t'] > 1.0): - print("🎮 键盘：起飞") + print("[INFO] 键盘：起飞") self.drone_controller.send_command("takeoff", 0.8) self._last_key_press['t'] = current_time @@ -502,7 +502,7 @@ def _simulation_loop(self): if keys[pygame.K_l]: if ('l' not in self._last_key_press or current_time - self._last_key_press['l'] > 1.0): - print("🎮 键盘：降落") + print("[INFO] 键盘：降落") self.drone_controller.send_command("land", 0.5) self._last_key_press['l'] = current_time @@ -510,7 +510,7 @@ def _simulation_loop(self): if keys[pygame.K_h]: if ('h' not in self._last_key_press or current_time - self._last_key_press['h'] > 1.0): - print("🎮 键盘：悬停") + print("[INFO] 键盘：悬停") self.drone_controller.send_command("hover") self._last_key_press['h'] = current_time @@ -518,7 +518,7 @@ def _simulation_loop(self): if keys[pygame.K_s]: if ('s' not in self._last_key_press or current_time - self._last_key_press['s'] > 1.0): - print("🎮 键盘：停止") + print("[INFO] 键盘：停止") self.drone_controller.send_command("stop") self._last_key_press['s'] = current_time @@ -707,7 +707,7 @@ def run(self): # 保存日志 self._save_log() - print("无人机仿真系统已安全关闭 ✓") + print("无人机仿真系统已安全关闭 [OK]") def load_config(): @@ -732,18 +732,18 @@ def load_config(): try: import pygame - print("✅ Pygame 已安装") + print("[OK] Pygame 已安装") except ImportError: - print("❌ 错误: Pygame 未安装!") + print("[ERROR] 错误: Pygame 未安装!") print("请运行: pip install pygame") sys.exit(1) try: import OpenGL - print("✅ PyOpenGL 已安装") + print("[OK] PyOpenGL 已安装") except ImportError: - print("❌ 错误: PyOpenGL 未安装!") + print("[ERROR] 错误: PyOpenGL 未安装!") print("请运行: pip install PyOpenGL PyOpenGL-accelerate") sys.exit(1) diff --git "a/src/drone_hand_gesture/\350\277\220\350\241\214\346\214\207\345\215\227.md" "b/src/drone_hand_gesture/\350\277\220\350\241\214\346\214\207\345\215\227.md" index 86d461764..fdec1dfbc 100644 --- "a/src/drone_hand_gesture/\350\277\220\350\241\214\346\214\207\345\215\227.md" +++ "b/src/drone_hand_gesture/\350\277\220\350\241\214\346\214\207\345\215\227.md" @@ -59,7 +59,7 @@ 2. **进入项目目录**： ```bash - cd d:\机械学习\nn\src\drone_hand_gesture + cd path/to/nn/src/drone_hand_gesture ``` 3. **运行 AirSim 版本**： From fc6cc5023e170002cd8f132a9ccd5bb189ab0b80 Mon Sep 17 00:00:00 2001 From: springstart11111 <1312982847@qq.com> Date: Sun, 12 Apr 2026 18:28:55 +0800 Subject: [PATCH 06/11] =?UTF-8?q?feat:=20=E5=8D=87=E7=BA=A7=E7=95=8C?= =?UTF-8?q?=E9=9D=A2=E7=BE=8E=E8=A7=82=E5=BA=A6=EF=BC=8C=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E6=8C=89=E9=94=AE=E5=8A=9F=E8=83=BD=E5=88=B0=E6=89=8B=E5=8A=BF?= =?UTF-8?q?=E6=8E=A7=E5=88=B6=E7=AA=97=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/drone_hand_gesture/gesture_detector.py | 3 +- .../gesture_detector_enhanced.py | 81 +++++++++++++++---- src/drone_hand_gesture/main.py | 28 ++++++- src/drone_hand_gesture/main_airsim.py | 11 +-- 4 files changed, 96 insertions(+), 27 deletions(-) diff --git a/src/drone_hand_gesture/gesture_detector.py b/src/drone_hand_gesture/gesture_detector.py index 9bf468686..3a9547a6a 100644 --- a/src/drone_hand_gesture/gesture_detector.py +++ b/src/drone_hand_gesture/gesture_detector.py @@ -33,13 +33,14 @@ def __init__(self): "ok_sign": "hover" # OK手势 - 悬停 } - def detect_gestures(self, image, simulation_mode=False): + def detect_gestures(self, image, simulation_mode=False, fps=0): """ 检测图像中的手势 Args: image: 输入图像 simulation_mode: 是否为仿真模式 + fps: 帧率（可选） Returns: processed_image: 处理后的图像 diff --git a/src/drone_hand_gesture/gesture_detector_enhanced.py b/src/drone_hand_gesture/gesture_detector_enhanced.py index ff09d3871..958885f55 100644 --- a/src/drone_hand_gesture/gesture_detector_enhanced.py +++ b/src/drone_hand_gesture/gesture_detector_enhanced.py @@ -152,7 +152,7 @@ def extract_landmarks_for_ml(self, hand_landmarks): return landmarks - def detect_gestures(self, image, simulation_mode=False): + def detect_gestures(self, image, simulation_mode=False, fps=0): """检测手势（支持中文显示）""" # 转换为RGB image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) @@ -162,15 +162,30 @@ def detect_gestures(self, image, simulation_mode=False): confidence = 0.0 landmarks_data = None + # 创建一个美观的界面 + # 1. 创建背景 + height, width = image.shape[:2] + display_width = min(width, 800) + display_height = min(height, 600) + + # 调整图像大小 + image = cv2.resize(image, (display_width, display_height)) + height, width = image.shape[:2] + + # 2. 添加半透明背景 + overlay = image.copy() + cv2.rectangle(overlay, (0, 0), (width, 150), (0, 0, 0), -1) + cv2.addWeighted(overlay, 0.7, image, 0.3, 0, image) + if results.multi_hand_landmarks: for hand_landmarks in results.multi_hand_landmarks: - # 绘制手部关键点（使用OpenCV绘制） + # 绘制手部关键点（使用更美观的样式） self.mp_drawing.draw_landmarks( image, hand_landmarks, self.mp_hands.HAND_CONNECTIONS, - self.mp_drawing_styles.get_default_hand_landmarks_style(), - self.mp_drawing_styles.get_default_hand_connections_style() + mp.solutions.drawing_styles.get_default_hand_landmarks_style(), + mp.solutions.drawing_styles.get_default_hand_connections_style() ) # 提取关键点用于机器学习 @@ -211,24 +226,33 @@ def detect_gestures(self, image, simulation_mode=False): # 设置字体 if self.chinese_font: font = self.chinese_font + font_small = ImageFont.truetype(self.chinese_font.path, 20) else: # 如果中文字体不可用，使用默认字体 font = ImageFont.load_default() + font_small = font print("⚠️ 使用默认字体，中文可能显示为方块") + # 绘制标题 + draw.text((20, 10), "手势控制无人机", fill=(255, 255, 255), font=font) + # 绘制手势信息 chinese_gesture = self.chinese_gesture_names.get(gesture, gesture) ml_info = " (训练模型)" if self.use_ml else " (规则)" text = f"手势: {chinese_gesture}{ml_info}" - draw.text((10, 30), text, fill=(0, 255, 0), font=font) + draw.text((20, 50), text, fill=(0, 255, 0), font=font_small) # 绘制置信度 - draw.text((10, 70), f"置信度: {confidence:.2f}", fill=(0, 255, 0), font=font) + draw.text((20, 80), f"置信度: {confidence:.2f}", fill=(255, 255, 0), font=font_small) # 绘制指令 command = self.gesture_commands.get(gesture, "none") chinese_command = self.chinese_command_names.get(command, command) - draw.text((10, 110), f"指令: {chinese_command}", fill=(255, 0, 0), font=font) + draw.text((20, 110), f"指令: {chinese_command}", fill=(0, 255, 255), font=font_small) + + # 绘制FPS + if fps > 0: + draw.text((width - 120, 10), f"FPS: {fps:.1f}", fill=(255, 255, 255), font=font_small) # 将PIL图像转换回OpenCV格式 image = cv2.cvtColor(np.array(image_pil), cv2.COLOR_RGB2BGR) @@ -237,12 +261,17 @@ def detect_gestures(self, image, simulation_mode=False): print(f"PIL绘制失败，回退到OpenCV英文显示: {e}") # 回退到英文显示 ml_info = " (ML)" if self.use_ml else " (Rule)" - cv2.putText(image, f"Gesture: {gesture}{ml_info}", (10, 30), - cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2) - cv2.putText(image, f"Confidence: {confidence:.2f}", (10, 70), + cv2.putText(image, "Drone Gesture Control", (20, 30), + cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2) + cv2.putText(image, f"Gesture: {gesture}{ml_info}", (20, 60), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) - cv2.putText(image, f"Command: {command}", (10, 110), - cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 2) + cv2.putText(image, f"Confidence: {confidence:.2f}", (20, 90), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 0), 2) + cv2.putText(image, f"Command: {command}", (20, 120), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 255), 2) + if fps > 0: + cv2.putText(image, f"FPS: {fps:.1f}", (width - 100, 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2) else: # 没有检测到手部时也使用PIL绘制 @@ -253,18 +282,38 @@ def detect_gestures(self, image, simulation_mode=False): if self.chinese_font: font = self.chinese_font + font_small = ImageFont.truetype(self.chinese_font.path, 20) else: font = ImageFont.load_default() - - draw.text((10, 30), "未检测到手部", fill=(0, 0, 255), font=font) + font_small = font + + # 绘制标题 + draw.text((20, 10), "手势控制无人机", fill=(255, 255, 255), font=font) + # 绘制未检测到手部 + draw.text((20, 50), "未检测到手部", fill=(0, 0, 255), font=font_small) + draw.text((20, 80), "请将手放在摄像头前", fill=(255, 255, 255), font=font_small) + + # 绘制FPS + if fps > 0: + draw.text((width - 120, 10), f"FPS: {fps:.1f}", fill=(255, 255, 255), font=font_small) # 转换回OpenCV格式 image = cv2.cvtColor(np.array(image_pil), cv2.COLOR_RGB2BGR) except Exception as e: print(f"PIL绘制失败: {e}") - cv2.putText(image, "No Hand Detected", (10, 30), - cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) + cv2.putText(image, "Drone Gesture Control", (20, 30), + cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2) + cv2.putText(image, "No Hand Detected", (20, 60), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) + cv2.putText(image, "Please place your hand in front of the camera", (20, 90), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2) + if fps > 0: + cv2.putText(image, f"FPS: {fps:.1f}", (width - 100, 30), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2) + + # 添加边框 + cv2.rectangle(image, (0, 0), (width-1, height-1), (0, 255, 0), 2) return image, gesture, confidence, landmarks_data diff --git a/src/drone_hand_gesture/main.py b/src/drone_hand_gesture/main.py index df16c242e..a8a4ca8fd 100644 --- a/src/drone_hand_gesture/main.py +++ b/src/drone_hand_gesture/main.py @@ -136,6 +136,9 @@ def __init__(self, config=None): self.control_intensity = 1.0 self.last_command_time = time.time() self.command_cooldown = 1.5 # 命令冷却时间（秒），从2.0降低到1.5 + + # 帧率计算 + self.last_frame_time = time.time() # 手势识别阈值（降低以提高灵敏度） # 如果是机器学习模式，阈值可以进一步降低 @@ -270,10 +273,15 @@ def _gesture_recognition_loop(self): cv2.putText(frame, "按 'q' 键退出", (50, 400), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 2) + # 计算帧率 + current_time = time.time() + fps = 1.0 / (current_time - getattr(self, 'last_frame_time', current_time)) + self.last_frame_time = current_time + # 手势检测 try: processed_frame, gesture, confidence, landmarks = \ - self.gesture_detector.detect_gestures(frame, simulation_mode=True) + self.gesture_detector.detect_gestures(frame, simulation_mode=True, fps=fps) # 更新共享数据 self.current_frame = processed_frame @@ -293,7 +301,7 @@ def _gesture_recognition_loop(self): self.current_gesture = None # 检查退出 - key = cv2.waitKey(1) & 0xFF + key = cv2.waitKey(10) & 0xFF if key == ord('q'): print("收到退出指令...") self.running = False @@ -305,6 +313,22 @@ def _gesture_recognition_loop(self): self._debug_gesture_detection() elif key == ord('m'): # 切换模式（如果有多个模型） self._switch_detection_mode() + elif key == ord('r'): # 重置无人机位置 + print("[INFO] 键盘：重置无人机位置") + self.drone_controller.reset() + print(" 无人机已重置到原点位置") + elif key == ord('t'): # 手动起飞 + print("[INFO] 键盘：起飞") + self.drone_controller.send_command("takeoff", 0.8) + elif key == ord('l'): # 手动降落 + print("[INFO] 键盘：降落") + self.drone_controller.send_command("land", 0.5) + elif key == ord('h'): # 悬停 + print("[INFO] 键盘：悬停") + self.drone_controller.send_command("hover") + elif key == ord('s'): # 停止 + print("[INFO] 键盘：停止") + self.drone_controller.send_command("stop") print("手势识别线程结束") diff --git a/src/drone_hand_gesture/main_airsim.py b/src/drone_hand_gesture/main_airsim.py index 209461a8e..d64aa8bb8 100644 --- a/src/drone_hand_gesture/main_airsim.py +++ b/src/drone_hand_gesture/main_airsim.py @@ -104,17 +104,12 @@ def main(show_window=True): frame_count += 1 - # 手势识别 - debug_frame, gesture, confidence, _ = detector.detect_gestures(frame) - - # 显示帧率 + # 计算帧率 elapsed = time.time() - start_time fps = frame_count / elapsed if elapsed > 0 else 0 - cv2.putText(debug_frame, f"FPS: {fps:.1f}", (10, 30), - cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2) - cv2.putText(debug_frame, f"Gesture: {gesture} ({confidence:.2f})", - (10, 60), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2) + # 手势识别 + debug_frame, gesture, confidence, _ = detector.detect_gestures(frame, fps=fps) # 处理手势 if gesture and gesture != "none" and gesture != "no_hand": From 08b2ce07182ed79dc0002e34e84f862f3561f8d0 Mon Sep 17 00:00:00 2001 From: springstart11111 <1312982847@qq.com> Date: Mon, 13 Apr 2026 12:47:29 +0800 Subject: [PATCH 07/11] =?UTF-8?q?feat:=20=E5=8D=87=E7=BA=A7=E6=9C=AC?= =?UTF-8?q?=E5=9C=B0=E8=BF=90=E8=A1=8C=E7=95=8C=E9=9D=A2=EF=BC=8C=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E4=BF=A1=E6=81=AF=E9=9D=A2=E6=9D=BF=E5=92=8C=E7=8A=B6?= =?UTF-8?q?=E6=80=81=E6=98=BE=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/drone_hand_gesture/drone_controller.py | 8 +- src/drone_hand_gesture/main.py | 189 ++++++++++++++++++--- 2 files changed, 166 insertions(+), 31 deletions(-) diff --git a/src/drone_hand_gesture/drone_controller.py b/src/drone_hand_gesture/drone_controller.py index d8218a34f..100276277 100644 --- a/src/drone_hand_gesture/drone_controller.py +++ b/src/drone_hand_gesture/drone_controller.py @@ -102,9 +102,9 @@ def _takeoff_simulation(self, intensity): target_height = self.control_params['takeoff_altitude'] * intensity # 设置向上的速度 self.state['velocity'][1] = 1.0 * intensity # Y轴向上 - print(f"✅ 仿真：无人机已解锁并起飞到 {target_height:.1f} 米高度") + print(f"[OK] 仿真：无人机已解锁并起飞到 {target_height:.1f} 米高度") else: - print("⚠️ 无人机已经解锁，无需再次起飞") + print("[WARNING] 无人机已经解锁，无需再次起飞") def _land_simulation(self, intensity): """仿真降落""" @@ -119,7 +119,7 @@ def _move_simulation(self, direction, intensity): print(f"[DEBUG] 当前armed状态: {self.state['armed']}") if not self.state['armed']: - print("❌ 警告：无人机未解锁，无法移动") + print("[ERROR] 警告：无人机未解锁，无法移动") print(" 请先做出'张开手掌'手势进行起飞解锁") return @@ -144,7 +144,7 @@ def _move_simulation(self, direction, intensity): self.state['velocity'][0] = speed # 向右（X轴正方向） self.state['mode'] = 'RIGHT' - print(f"✅ 仿真：无人机{direction}移动，速度{speed:.1f}m/s") + print(f"[OK] 仿真：无人机{direction}移动，速度{speed:.1f}m/s") def _hover_simulation(self): """仿真悬停""" diff --git a/src/drone_hand_gesture/main.py b/src/drone_hand_gesture/main.py index a8a4ca8fd..64e50e1e6 100644 --- a/src/drone_hand_gesture/main.py +++ b/src/drone_hand_gesture/main.py @@ -136,9 +136,6 @@ def __init__(self, config=None): self.control_intensity = 1.0 self.last_command_time = time.time() self.command_cooldown = 1.5 # 命令冷却时间（秒），从2.0降低到1.5 - - # 帧率计算 - self.last_frame_time = time.time() # 手势识别阈值（降低以提高灵敏度） # 如果是机器学习模式，阈值可以进一步降低 @@ -273,15 +270,10 @@ def _gesture_recognition_loop(self): cv2.putText(frame, "按 'q' 键退出", (50, 400), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 2) - # 计算帧率 - current_time = time.time() - fps = 1.0 / (current_time - getattr(self, 'last_frame_time', current_time)) - self.last_frame_time = current_time - # 手势检测 try: processed_frame, gesture, confidence, landmarks = \ - self.gesture_detector.detect_gestures(frame, simulation_mode=True, fps=fps) + self.gesture_detector.detect_gestures(frame, simulation_mode=True) # 更新共享数据 self.current_frame = processed_frame @@ -292,17 +284,24 @@ def _gesture_recognition_loop(self): # 处理手势命令（使用降低的阈值） self._process_gesture_command(gesture, confidence) + # 增强界面显示 + enhanced_frame = self._enhance_interface(processed_frame, gesture, confidence) + # 显示手势识别窗口 - cv2.imshow('Gesture Control', processed_frame) + cv2.imshow('Gesture Control', enhanced_frame) except Exception as e: print(f"手势检测错误: {e}") self.current_frame = frame self.current_gesture = None + # 增强界面显示（错误情况） + enhanced_frame = self._enhance_interface(frame, "error", 0.0) + cv2.imshow('Gesture Control', enhanced_frame) + # 检查退出 key = cv2.waitKey(10) & 0xFF - if key == ord('q'): + if key == ord('q') or key == 27: # q 或 ESC 键退出 print("收到退出指令...") self.running = False break @@ -313,25 +312,161 @@ def _gesture_recognition_loop(self): self._debug_gesture_detection() elif key == ord('m'): # 切换模式（如果有多个模型） self._switch_detection_mode() - elif key == ord('r'): # 重置无人机位置 - print("[INFO] 键盘：重置无人机位置") - self.drone_controller.reset() - print(" 无人机已重置到原点位置") - elif key == ord('t'): # 手动起飞 - print("[INFO] 键盘：起飞") - self.drone_controller.send_command("takeoff", 0.8) - elif key == ord('l'): # 手动降落 - print("[INFO] 键盘：降落") - self.drone_controller.send_command("land", 0.5) - elif key == ord('h'): # 悬停 - print("[INFO] 键盘：悬停") - self.drone_controller.send_command("hover") - elif key == ord('s'): # 停止 - print("[INFO] 键盘：停止") - self.drone_controller.send_command("stop") + elif key == ord('h'): # 显示帮助 + self._show_help() + elif key == ord('f'): # 切换全屏 + self._toggle_fullscreen() print("手势识别线程结束") + def _enhance_interface(self, frame, gesture, confidence): + """增强界面显示""" + # 创建一个更大的画布，包含摄像头画面和信息面板 + height, width = frame.shape[:2] + panel_width = 300 + total_width = width + panel_width + enhanced_frame = np.ones((height, total_width, 3), dtype=np.uint8) * 20 # 深灰色背景 + + # 复制摄像头画面 + enhanced_frame[:, :width] = frame + + # 绘制信息面板边框 + cv2.rectangle(enhanced_frame, (width, 0), (total_width, height), (50, 50, 50), 2) + + # 显示标题 + cv2.putText(enhanced_frame, "DRONE CONTROL", (width + 20, 40), + cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 255, 255), 2) + + # 显示手势信息 + y_offset = 80 + if gesture and gesture != "no_hand": + # 手势名称 + cv2.putText(enhanced_frame, f"GESTURE: {gesture.upper()}", + (width + 20, y_offset), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 1) + y_offset += 30 + + # 置信度 + confidence_color = (0, 255, 0) if confidence > 0.7 else (0, 255, 255) if confidence > 0.5 else (0, 0, 255) + cv2.putText(enhanced_frame, f"CONFIDENCE: {confidence:.2f}", + (width + 20, y_offset), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, confidence_color, 1) + y_offset += 40 + else: + cv2.putText(enhanced_frame, "GESTURE: NO HAND", + (width + 20, y_offset), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 0), 1) + y_offset += 40 + + # 显示无人机状态 + cv2.putText(enhanced_frame, "DRONE STATUS", + (width + 20, y_offset), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 1) + y_offset += 25 + + # 获取无人机状态 + drone_state = self.drone_controller.get_state() + + # 状态信息 + status_info = [ + f"MODE: {drone_state['mode'].upper()}", + f"ARMED: {'YES' if drone_state['armed'] else 'NO'}", + f"BATTERY: {drone_state['battery']:.1f}%", + f"ALTITUDE: {abs(drone_state['position'][2]):.2f}m" + ] + + for info in status_info: + cv2.putText(enhanced_frame, info, + (width + 20, y_offset), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (150, 150, 255), 1) + y_offset += 20 + + y_offset += 10 + + # 显示位置信息 + cv2.putText(enhanced_frame, "POSITION", + (width + 20, y_offset), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 1) + y_offset += 25 + + pos = drone_state['position'] + cv2.putText(enhanced_frame, f"X: {pos[0]:.2f}m", + (width + 20, y_offset), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (150, 255, 150), 1) + y_offset += 15 + cv2.putText(enhanced_frame, f"Y: {pos[1]:.2f}m", + (width + 20, y_offset), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (150, 255, 150), 1) + y_offset += 15 + cv2.putText(enhanced_frame, f"Z: {pos[2]:.2f}m", + (width + 20, y_offset), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (150, 255, 150), 1) + y_offset += 30 + + # 显示控制提示 + cv2.putText(enhanced_frame, "CONTROLS", + (width + 20, y_offset), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 1) + y_offset += 25 + + controls = [ + "Q/ESC: Exit", + "C: Switch Camera", + "D: Debug Info", + "H: Help", + "F: Fullscreen" + ] + + for control in controls: + cv2.putText(enhanced_frame, control, + (width + 20, y_offset), + cv2.FONT_HERSHEY_SIMPLEX, 0.45, (200, 200, 200), 1) + y_offset += 15 + + # 显示帧率 + current_time = time.time() + if hasattr(self, 'last_frame_time'): + fps = 1.0 / (current_time - self.last_frame_time) + cv2.putText(enhanced_frame, f"FPS: {fps:.1f}", + (width + 20, height - 20), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) + self.last_frame_time = current_time + + return enhanced_frame + + def _show_help(self): + """显示帮助信息""" + print("=" * 60) + print("手势控制无人机 - 帮助信息") + print("=" * 60) + print("手势指令:") + print(" 张开手掌 - 起飞") + print(" 握拳 - 降落") + print(" 胜利手势 - 前进") + print(" 大拇指 - 后退") + print(" 食指上指 - 上升") + print(" 食指向下 - 下降") + print(" OK手势 - 悬停") + print(" 大拇指向下 - 停止") + print("=" * 60) + print("键盘控制:") + print(" Q/ESC - 退出") + print(" C - 切换摄像头") + print(" D - 显示调试信息") + print(" H - 显示帮助") + print(" F - 切换全屏") + print(" R - 重置无人机位置") + print(" T - 手动起飞") + print(" L - 手动降落") + print(" H - 悬停") + print(" S - 停止") + print("=" * 60) + + def _toggle_fullscreen(self): + """切换全屏模式""" + # 简化实现，实际需要更复杂的窗口管理 + print("全屏模式切换功能已触发") + def _switch_detection_mode(self): """切换检测模式（如果有多个可用模型）""" if not HAS_ENHANCED_DETECTOR: From bb0adb568fbb17c5dfe29b5ef20cf8df1e76fff8 Mon Sep 17 00:00:00 2001 From: springstart11111 <1312982847@qq.com> Date: Mon, 13 Apr 2026 12:57:42 +0800 Subject: [PATCH 08/11] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E7=BC=96?= =?UTF-8?q?=E7=A0=81=E9=97=AE=E9=A2=98=E5=92=8C=E7=95=8C=E9=9D=A2=E5=A2=9E?= =?UTF-8?q?=E5=BC=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/drone_hand_gesture/gesture_detector.py | 3 +- .../gesture_detector_enhanced.py | 81 ++++--------------- src/drone_hand_gesture/main_airsim.py | 11 ++- 3 files changed, 25 insertions(+), 70 deletions(-) diff --git a/src/drone_hand_gesture/gesture_detector.py b/src/drone_hand_gesture/gesture_detector.py index 3a9547a6a..9bf468686 100644 --- a/src/drone_hand_gesture/gesture_detector.py +++ b/src/drone_hand_gesture/gesture_detector.py @@ -33,14 +33,13 @@ def __init__(self): "ok_sign": "hover" # OK手势 - 悬停 } - def detect_gestures(self, image, simulation_mode=False, fps=0): + def detect_gestures(self, image, simulation_mode=False): """ 检测图像中的手势 Args: image: 输入图像 simulation_mode: 是否为仿真模式 - fps: 帧率（可选） Returns: processed_image: 处理后的图像 diff --git a/src/drone_hand_gesture/gesture_detector_enhanced.py b/src/drone_hand_gesture/gesture_detector_enhanced.py index 958885f55..ff09d3871 100644 --- a/src/drone_hand_gesture/gesture_detector_enhanced.py +++ b/src/drone_hand_gesture/gesture_detector_enhanced.py @@ -152,7 +152,7 @@ def extract_landmarks_for_ml(self, hand_landmarks): return landmarks - def detect_gestures(self, image, simulation_mode=False, fps=0): + def detect_gestures(self, image, simulation_mode=False): """检测手势（支持中文显示）""" # 转换为RGB image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) @@ -162,30 +162,15 @@ def detect_gestures(self, image, simulation_mode=False, fps=0): confidence = 0.0 landmarks_data = None - # 创建一个美观的界面 - # 1. 创建背景 - height, width = image.shape[:2] - display_width = min(width, 800) - display_height = min(height, 600) - - # 调整图像大小 - image = cv2.resize(image, (display_width, display_height)) - height, width = image.shape[:2] - - # 2. 添加半透明背景 - overlay = image.copy() - cv2.rectangle(overlay, (0, 0), (width, 150), (0, 0, 0), -1) - cv2.addWeighted(overlay, 0.7, image, 0.3, 0, image) - if results.multi_hand_landmarks: for hand_landmarks in results.multi_hand_landmarks: - # 绘制手部关键点（使用更美观的样式） + # 绘制手部关键点（使用OpenCV绘制） self.mp_drawing.draw_landmarks( image, hand_landmarks, self.mp_hands.HAND_CONNECTIONS, - mp.solutions.drawing_styles.get_default_hand_landmarks_style(), - mp.solutions.drawing_styles.get_default_hand_connections_style() + self.mp_drawing_styles.get_default_hand_landmarks_style(), + self.mp_drawing_styles.get_default_hand_connections_style() ) # 提取关键点用于机器学习 @@ -226,33 +211,24 @@ def detect_gestures(self, image, simulation_mode=False, fps=0): # 设置字体 if self.chinese_font: font = self.chinese_font - font_small = ImageFont.truetype(self.chinese_font.path, 20) else: # 如果中文字体不可用，使用默认字体 font = ImageFont.load_default() - font_small = font print("⚠️ 使用默认字体，中文可能显示为方块") - # 绘制标题 - draw.text((20, 10), "手势控制无人机", fill=(255, 255, 255), font=font) - # 绘制手势信息 chinese_gesture = self.chinese_gesture_names.get(gesture, gesture) ml_info = " (训练模型)" if self.use_ml else " (规则)" text = f"手势: {chinese_gesture}{ml_info}" - draw.text((20, 50), text, fill=(0, 255, 0), font=font_small) + draw.text((10, 30), text, fill=(0, 255, 0), font=font) # 绘制置信度 - draw.text((20, 80), f"置信度: {confidence:.2f}", fill=(255, 255, 0), font=font_small) + draw.text((10, 70), f"置信度: {confidence:.2f}", fill=(0, 255, 0), font=font) # 绘制指令 command = self.gesture_commands.get(gesture, "none") chinese_command = self.chinese_command_names.get(command, command) - draw.text((20, 110), f"指令: {chinese_command}", fill=(0, 255, 255), font=font_small) - - # 绘制FPS - if fps > 0: - draw.text((width - 120, 10), f"FPS: {fps:.1f}", fill=(255, 255, 255), font=font_small) + draw.text((10, 110), f"指令: {chinese_command}", fill=(255, 0, 0), font=font) # 将PIL图像转换回OpenCV格式 image = cv2.cvtColor(np.array(image_pil), cv2.COLOR_RGB2BGR) @@ -261,17 +237,12 @@ def detect_gestures(self, image, simulation_mode=False, fps=0): print(f"PIL绘制失败，回退到OpenCV英文显示: {e}") # 回退到英文显示 ml_info = " (ML)" if self.use_ml else " (Rule)" - cv2.putText(image, "Drone Gesture Control", (20, 30), - cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2) - cv2.putText(image, f"Gesture: {gesture}{ml_info}", (20, 60), + cv2.putText(image, f"Gesture: {gesture}{ml_info}", (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2) + cv2.putText(image, f"Confidence: {confidence:.2f}", (10, 70), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2) - cv2.putText(image, f"Confidence: {confidence:.2f}", (20, 90), - cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 0), 2) - cv2.putText(image, f"Command: {command}", (20, 120), - cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 255), 2) - if fps > 0: - cv2.putText(image, f"FPS: {fps:.1f}", (width - 100, 30), - cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2) + cv2.putText(image, f"Command: {command}", (10, 110), + cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 2) else: # 没有检测到手部时也使用PIL绘制 @@ -282,38 +253,18 @@ def detect_gestures(self, image, simulation_mode=False, fps=0): if self.chinese_font: font = self.chinese_font - font_small = ImageFont.truetype(self.chinese_font.path, 20) else: font = ImageFont.load_default() - font_small = font - - # 绘制标题 - draw.text((20, 10), "手势控制无人机", fill=(255, 255, 255), font=font) - # 绘制未检测到手部 - draw.text((20, 50), "未检测到手部", fill=(0, 0, 255), font=font_small) - draw.text((20, 80), "请将手放在摄像头前", fill=(255, 255, 255), font=font_small) - - # 绘制FPS - if fps > 0: - draw.text((width - 120, 10), f"FPS: {fps:.1f}", fill=(255, 255, 255), font=font_small) + + draw.text((10, 30), "未检测到手部", fill=(0, 0, 255), font=font) # 转换回OpenCV格式 image = cv2.cvtColor(np.array(image_pil), cv2.COLOR_RGB2BGR) except Exception as e: print(f"PIL绘制失败: {e}") - cv2.putText(image, "Drone Gesture Control", (20, 30), - cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2) - cv2.putText(image, "No Hand Detected", (20, 60), - cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) - cv2.putText(image, "Please place your hand in front of the camera", (20, 90), - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2) - if fps > 0: - cv2.putText(image, f"FPS: {fps:.1f}", (width - 100, 30), - cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2) - - # 添加边框 - cv2.rectangle(image, (0, 0), (width-1, height-1), (0, 255, 0), 2) + cv2.putText(image, "No Hand Detected", (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) return image, gesture, confidence, landmarks_data diff --git a/src/drone_hand_gesture/main_airsim.py b/src/drone_hand_gesture/main_airsim.py index d64aa8bb8..209461a8e 100644 --- a/src/drone_hand_gesture/main_airsim.py +++ b/src/drone_hand_gesture/main_airsim.py @@ -104,12 +104,17 @@ def main(show_window=True): frame_count += 1 - # 计算帧率 + # 手势识别 + debug_frame, gesture, confidence, _ = detector.detect_gestures(frame) + + # 显示帧率 elapsed = time.time() - start_time fps = frame_count / elapsed if elapsed > 0 else 0 - # 手势识别 - debug_frame, gesture, confidence, _ = detector.detect_gestures(frame, fps=fps) + cv2.putText(debug_frame, f"FPS: {fps:.1f}", (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2) + cv2.putText(debug_frame, f"Gesture: {gesture} ({confidence:.2f})", + (10, 60), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2) # 处理手势 if gesture and gesture != "none" and gesture != "no_hand": From 01d7006a6d1ab56c1f61ef2efa025b62931bb6ce Mon Sep 17 00:00:00 2001 From: springstart11111 <1312982847@qq.com> Date: Wed, 15 Apr 2026 20:32:12 +0800 Subject: [PATCH 09/11] =?UTF-8?q?chore:=20=E5=88=A0=E9=99=A4=E9=87=8D?= =?UTF-8?q?=E5=A4=8D=E7=9A=84=E4=B8=AD=E6=96=87=E8=84=9A=E6=9C=AC=E6=96=87?= =?UTF-8?q?=E4=BB=B6=EF=BC=8C=E5=8F=AA=E4=BF=9D=E7=95=99=E8=8B=B1=E6=96=87?= =?UTF-8?q?=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...7\220\350\241\214_AirSim \347\211\210.bat" | 32 ------------------- ...4\345\234\260\344\273\277\347\234\237.bat" | 16 ---------- 2 files changed, 48 deletions(-) delete mode 100644 "src/drone_hand_gesture/\350\277\220\350\241\214_AirSim \347\211\210.bat" delete mode 100644 "src/drone_hand_gesture/\350\277\220\350\241\214_\346\234\254\345\234\260\344\273\277\347\234\237.bat" diff --git "a/src/drone_hand_gesture/\350\277\220\350\241\214_AirSim \347\211\210.bat" "b/src/drone_hand_gesture/\350\277\220\350\241\214_AirSim \347\211\210.bat" deleted file mode 100644 index be7b964e7..000000000 --- "a/src/drone_hand_gesture/\350\277\220\350\241\214_AirSim \347\211\210.bat" +++ /dev/null @@ -1,32 +0,0 @@ -@echo off -chcp 65001 >nul -echo ================================================ -echo 手势控制无人机 - AirSim 真实模拟器版 -echo ================================================ -echo. -echo 检查 AirSim 是否运行... -echo. - -REM 检查 AirSim 是否运行 -tasklist /FI "WINDOWTITLE eq Blocks" 2>nul | find "Blocks.exe" >nul -if %ERRORLEVEL% NEQ 0 ( - echo [警告] AirSim 未运行！ - echo. - echo 请先启动 AirSim 模拟器: - echo 双击运行：d:\机械学习\air\Blocks\WindowsNoEditor\Blocks.exe - echo. - echo 按任意键继续（如果 AirSim 已启动）... - pause >nul -) - -echo. -echo 正在启动手势控制程序... -echo. - -cd /d "%~dp0" -python main_airsim.py - -echo. -echo ================================================ -echo 程序已退出 -pause diff --git "a/src/drone_hand_gesture/\350\277\220\350\241\214_\346\234\254\345\234\260\344\273\277\347\234\237.bat" "b/src/drone_hand_gesture/\350\277\220\350\241\214_\346\234\254\345\234\260\344\273\277\347\234\237.bat" deleted file mode 100644 index a5393cb4c..000000000 --- "a/src/drone_hand_gesture/\350\277\220\350\241\214_\346\234\254\345\234\260\344\273\277\347\234\237.bat" +++ /dev/null @@ -1,16 +0,0 @@ -@echo off -chcp 65001 >nul -echo ================================================ -echo 手势控制无人机 - 本地仿真版 -echo ================================================ -echo. -echo 正在启动... -echo. - -cd /d "%~dp0" -python main.py - -echo. -echo ================================================ -echo 程序已退出 -pause From 069e2bf9e132b6bd5f27d4fc8e3a1d132c661ddb Mon Sep 17 00:00:00 2001 From: springstart11111 <1312982847@qq.com> Date: Fri, 17 Apr 2026 00:07:29 +0800 Subject: [PATCH 10/11] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=203D=20?= =?UTF-8?q?=E4=BB=BF=E7=9C=9F=E7=AA=97=E5=8F=A3=E6=96=87=E6=9C=AC=E7=BB=98?= =?UTF-8?q?=E5=88=B6=E9=97=AE=E9=A2=98=EF=BC=8C=E7=A6=81=E7=94=A8=E5=BC=82?= =?UTF-8?q?=E5=B8=B8=E7=9A=84=E7=8A=B6=E6=80=81=E4=BF=A1=E6=81=AF=E6=98=BE?= =?UTF-8?q?=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/drone_hand_gesture/simulation_3d.py | 105 +----------------------- 1 file changed, 4 insertions(+), 101 deletions(-) diff --git a/src/drone_hand_gesture/simulation_3d.py b/src/drone_hand_gesture/simulation_3d.py index fe04eb30d..1286189f8 100644 --- a/src/drone_hand_gesture/simulation_3d.py +++ b/src/drone_hand_gesture/simulation_3d.py @@ -193,110 +193,13 @@ def _draw_trajectory(self, trajectory): def _draw_status_overlay(self, state): """绘制状态信息覆盖层""" - # 切换到2D模式 - glMatrixMode(GL_PROJECTION) - glPushMatrix() - glLoadIdentity() - gluOrtho2D(0, self.width, 0, self.height) - - glMatrixMode(GL_MODELVIEW) - glPushMatrix() - glLoadIdentity() - - # 禁用深度测试和光照 - glDisable(GL_DEPTH_TEST) - glDisable(GL_LIGHTING) - - # 绘制状态文本 - font = pygame.font.SysFont(None, 24) - - status_info = [ - f"模式: {state.get('mode', 'TEST')}", - f"位置: X={state.get('position', [0, 0, 0])[0]:.2f}m, Y={state.get('position', [0, 0, 0])[1]:.2f}m, Z={state.get('position', [0, 0, 0])[2]:.2f}m", - f"电池: {state.get('battery', 100):.1f}%", - f"解锁: {'是' if state.get('armed', True) else '否'}", - f"俯仰: {np.degrees(state.get('orientation', [0, 0, 0])[1]):.1f}°, 横滚: {np.degrees(state.get('orientation', [0, 0, 0])[0]):.1f}°, 偏航: {np.degrees(state.get('orientation', [0, 0, 0])[2]):.1f}°" - ] - - for i, text in enumerate(status_info): - text_surface = font.render(text, True, (255, 255, 255)) - text_data = pygame.image.tostring(text_surface, "RGBA", True) - - glRasterPos2d(10, self.height - 30 - i * 25) - glDrawPixels(text_surface.get_width(), text_surface.get_height(), - GL_RGBA, GL_UNSIGNED_BYTE, text_data) - - # 绘制控制提示 - controls = [ - "控制提示:", - "ESC - 退出仿真", - "G - 切换网格显示", - "T - 切换轨迹显示", - "A - 切换坐标轴显示", - "↑↓←→ - 旋转视角", - "+/- - 缩放视角", - "空格 - 重置视角" - ] - - for i, text in enumerate(controls): - text_surface = font.render(text, True, (200, 200, 255)) - text_data = pygame.image.tostring(text_surface, "RGBA", True) - - glRasterPos2d(self.width - 300, self.height - 30 - i * 25) - glDrawPixels(text_surface.get_width(), text_surface.get_height(), - GL_RGBA, GL_UNSIGNED_BYTE, text_data) - - # 恢复设置 - glEnable(GL_DEPTH_TEST) - glEnable(GL_LIGHTING) - - glMatrixMode(GL_PROJECTION) - glPopMatrix() - glMatrixMode(GL_MODELVIEW) - glPopMatrix() + # 暂时禁用文本绘制，只保留3D场景渲染 + pass def _draw_default_overlay(self): """绘制默认覆盖层（无状态数据时）""" - # 切换到2D模式 - glMatrixMode(GL_PROJECTION) - glPushMatrix() - glLoadIdentity() - gluOrtho2D(0, self.width, 0, self.height) - - glMatrixMode(GL_MODELVIEW) - glPushMatrix() - glLoadIdentity() - - # 禁用深度测试和光照 - glDisable(GL_DEPTH_TEST) - glDisable(GL_LIGHTING) - - # 绘制状态文本 - font = pygame.font.SysFont(None, 24) - - status_info = [ - "3D无人机仿真系统", - "测试模式运行中", - "等待连接主程序...", - "按ESC键退出" - ] - - for i, text in enumerate(status_info): - text_surface = font.render(text, True, (255, 255, 255)) - text_data = pygame.image.tostring(text_surface, "RGBA", True) - - glRasterPos2d(10, self.height - 30 - i * 25) - glDrawPixels(text_surface.get_width(), text_surface.get_height(), - GL_RGBA, GL_UNSIGNED_BYTE, text_data) - - # 恢复设置 - glEnable(GL_DEPTH_TEST) - glEnable(GL_LIGHTING) - - glMatrixMode(GL_PROJECTION) - glPopMatrix() - glMatrixMode(GL_MODELVIEW) - glPopMatrix() + # 暂时禁用文本绘制，只保留3D场景渲染 + pass def handle_events(self): """处理窗口事件""" From 55d14f830059fc686165fd763fb30763dca6b227 Mon Sep 17 00:00:00 2001 From: springstart11111 <1312982847@qq.com> Date: Sat, 18 Apr 2026 12:30:59 +0800 Subject: [PATCH 11/11] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8DAirSim=E7=89=88?= =?UTF-8?q?=E6=89=8B=E5=8A=BF=E6=A3=80=E6=B5=8B=E4=B8=8D=E5=87=86=E7=A1=AE?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/drone_hand_gesture/main_airsim.py | 142 ++++++++++++++++---------- src/drone_hand_gesture/run_airsim.bat | 23 +---- 2 files changed, 91 insertions(+), 74 deletions(-) diff --git a/src/drone_hand_gesture/main_airsim.py b/src/drone_hand_gesture/main_airsim.py index 209461a8e..f73c9a0c5 100644 --- a/src/drone_hand_gesture/main_airsim.py +++ b/src/drone_hand_gesture/main_airsim.py @@ -13,7 +13,7 @@ sys.path.append(os.path.dirname(os.path.abspath(__file__))) from airsim_controller import AirSimController -from gesture_detector_enhanced import EnhancedGestureDetector +from gesture_detector import GestureDetector def main(show_window=True): @@ -42,10 +42,10 @@ def main(show_window=True): # 2. 初始化手势检测器 print("\n[2/4] 正在初始化手势检测器...") - print("[DEBUG] 正在创建 EnhancedGestureDetector 实例...") - detector = EnhancedGestureDetector(use_ml=False) - print("[DEBUG] EnhancedGestureDetector 实例创建成功") - print("[OK] 手势检测器就绪") + print("[DEBUG] 正在创建 GestureDetector 实例...") + detector = GestureDetector() + print("[DEBUG] GestureDetector 实例创建成功") + print("[OK] 手势检测器就绪（规则检测）") # 3. 初始化摄像头 print("\n[3/4] 正在初始化摄像头...") @@ -69,13 +69,14 @@ def main(show_window=True): print("\n[4/4] 系统就绪！") print("\n" + "=" * 70) print("手势控制:") - print(" 张开手掌 - 悬停") - print(" 食指向上 - 上升") - print(" 食指向下 - 下降") - print(" 指向左侧 - 左移") - print(" 指向右侧 - 右移") - print(" 两指向前 - 前进") - print(" 握拳 - 降落") + print(" 张开手掌 - 起飞") + print(" 握拳 - 降落") + print(" 食指上指 - 上升") + print(" 食指向下 - 下降") + print(" 胜利手势 - 前进") + print(" 大拇指 - 后退") + print(" OK手势 - 悬停") + print(" 大拇指向下 - 停止") print("\n键盘控制:") print(" 空格键 - 起飞/降落") print(" T - 手动起飞") @@ -87,9 +88,12 @@ def main(show_window=True): # 主循环 is_flying = False - last_gesture_time = 0 + last_command_time = 0 current_gesture = "" - gesture_hold_duration = 1.0 + last_processed_gesture = "" + last_processed_time = 0 + command_cooldown = 1.5 # 命令冷却时间（秒） + gesture_threshold = 0.5 # 置信度阈值（gesture_detector返回0.75-0.95） frame_count = 0 start_time = time.time() @@ -102,6 +106,9 @@ def main(show_window=True): print("[WARNING] 无法读取摄像头画面") break + # 镜像翻转画面，让操作更自然 + frame = cv2.flip(frame, 1) + frame_count += 1 # 手势识别 @@ -116,48 +123,73 @@ def main(show_window=True): cv2.putText(debug_frame, f"Gesture: {gesture} ({confidence:.2f})", (10, 60), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0), 2) - # 处理手势 - if gesture and gesture != "none" and gesture != "no_hand": - current_time = time.time() + # 处理手势（参考main.py的优化逻辑） + current_time = time.time() + in_cooldown = current_time - last_command_time <= command_cooldown + same_gesture = (gesture == last_processed_gesture and + current_time - last_processed_time < 2.0) + + # 使用detector的get_command方法获取指令（与main.py一致） + command = detector.get_command(gesture) + + if (gesture not in ["no_hand", "hand_detected", "none"] + and confidence > gesture_threshold + and not in_cooldown + and not same_gesture + and command != "none"): + + print(f"[CMD] 手势：{gesture} (置信度: {confidence:.2f}) -> 执行: {command}") + + # 根据command执行对应操作（与main.py的drone_controller.send_command逻辑一致） + if command == "land": + if is_flying: + print("[INFO] 降落...") + controller.land() + is_flying = False + + elif command == "up": + print("[INFO] 上升") + controller.move_by_velocity(0, 0, -1.0, duration=0.5) + + elif command == "down": + print("[INFO] 下降") + controller.move_by_velocity(0, 0, 1.0, duration=0.5) + + elif command == "left": + print("[INFO] 左移") + controller.move_by_velocity(-1.0, 0, 0, duration=0.5) + + elif command == "right": + print("[INFO] 右移") + controller.move_by_velocity(1.0, 0, 0, duration=0.5) + + elif command == "forward": + print("[INFO] 前进") + controller.move_by_velocity(0, 1.0, 0, duration=0.5) + + elif command == "backward": + print("[INFO] 后退") + controller.move_by_velocity(0, -1.0, 0, duration=0.5) + + elif command == "hover": + print("[INFO] 悬停") + controller.hover() + + elif command == "takeoff": + if not is_flying: + print("[INFO] 起飞...") + controller.takeoff() + is_flying = True + + elif command == "stop": + print("[INFO] 停止") + controller.hover() - if gesture == current_gesture: - if current_time - last_gesture_time > gesture_hold_duration: - print(f"[CMD] 手势：{gesture}") - - if gesture == 'fist': - if is_flying: - print("[INFO] 降落...") - controller.land() - is_flying = False - - elif gesture == 'point_up': - print("[INFO] 上升") - controller.move_by_velocity(0, 0, -1.0, duration=0.5) - - elif gesture == 'point_down': - print("[INFO] 下降") - controller.move_by_velocity(0, 0, 1.0, duration=0.5) - - elif gesture == 'point_left': - print("[INFO] 左移") - controller.move_by_velocity(-1.0, 0, 0, duration=0.5) - - elif gesture == 'point_right': - print("[INFO] 右移") - controller.move_by_velocity(1.0, 0, 0, duration=0.5) - - elif gesture == 'victory': - print("[INFO] 前进") - controller.move_by_velocity(0, 1.0, 0, duration=0.5) - - elif gesture == 'open_palm': - print("[INFO] 悬停") - controller.hover() - - last_gesture_time = current_time - else: - current_gesture = gesture - last_gesture_time = current_time + # 更新状态 + last_command_time = current_time + last_processed_gesture = gesture + last_processed_time = current_time + current_gesture = gesture # 显示画面 if show_window: diff --git a/src/drone_hand_gesture/run_airsim.bat b/src/drone_hand_gesture/run_airsim.bat index 89c3db632..9258d56e9 100644 --- a/src/drone_hand_gesture/run_airsim.bat +++ b/src/drone_hand_gesture/run_airsim.bat @@ -1,28 +1,12 @@ @echo off -chcp 65001 >nul echo ================================================ echo Drone Gesture Control - AirSim Version echo ================================================ -echo. -echo Checking if AirSim is running... -echo. - -REM Check if AirSim is running -tasklist /FI "WINDOWTITLE eq Blocks" 2>nul | find "Blocks.exe" >nul -if %ERRORLEVEL% NEQ 0 ( - echo [WARNING] AirSim is not running! - echo. - echo Please start AirSim simulator first: - echo Double click: Blocks.exe - echo Note: Blocks.exe is usually in the AirSim installation directory - echo. - echo Press any key to continue (if AirSim is already running)... - pause >nul -) - echo. echo Starting gesture control program... echo. +echo Please make sure AirSim (Blocks.exe) is running! +echo. cd /d "%~dp0" python main_airsim.py @@ -30,4 +14,5 @@ python main_airsim.py echo. echo ================================================ echo Program exited -pause +echo ================================================ +pause \ No newline at end of file