计算机视觉cv2入门之实时手势检测 - IT评测·应用市场-qidao123.com技术社区

import cv2
import os
DATASET_DIR='GesturesPhotos'#保存所有待采集手势的图片的文件夹的路径
gesture_kinds=5#手势种类:单手可以是1-10,我这里是1-5
photo_num=10#图片数量
classes=list(range(1,gesture_kinds+1,1))#使用1-gesture_kinds来表示所有待预测类别
###############################################
gestures=photo_num//gesture_kinds*classes#photo_num//gesture_kinds=10//5=2,2*[1,2,3,4,5]=[1,2,3,4,5,1,2,3,4,5]
gestures.extend(classes[:photo_num%gesture_kinds])#photo_num%5=10%5=0,extend([:0])相当于extend([])
'''
经过这两步运算,gestures为长度与图片数量一致且由类别构成的列表
gestures主要用来标定每次采集的种类
比如,gesture_kinds=5,photo_num=7,手势种类为5,那么这7次要采集的顺序为[1,2,3,4,5,1,2]
'''
###############################################
os.makedirs(DATASET_DIR, exist_ok=True)#exist_ok=True可以避免二次采集时重建新文件夹
def capture_gestures(gesture:str,count:int):
'''
Args:
gesture:每次采集的手势,要标记在视频中,防止忘记采集的手势是多少导致实际类别与真实采集结果不一致从而成为噪声！\n
count:用来命名每次保存的图片,这里直接用记录图片数量来命名\n
'''
cv2.namedWindow('Data Collection', cv2.WND_PROP_FULLSCREEN)
cv2.setWindowProperty('Data Collection', cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN)
cap=cv2.VideoCapture(0)
print(f'采集手势{gesture}(按ESC保存并退出)')
while True:
ret,frame=cap.read()
if not ret: break
roi=frame[160:440,50:250]#roi区域,可以自行修改
cv2.rectangle(frame, (50,160),(250,440),(0,255,0), 2)#roi区域处绘制方框
cv2.putText(frame,text=f'No.{count+1} Photo gesture {gesture}',org=(250,100),fontScale=2,thickness=5,color=(0,0,255),fontFace=1)
cv2.imshow(f'Data Collection',frame)
key=cv2.waitKey(1)
if key==27:#按下ESC保存并退出
img_path=f'{DATASET_DIR}/{count}.jpg'
cv2.imwrite(img_path,roi)
break
cap.release()
cv2.destroyAllWindows()
for i in range(len(gestures)):
capture_gestures(gestures[i],i)

复制代码

#数据预处理
from torch.utils.data import Dataset, DataLoader
import torch
import torch.nn as nn
import torch.optim as optim
from torchvision import transforms
class GestureDataset(Dataset):
def __init__(self, data_dir=DATASET_DIR,gesture_kinds=gesture_kinds,transform=None):
self.data_dir = data_dir
self.transform = transform
self.image_paths = []
self.labels = []
# 读取数据集
for img_name in os.listdir(data_dir):
if img_name.endswith('.jpg'):
self.image_paths.append(os.path.join(data_dir, img_name))
self.labels.append(int(img_name.split('.')[0])%gesture_kinds)#0-4对于1-5
def __len__(self):
return len(self.image_paths)
def __getitem__(self, idx):
img_path=self.image_paths[idx]
image=cv2.imread(img_path)
image=cv2.cvtColor(image, cv2.COLOR_BGR2RGB) # 转换为RGB
label=self.labels[idx]
if self.transform:
image=self.transform(image)
return image, label
def process_data(data_dir=DATASET_DIR, batch_size=4):
# 数据预处理
transform = transforms.Compose([
transforms.ToPILImage(),
transforms.Resize((64, 64)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
])
dataset=GestureDataset(data_dir, transform=transform)
train_size=int(0.8 * len(dataset))
test_size=len(dataset) - train_size
train_dataset, test_dataset = torch.utils.data.random_split(dataset, [train_size, test_size])
train_loader=DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
test_loader=DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
return train_loader, test_loader

复制代码

#CNN模型
class GestureCNN(nn.Module):
def __init__(self, num_classes=5):
super(GestureCNN, self).__init__()
self.conv1=nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1)
self.relu=nn.ReLU()
self.maxpool=nn.MaxPool2d(kernel_size=2, stride=2)
self.conv2=nn.Conv2d(16, 32, kernel_size=3, stride=1, padding=1)
self.fc1=nn.Linear(32*16*16, 128)
self.fc2=nn.Linear(128, num_classes)
def forward(self, x):
x=self.conv1(x)
x=self.relu(x)
x=self.maxpool(x)
x=self.conv2(x)
x=self.relu(x)
x=self.maxpool(x)
x=x.view(x.size(0), -1)
x=self.fc1(x)
x=self.relu(x)
x=self.fc2(x)
return x
def train_model(train_loader, test_loader, num_epochs=10):
device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model=GestureCNN(num_classes=5).to(device)
criterion=nn.CrossEntropyLoss()
optimizer=optim.Adam(model.parameters(), lr=0.001)
for epoch in range(num_epochs):
model.train()
running_loss=0.0
correct=0
total=0
for images, labels in train_loader:
images=images.to(device)
labels=labels.to(device)
optimizer.zero_grad()
outputs=model(images)
loss=criterion(outputs, labels)
loss.backward()
optimizer.step()
running_loss+=loss.item()
_, predicted=torch.max(outputs.data, 1)
total+=labels.size(0)
correct+=(predicted==labels).sum().item()
train_loss = running_loss / len(train_loader)
train_acc = 100 * correct / total
# 测试集评估
model.eval()
test_correct = 0
test_total = 0
with torch.no_grad():
for images, labels in test_loader:
images=images.to(device)
labels=labels.to(device)
outputs=model(images)
_, predicted=torch.max(outputs.data, 1)
test_total+=labels.size(0)
test_correct+=(predicted==labels).sum().item()
test_acc=100*test_correct/test_total
print(f'Epoch [{epoch+1}/{num_epochs}], '
f'Train Loss: {train_loss:.4f}, '
f'Train Acc: {train_acc:.2f}%, '
f'Test Acc: {test_acc:.2f}%')
# 保存模型
torch.save(model.state_dict(), 'gesture_cnn.pth')
print('训练完成，模型已保存为 gesture_cnn.pth')
return model

复制代码

#实时预测
def realtime_prediction(model_path='gesture_cnn.pth'):
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
#加载模型
model = GestureCNN(num_classes=5).to(device)
model.load_state_dict(torch.load(model_path))
model.eval()
#预处理
transform=transforms.Compose([
transforms.ToPILImage(),
transforms.Resize((64, 64)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
])
cap=cv2.VideoCapture(0)
cv2.namedWindow('Gesture Recognition', cv2.WND_PROP_FULLSCREEN)
cv2.setWindowProperty('Gesture Recognition', cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN)
CLASSES=gestures
with torch.no_grad():
while True:
ret, frame = cap.read()
if not ret:
break
# 手势检测区域
roi = frame[160:440, 50:250]
cv2.rectangle(frame, (50, 160), (250, 440), (0, 255, 0), 2)
try:
input_tensor = transform(cv2.cvtColor(roi, cv2.COLOR_BGR2RGB)).unsqueeze(0).to(device)
output = model(input_tensor)
_, pred=torch.max(output, 1)
probabilities=torch.nn.functional.softmax(output[0], dim=0)
confidence, pred=torch.max(probabilities, 0)
confidence=confidence.item()*100 #转换为百分比
confidence=round(confidence,2)
cv2.putText(frame, f'Prediction: {CLASSES[pred.item()]}', (50, 40), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
cv2.putText(frame,f'confidence:{confidence}',(70,70),cv2.FONT_HERSHEY_SIMPLEX,0.5, (0, 0, 255), 2)
except Exception as e:
print(f"预测错误: {e}")
cv2.imshow('Gesture Recognition', frame)
if cv2.waitKey(1)==27:
break
cap.release()
cv2.destroyAllWindows()
train_loader, test_loader = process_data()
model=train_model(train_loader, test_loader, num_epochs=10)
realtime_prediction()

复制代码