webglToy/src/page/index/HandTrack/index.vue
rucky f1545677c3 完成手势位置识别
等待校准坐标
2021-12-28 19:24:55 +08:00

720 lines
18 KiB
Vue
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

<template>
<div class="hand-track">
<video
id="gum-local"
class="video"
width="100%"
height="100%"
autoplay
playsinline
webkit-playsinline
x5-playsinline
muted
ref="myVideo"
></video>
<!-- 获取权限 -->
<button
id="showVideo"
class="btn"
ref="showVideoBtn"
@click="initVideo($event)"
>
开启
</button>
<!-- info -->
<div class="info-dialog" ref="info">
<div class="progress" ref="progress">
当前进度: {{ progressText }}..
</div>
<div class="status">
当前手势: <span>{{ handStatus }}</span>
</div>
</div>
<!-- progress -->
<div class="progress-bar" ref="progressBar">
<div class="bar">
<div class="progress" :style="'width:' + progress + '%'"></div>
</div>
</div>
<!-- canvas -->
<div class="canvas-container" ref="container"></div>
<!-- hand -->
<div
class="hand-dialog"
ref="handDialog"
v-if="handStep > 0 && handStep < 3"
>
<p>请在镜头前尝试做出以下手势</p>
<div class="hand" :class="'hand-' + handStep" ref="hand"></div>
</div>
<!-- unsuit-dialog -->
<div class="des-dialog" ref="unsuit"></div>
</div>
</template>
<script>
// @ is an alias to /src
import gsap from "gsap";
import {
IEngine,
Load,
IHandTrackingApi,
MediaStreamErrorEnum,
EventEnum,
} from "@handtracking.io/yoha";
import * as THREE from "three";
export default {
name: "handTrack",
components: {},
data() {
return {
progress: 0,
progressText: "0%", //进度文字
BORDER_PADDING_FACTOR: 0.02, //边界阀值
VIDEO_WIDTH_FACTOR: 0.8, //宽度阀值
videoRealSize: {},
handStatus: "未检测到手",
handStep: 1,
meshAdded: false,
};
},
mounted() {
this.video = this.$refs.myVideo;
this.canvas = this.$refs.canvas;
this.checkUserDeviceSuitable();
},
methods: {
// 获取ios版本号
getIOSVersion() {
let str = navigator.userAgent.toLowerCase();
let ver = str.match(/cpu iphone os (.*?) like mac os/);
if (!ver) {
console.log("请在Ios系统中打开");
} else {
console.log(
"你当前的Ios系统版本为" + ver[1].replace(/_/g, ".")
);
return ver[1].replace(/_/g, ".");
}
},
// 检测用户手机是否兼容
// > 14.3 ios ok
checkUserDeviceSuitable() {
if (
window.deviceInfo.system === "IOS"
// && window.deviceInfo.app === "WX"
) {
let iosV = this.getIOSVersion();
if (
!this.versionStringCompare(iosV, "14.3")
// && this.versionStringCompare(iosV, "14.0")
) {
// this.$Utils.showTips({
// message: "请点击右上角 ...</br>在浏览器打开",
// autoClose: false,
// });
// } else if (!this.versionStringCompare(iosV, "14.0")) {
this.$Utils.showTips({
message:
"😭 很抱歉</br>⚠️ 当前系统版本过低</br>无法体验该Demo",
autoClose: false,
});
}
}
},
// 版本号比较
versionStringCompare(preVersion = "", lastVersion = "") {
// 将版本号格式化为相同位数数字字符串
function toNum(a) {
var a = a.toString();
//也可以这样写 var c=a.split(/\./);
var c = a.split(".");
// eslint-disable-next-line camelcase
var num_place = ["", "0", "00", "000", "0000"],
// eslint-disable-next-line camelcase
r = num_place.reverse();
for (var i = 0; i < c.length; i++) {
var len = c[i].length;
c[i] = r[len] + c[i];
}
var res = c.join("");
return res;
}
var _a = toNum(preVersion),
_b = toNum(lastVersion);
console.log(_a, _b);
if (_a == _b || _a > _b) return true;
else return false;
},
// 获取设备权限
getDevicePermission(constraints) {
if (navigator.mediaDevices.getUserMedia === undefined) {
navigator.mediaDevices.getUserMedia = function (constraints) {
var getUserMedia =
navigator.getUserMedia ||
navigator.webkitGetUserMedia ||
navigator.mozGetUserMedia ||
navigator.msGetUserMedia ||
navigator.oGetUserMedia;
if (!getUserMedia) {
return Promise.reject(
new Error(
"getUserMedia is not implemented in this browser"
)
);
}
return new Promise(function (resolve, reject) {
getUserMedia.call(
navigator,
constraints,
resolve,
reject
);
});
};
}
return navigator.mediaDevices
.getUserMedia(constraints)
.then((stream) => {
if (stream) {
const tracks = stream.getTracks();
tracks.forEach(function (track) {
track.stop();
});
// stopStreamTracks(stream);
return true;
}
return Promise.reject(new Error("EmptyStreamError"));
})
.catch((errMsg) => {
if (errMsg && "NotAllowedError" === errMsg.name) {
return false;
}
return Promise.reject(errMsg);
});
},
// IOS 用户获取授权
requestPermission() {
return (
this.getDevicePermission({ video: true })
// .catch(() =>
// this.getDevicePermission({ video: false, audio: true })
// )
// .catch(() =>
// this.getDevicePermission({ video: true, audio: true })
// )
.catch(() => true)
);
},
// 获取摄像头设备id
async getBackCameraDeviceId() {
await this.requestPermission();
let source = new Array();
console.log("start get camera devices");
let devices = await navigator.mediaDevices.enumerateDevices();
console.log("devices array:", devices);
if (devices.length > 0) {
devices.forEach(function (device) {
if (device.kind == "videoinput") {
source.push(device);
}
});
}
console.log("devices source:", source);
return source[source.length - 1].deviceId;
},
// 初始化 video
async initVideo(e) {
const constraints = {
video: {
deviceId: { exact: await this.getBackCameraDeviceId() },
facingMode: "environment",
// width: {
// min: 480,
// },
// height: {
// min: 640,
// },
},
// video: {
// facingMode: { exact: "environment" },
// width: { min: 375, ideal: 375, max: 750 },
// height: { min: 590, ideal: 590, max: 1624 },
// },
};
try {
this.stream = await navigator.mediaDevices.getUserMedia(
constraints
);
gsap.to(e.target, { autoAlpha: 0 });
e.target.disabled = true;
// 处理成功回调
this.handleSuccess(this.stream, constraints);
} catch (e) {
this.handleError(e);
}
},
// 处理stream获取成功
async handleSuccess(stream, constraints) {
const videoTracks = stream.getVideoTracks();
this.stream = stream; // make variable available to browser console
let streamSize = this.GetStreamDimensions(stream);
// 设置视频
this.responseVideoAndCanvas();
let handledSize = this.ScaleResolutionToWidth(
{ width: this.video.width, height: this.video.height },
window.innerWidth
);
this.videoRealSize = handledSize;
// 初始化three
this.initTHREELayer();
console.log("Got stream with constraints:", constraints);
console.log(`Using video device: ${videoTracks[0].label}`);
console.log("Got stream size:", videoTracks);
console.log("Got stream size:", streamSize);
console.log(
"Got streamHandled size:",
window.innerHeight,
handledSize
);
this.video.srcObject = stream;
this.video.play();
// 开始识别
this.startDraw();
},
// 处理stream获取错误
handleError(error) {
console.error(error);
},
// 视频设置
responseVideoAndCanvas(size) {
this.video.width = window.innerWidth;
this.video.height = window.innerHeight;
},
// 获得stream大小
GetStreamDimensions(stream) {
return {
width: stream.getVideoTracks()[0].getSettings().width,
height: stream.getVideoTracks()[0].getSettings().height,
};
},
// 等比缩放视频 宽
ScaleResolutionToWidth(resolution, width) {
const cw = resolution.width;
const ch = resolution.height;
const tw = width;
return {
width: tw,
height: ch / (cw / tw),
};
},
// 等比缩放视频 高
ScaleResolutionToHeight(resolution, height) {
const cw = resolution.width;
const ch = resolution.height;
const th = height;
return {
width: cw / (ch / th),
height: th,
};
},
// 创建手势识别引擎
async CreateEngine() {
let API = IHandTrackingApi;
if (API) {
return API.CreateEngine();
}
API = await Load("./yoha");
return API.CreateEngine();
},
// 开始识别
async startDraw() {
console.log("engine start load");
gsap.to(this.$refs.progressBar, {
autoAlpha: 1,
delay: 0.5,
});
this.engine = await this.CreateEngine();
console.log("engine model start load");
await this.engine.DownloadModel((progress) => {
this.progress = Math.round(progress * 100);
this.progressText = `${this.progress}%`;
});
console.log("engine model loaded");
this.progressText = "引擎准备中…";
this.engine.Configure({
// Webcam video is usually flipped so we want the coordinates to be flipped as well.
flipX: false,
// Crop away a small area at the border to prevent the user to move out of view
// when reaching for border areas on the canvas.
padding: this.BORDER_PADDING_FACTOR,
});
this.engine.SetUpCustomTrackSource(this.video);
// await this.engine.SetUpCameraTrackSource();
// ios需要预热
if (window.deviceInfo.system === "IOS") {
await this.engine.Warmup();
}
this.progressText = "引擎已就绪";
gsap.to([this.$refs.progress, this.$refs.progressBar], {
autoAlpha: 0,
delay: 0.5,
onComplete: () => {
this.$refs.progress.style.display = "none";
this.$refs.myVideo.style.display = "block";
},
});
gsap.to([this.$refs.handDialog, this.$refs.info], {
autoAlpha: 1,
delay: 0.5,
});
// 识别引擎启动
this.engineStart();
},
// 识别引擎启动
engineStart() {
this.engine.Start((e) => {
let cursorPos;
if (e.coordinates) {
cursorPos = this.ExponentialCoordinateAverage(
this.ComputeCursorPositionFromCoordinates(e.coordinates)
);
// console.log(cursorPos);
}
// console.log(e.type);
if (e.type === EventEnum.RESULT) {
this.drawPoint(cursorPos[0], cursorPos[1]);
this.Render();
if (e.poses.fist) {
this.handStatus = "握拳";
if (this.handStep == 1) {
this.handStep++;
}
// document.getElementById("status").innerText = "握拳";
} else if (e.poses.pinch) {
this.handStatus = "捏合";
} else {
this.handStatus = "张开";
if (this.handStep == 2) {
this.handStep++;
}
// document.getElementById("status").innerText = "张开";
}
} else if (e.type === EventEnum.LOST) {
this.handStatus = "未检测到手";
// document.getElementById("status").innerText = "未检测到手";
}
});
},
// 计算坐标值平均数
ExponentialMovingAverage(value, alpha) {
if (this.curValue_ === undefined || this.curValue_ === null) {
this.curValue_ = value;
} else {
this.curValue_ = this.curValue_ * (1 - alpha) + value * alpha;
}
return this.curValue_;
},
// 计算出手指四个点
ExponentialCoordinateAverage(coord, alpha = 0.85) {
let xAvg_ = this.ExponentialMovingAverage(coord[0], alpha),
yAvg_ = this.ExponentialMovingAverage(coord[1], alpha),
zAbg_ = this.ExponentialMovingAverage(coord[2], alpha),
aAvg_ = this.ExponentialMovingAverage(coord[3], alpha);
return [xAvg_, yAvg_, zAbg_, aAvg_];
},
// 计算几个点的中间值
ComputeCursorPositionFromCoordinates(coords) {
// return [
// (coords[3][0] + coords[7][0]) / 2,
// (coords[3][1] + coords[7][1]) / 2,
// ];
if (coords[17][0] < coords[2][0]) {
// console.log('Right')
return [
((coords[0][0] + coords[9][0]) / 2) * 1,
(coords[0][1] + coords[9][1]) / 2,
coords[3][0] - coords[18][0],
coords[3][1] - coords[18][1],
];
} else {
// console.log('Left')
return [
((coords[0][0] + coords[9][0]) / 2) * 1.1,
(coords[0][1] + coords[9][1]) / 2,
coords[3][0] - coords[18][0],
coords[3][1] - coords[18][1],
];
}
},
// init THREELayer
initTHREELayer() {
this.container = this.$refs.container;
this.camera_ = new THREE.PerspectiveCamera(
45,
window.innerWidth / window.innerHeight,
0.1,
1000
);
// const distance =
// this.videoRealSize.height /
// (2 * Math.tan((this.camera_.fov * Math.PI) / 360));
// this.camera_.up.set(0, -1, 0);
// this.camera_.position.set(
// this.videoRealSize.width / 2,
// this.videoRealSize.height / 2,
// -distance
// );
// this.camera_.lookAt(
// this.videoRealSize.width,
// this.videoRealSize.height / 2,
// 0
// );
this.camera_.position.set(0, 0, 1000);
this.camera_.lookAt(0, 0, 0);
this.scene_ = new THREE.Scene();
this.renderer_ = new THREE.WebGLRenderer({
alpha: true,
powerPreference: "high-performance",
failIfMajorPerformanceCaveat: false,
antialias: true,
});
this.renderer_.debug.checkShaderErrors = false;
this.renderer_.setSize(
this.videoRealSize.width,
this.videoRealSize.height
);
this.renderer_.setPixelRatio(window.devicePixelRatio);
this.renderer_.setClearColor(0x000000, 0.0);
this.renderer_.autoClear = true;
this.renderer_.domElement.id = "canvas";
this.container.appendChild(this.renderer_.domElement);
this.renderer_.domElement.width = this.video.width;
this.renderer_.domElement.height = this.video.height;
// this.animate();
this.onWindowResize();
window.addEventListener(
"resize",
() => {
this.onWindowResize();
},
false
);
},
onWindowResize(event) {
this.camera_.aspect = window.innerWidth / window.innerHeight;
this.camera_.updateProjectionMatrix();
this.renderer_.setSize(window.innerWidth, window.innerHeight);
},
addTestBall() {
//light
var light = new THREE.DirectionalLight(0xffffff);
var light2 = new THREE.DirectionalLight(0xffffff);
light.position.set(200, 0, 200);
light2.position.set(-200, 0, 200);
this.scene_.add(light);
this.group3D = new THREE.Object3D();
//放入些mesh做实验
var material = new THREE.MeshLambertMaterial({
color: "#2194ce",
});
// var cubeGeometry = new THREE.CubeGeometry(100, 100, 100);
var geo2 = new THREE.SphereGeometry(40, 40, 40);
for (var i = 0; i < 50; i++) {
var spr = new THREE.Mesh(geo2, material);
spr.position.set(
Math.random() * 800 - 400,
Math.random() * 800 - 400,
Math.random() * 800 - 400
);
this.group3D.add(spr);
}
this.scene_.add(this.group3D);
this.ambientLight = new THREE.AmbientLight(0x8a7e7e, 0.2);
this.scene_.add(this.ambientLight);
},
// draw pointer
drawPoint(x, y) {
if (!this.meshAdded) {
this.geo_ = new THREE.RingGeometry(0, 10, 32);
this.mat_ = new THREE.MeshBasicMaterial({
color: "red",
side: THREE.DoubleSide,
});
this.mesh_ = new THREE.Mesh(this.geo_, this.mat_);
this.scene_.add(this.mesh_);
this.meshAdded = true;
console.log("three mesh added");
}
this.mesh_.position.x = x * this.videoRealSize.width;
this.mesh_.position.y = y * this.videoRealSize.height;
this.mesh_.position.z = 0;
},
animate() {
requestAnimationFrame(this.animate);
this.Render();
},
// THREE canvas render
Render() {
this.renderer_.render(this.scene_, this.camera_);
},
},
};
</script>
<!-- Add "scoped" attribute to limit CSS to this component only -->
<style scoped lang="less">
.hand-track {
.prLayout(100%,100vh);
overflow: hidden;
display: flex;
flex-direction: column;
justify-content: center;
align-content: center;
align-items: center;
.video {
.paLayout(50%,50%,auto,100%,0);
transform: translate(-50%, -50%);
// transform: scaleX(-1);
// width: 750px;
display: none;
// height: 700px;
}
.info-dialog {
visibility: hidden;
.paLayout(0,0, 50%,130px,2);
background-color: rgba(255, 255, 255, 0.25);
border-radius: 0 0 30px 0;
padding: 20px;
font-size: 28px;
display: flex;
flex-direction: column;
align-content: center;
align-items: center;
text-align: left;
text-indent: 10px;
justify-content: center;
div {
width: 100%;
line-height: 40px;
color: #fff;
span {
font-weight: bold;
color: #f00;
}
}
.progress {
}
}
.hand-dialog {
visibility: hidden;
.paLayout(0,0,100%,100%,11);
background-color: rgba(0, 0, 0, 0.4);
color: #fff;
text-align: center;
line-height: 50px;
font-size: 30px;
display: flex;
flex-direction: column;
justify-content: center;
align-content: center;
align-items: center;
pointer-events: none;
.hand {
margin-top: 20px;
.prLayout(160px*2.5,277px*2.5);
.bg-norepeat("hand_1","png");
background-size: contain;
&.hand-1 {
.bg-norepeat("hand_1","png");
background-size: contain;
}
&.hand-2 {
.bg-norepeat("hand_2","png");
background-size: contain;
}
}
}
.canvas-container {
.paLayout(50%,50%,auto,100%,100);
transform: translate(-50%, -50%);
pointer-events: none;
background-color: rgba(255, 255, 255, 0.1);
}
.btn {
.paCenterBottom(50%,280px,80px,2);
background-color: rgba(255, 255, 255, 0.85);
line-height: 40px;
text-align: center;
border-radius: 8px;
font-size: 25px;
}
.progress-bar {
visibility: hidden;
.paCenter(50%,80%,40px,12);
margin-top: -20px;
border: 1px solid #fff;
border-radius: 25px;
padding: 4px;
.bar {
.prLayout(100%,100%);
border-radius: 12px;
overflow: hidden;
.progress {
width: 0%;
height: 100%;
background-color: #fff;
}
}
}
.unsuit-dialog {
visibility: hidden;
.paCenterBottom(0, 100%,80px,10);
position: fixed;
background-color: rgba(255, 255, 255, 0.35);
color: #fff;
line-height: 80px;
text-align: center;
font-size: 40px;
}
}
</style>