I am working on a project to control a character in threejs using a microsoft kinect.
The data for the connect is coming in correctly, The mapping for the bones is working I guess.
But when you look at the movement in the top left – that is the kinect data mapped to points displaying a skeleton – the character is not moving the same way :slight_smile:
Video: https://global.discourse-cdn.com/flex035/uploads/threejs/original/3X/9/7/97c1c8c941aa8b2b98b22602136629070a9c9008.mov
Would someone be willing to help out/collab on this or does anyone see where I have gone wrong?
All input is much appreciated!
<!DOCTYPE html>
<html>
<head>
<title>Kinect ThreeJS Controller</title>
<style>
body {
margin: 0;
}
canvas {
display: block;
position: absolute;
top: 0;
left: 0;
}
#bodyCanvas {
z-index: 1;
pointer-events: none;
}
</style>
</head>
<body>
<canvas id="artifactCanvas"></canvas>
<canvas id="bodyCanvas" width="512" height="424"></canvas>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/umd/index.min.js"></script>
<script src="/socket.io/socket.io.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/examples/js/loaders/FBXLoader.js"></script>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/examples/js/controls/OrbitControls.js"></script>
<script>
const socket = io.connect('http://localhost:8000/');
// Set up scene, camera, and renderer
const windowWidth = window.innerWidth;
const windowHeight = window.innerHeight;
const scene = new THREE.Scene();
const camera = new THREE.PerspectiveCamera(75, windowWidth / windowHeight, 0.1, 1000);
const renderer = new THREE.WebGLRenderer({ canvas: artifactCanvas });
renderer.setSize(windowWidth, windowHeight);
//document.body.appendChild(renderer.domElement);
// Add lights
const ambientLight = new THREE.AmbientLight(0xffffff, 0.5);
scene.add(ambientLight);
const directionalLight = new THREE.DirectionalLight(0xffffff, 0.8);
directionalLight.position.set(0, 1, 1);
scene.add(directionalLight);
// Set up camera position and controls
camera.position.set(0, 100, 300); // Adjust these values
camera.lookAt(0, 0, 0);
const controls = new THREE.OrbitControls(camera, renderer.domElement);
let skeleton;
// Load FBX model
const fbxLoader = new THREE.FBXLoader();
fbxLoader.load('santa.fbx', (fbx) => {
// Try a larger scale
//fbx.scale.setScalar(1.0); // Adjust this value to match skeleton size
scene.add(fbx);
skeleton = fbx.skeleton;
// Add skeleton helper to visualize the skeleton
const skeletonHelper = new THREE.SkeletonHelper(fbx);
skeletonHelper.material.linewidth = 2; // Make bones more visible
skeletonHelper.visible = true;
scene.add(skeletonHelper);
// Store bone references for easier access
const bones = {};
const initialPositions = {};
fbx.traverse((bone) => {
if (bone.isBone) {
const name = bone.name.toLowerCase();
bones[name] = bone;
initialPositions[name] = bone.position.clone();
}
});
// Connect to socket
socket.on('bodyFrame', (data) => {
const joints = JSON.parse(data);
// Map Kinect joints to model bones
for (const jointName in joints) {
const joint = joints[jointName];
const boneName = mapJointToBone(jointName);
if (bones[boneName]) {
// Apply position relative to initial pose
bones[boneName].position.set(
initialPositions[boneName].x + joint.cameraX * 10,
initialPositions[boneName].y + joint.cameraY * 10,
initialPositions[boneName].z - joint.cameraZ
);
}
}
});
});
// Helper function to map Kinect joint names to model bone names
function mapJointToBone(jointName) {
/*
"mixamorigHead"
"mixamorigHeadTop_End"
"mixamorigHips"
"mixamorigLeftArm"
"mixamorigLeftFoot"
"mixamorigLeftForeArm"
"mixamorigLeftHand"
"mixamorigLeftLeg"
"mixamorigLeftShoulder"
"mixamorigLeftToe_End"
"mixamorigLeftToeBase"
"mixamorigLeftUpLeg"
"mixamorigNeck"
"mixamorigRightArm"
"mixamorigRightFoot"
"mixamorigRightForeArm"
"mixamorigRightHand"
"mixamorigRightLeg"
"mixamorigRightShoulder"
"mixamorigRightToe_End"
"mixamorigRightToeBase"
"mixamorigRightUpLeg"
"mixamorigSpine"
"mixamorigSpine1"
"mixamorigSpine2"
*/
const mapping = {
0: 'mixamorigspine',
2: 'mixamorigneck',
3: 'mixamorighead',
4: 'mixamorigleftshoulder',
5: 'mixamorigleftarm',
6: 'mixamorigleftforearm',
7: 'mixamoriglefthand',
8: 'mixamorigrightshoulder',
9: 'mixamorigrightarm',
10: 'mixamorigrightforearm',
11: 'mixamorigrighthand',
12: 'mixamorigleftupleg',
13: 'mixamorigleftleg',
14: 'mixamoriglefttoe_end',
16: 'mixamorigrightupleg',
17: 'mixamorigrightleg',
18: 'mixamorigrighttoe_end'
};
return mapping[jointName] || jointName;
}
// Animation loop
const clock = new THREE.Clock();
function animate() {
requestAnimationFrame(animate);
controls.update();
renderer.render(scene, camera);
}
// Handle window resize
window.addEventListener('resize', () => {
camera.aspect = window.innerWidth / window.innerHeight;
camera.updateProjectionMatrix();
renderer.setSize(window.innerWidth, window.innerHeight);
});
animate();
var canvas = document.getElementById('bodyCanvas');
var ctx = canvas.getContext('2d');
var colors = ['#ff0000', '#00ff00', '#0000ff', '#ffff00', '#00ffff', '#ff00ff'];
// handstate circle size
var HANDSIZE = 2;
// closed hand state color
var HANDCLOSEDCOLOR = "red";
// open hand state color
var HANDOPENCOLOR = "green";
// lasso hand state color
var HANDLASSOCOLOR = "blue";
function updateHandState(handState, jointPoint) {
switch (handState) {
case 3:
drawHand(jointPoint, HANDCLOSEDCOLOR);
break;
case 2:
drawHand(jointPoint, HANDOPENCOLOR);
break;
case 4:
drawHand(jointPoint, HANDLASSOCOLOR);
break;
}
}
function drawHand(jointPoint, handColor) {
// draw semi transparent hand cicles
ctx.globalAlpha = 0.75;
ctx.beginPath();
ctx.fillStyle = handColor;
ctx.arc(jointPoint.depthX * 512, jointPoint.depthY * 424, HANDSIZE, 0, Math.PI * 2, true);
ctx.fill();
ctx.closePath();
ctx.globalAlpha = 1;
}
socket.on('bodyFrame', function (bodyFrame) {
bodyFrame = JSON.parse(bodyFrame);
ctx.clearRect(0, 0, canvas.width, canvas.height);
var index = 0;
bodyFrame.forEach(function (aJoint) {
for (var jointType in aJoint) {
ctx.fillStyle = colors[index];
ctx.fillRect(aJoint.depthX * 512, aJoint.depthY * 424, HANDSIZE, HANDSIZE);
}
//updateElbowState(body.joints[5], body.joints[9]);
index++;
});
});
// Add grid helper for reference
const gridHelper = new THREE.GridHelper(1000, 100);
scene.add(gridHelper);
</script>
</body>
</html>
