Output of Kinect 2 on browser using Node.js and Javascript

I am trying to show the output of Kinect 2 on the web browser, following a tutorial given here, http://www.webondevices.com/xbox-kinect-2-javascript-gesture-tracking/

I have been able to get the device output as JSON objects in the browser console using this code in server.js

          var Kinect2 = require('kinect2'),
            express = require('express'),
            app = express(),
            server = require('http').createServer(app),
            io = require('socket.io').listen(server);

          var kinect = new Kinect2();

          app.use(express.static(__dirname + '/View'));
          app.use(express.static(__dirname + '/Script'));

          if(kinect.open()) {

            console.log('kinect opened');
            server.listen(8000);
            console.log('Server listening on port 8000');

            kinect.on('bodyFrame', function(bodyFrame){
              io.sockets.emit('bodyFrame', bodyFrame);
            });

            kinect.openBodyReader();

            app.get('/', function(req, res) {

              res.sendFile(__dirname + '/View/output.html');
            });

            setTimeout(function(){
              kinect.close();
              console.log("Kinect Closed");
            }, 100000);
           }

JavaScript
​x
 
          var Kinect2 = require('kinect2'),
            express = require('express'),
            app = express(),
            server = require('http').createServer(app),
            io = require('socket.io').listen(server);
​
          var kinect = new Kinect2();
​
          app.use(express.static(__dirname + '/View'));
          app.use(express.static(__dirname + '/Script'));
​
          if(kinect.open()) {
​
            console.log('kinect opened');
            server.listen(8000);
            console.log('Server listening on port 8000');
​
            kinect.on('bodyFrame', function(bodyFrame){
              io.sockets.emit('bodyFrame', bodyFrame);
            });
​
            kinect.openBodyReader();
​
            app.get('/', function(req, res) {
​
              res.sendFile(__dirname + '/View/output.html');
            });
​
            setTimeout(function(){
              kinect.close();
              console.log("Kinect Closed");
            }, 100000);
           }
​

output.html, the page where i want to show the output on a canvas looks like this

<html>
<head>
    <title>
        Kinect Output On Canvas
    </title>                                                                                
    <script src="https://cdn.socket.io/socket.io-1.3.5.js"></script>                        
    <link rel="stylesheet" href="/style.css" />                                                     
</head>
<body>
    <h1>Kinect &amp; HTML5 WebSockets</h1>
    <canvas id="canvas" width="640" height="480"></canvas>
    <script>

        var socket = io.connect('http://localhost:8000/');
        socket.on('bodyFrame', interpretData);

        function interpretData(bodyFrame) {
        // Web Socket message:
           console.log(bodyFrame);      //outputs each bodyframe as a JSON    object, 30+ frames/JSON objects in the browser per second          

        }

    </script>

</body>

JavaScript
 
<html>
<head>
    <title>
        Kinect Output On Canvas
    </title>                                                                                
    <script src="https://cdn.socket.io/socket.io-1.3.5.js"></script>                        
    <link rel="stylesheet" href="/style.css" />                                                     
</head>
<body>
    <h1>Kinect &amp; HTML5 WebSockets</h1>
    <canvas id="canvas" width="640" height="480"></canvas>
    <script>
​
        var socket = io.connect('http://localhost:8000/');
        socket.on('bodyFrame', interpretData);
​
        function interpretData(bodyFrame) {
        // Web Socket message:
           console.log(bodyFrame);      //outputs each bodyframe as a JSON    object, 30+ frames/JSON objects in the browser per second          
​
        }
​
    </script>
​
</body>
​

the structure of each JSON object showing position of each skeleton tracked is as such

  { bodyIndex: 5,
    tracked: true,
    trackingId: '72057594038115298',
    leftHandState: 1,
    rightHandState: 1,
    joints:
     [ { depthX: 0.24323934316635132,
         depthY: 0.5925129055976868,
         colorX: 0.33547070622444153,
         colorY: 0.6129662394523621,
         cameraX: -0.34261977672576904,
         cameraY: -0.10602515190839767,
         cameraZ: 0.9753329753875732,
         orientationX: -0.04046249017119408,
         orientationY: 0.9915661215782166,
         orientationZ: -0.05280650407075882,
         orientationW: 0.11122455447912216 },
       { depthX: 0.21760234236717224,
         depthY: 0.3140539526939392,
         colorX: 0.31521913409233093,
         colorY: 0.2960273027420044,
         cameraX: -0.36364009976387024,
         cameraY: 0.19814369082450867,
         cameraZ: 0.9404330253601074,
         orientationX: -0.04830155894160271,
         orientationY: 0.9615150094032288,
         orientationZ: -0.04574603587388992,
         orientationW: 0.26657652854919434 },

JavaScript
 
  { bodyIndex: 5,
    tracked: true,
    trackingId: '72057594038115298',
    leftHandState: 1,
    rightHandState: 1,
    joints:
     [ { depthX: 0.24323934316635132,
         depthY: 0.5925129055976868,
         colorX: 0.33547070622444153,
         colorY: 0.6129662394523621,
         cameraX: -0.34261977672576904,
         cameraY: -0.10602515190839767,
         cameraZ: 0.9753329753875732,
         orientationX: -0.04046249017119408,
         orientationY: 0.9915661215782166,
         orientationZ: -0.05280650407075882,
         orientationW: 0.11122455447912216 },
       { depthX: 0.21760234236717224,
         depthY: 0.3140539526939392,
         colorX: 0.31521913409233093,
         colorY: 0.2960273027420044,
         cameraX: -0.36364009976387024,
         cameraY: 0.19814369082450867,
         cameraZ: 0.9404330253601074,
         orientationX: -0.04830155894160271,
         orientationY: 0.9615150094032288,
         orientationZ: -0.04574603587388992,
         orientationW: 0.26657652854919434 },
​

…… there are 24 arrays with similar parameters in the Joints array corresponding to each 24 joints tracked.

To show a skeleton in the browser, i have tried

    var ctx = document.getElementById('canvas').getContext('2d'); 
    ctx.fillStyle = "red";
    ctx.fillRect(10, 10, 20, 20);
    var imgData = ctx.getImageData(10, 10, 15, 15);

JavaScript
 
    var ctx = document.getElementById('canvas').getContext('2d'); 
    ctx.fillStyle = "red";
    ctx.fillRect(10, 10, 20, 20);
    var imgData = ctx.getImageData(10, 10, 15, 15);
​

inside the for loop for each JSON object received

    ctx.putImageData(imgData, x, y); // x and y are the depth x positions of      left and right hands

JavaScript
 
    ctx.putImageData(imgData, x, y); // x and y are the depth x positions of      left and right hands 
​

what this outputs is just one square spot on the canvas and another at the left corner of the canvas, which i understand why because that’s the coordinates fed to it. I want to know how to interpret data i get in the Joints array for any joint as a formula and show a tracked spot on the browser so that i can show a skeleton in the browser. I am checking actual tracking results and video output in the Kinect Studio v2.0 Desktop app.

Any suggestion will be appreciated

Answer

This is what successfully generated an output of tracked skeletons on the browser screen. In output.html, sockets.io receives the JSON objects and Javascript is used to create projects with respect to each point on the screen, according to the depthX parameter. In output.html

function interpretData(bodyFrame) {
  ctx.clearRect(0, 0, c.width, c.height);

  console.log(bodyFrame);

  for (var i = 0; i < bodyFrame.bodies.length; i++) {
    if (bodyFrame.bodies[i].tracked == true) {
      console.log('tracked');
      for (var j = 0; j < bodyFrame.bodies[i].joints.length; j++) {
        var joint = bodyFrame.bodies[i].joints[j];
        ctx.fillStyle = "#FF0000";
        ctx.beginPath();
        ctx.arc(joint.depthX * 400, joint.depthY * 400, 10, 0, Math.PI * 2, true); //multiplied with static integer 400 in order to adjust position on canvas as without it skeleton projection formed is only visible in a corner as DepthX values were always less than 1
        ctx.closePath();
        ctx.fill(); //drawing a circle for each joint on the canvas 
      }
    }
  }
}

JavaScript
 
function interpretData(bodyFrame) {
  ctx.clearRect(0, 0, c.width, c.height);
​
  console.log(bodyFrame);
​
  for (var i = 0; i < bodyFrame.bodies.length; i++) {
    if (bodyFrame.bodies[i].tracked == true) {
      console.log('tracked');
      for (var j = 0; j < bodyFrame.bodies[i].joints.length; j++) {
        var joint = bodyFrame.bodies[i].joints[j];
        ctx.fillStyle = "#FF0000";
        ctx.beginPath();
        ctx.arc(joint.depthX * 400, joint.depthY * 400, 10, 0, Math.PI * 2, true); //multiplied with static integer 400 in order to adjust position on canvas as without it skeleton projection formed is only visible in a corner as DepthX values were always less than 1
        ctx.closePath();
        ctx.fill(); //drawing a circle for each joint on the canvas 
      }
    }
  }
}
​

Advertisement

Answer