#if defined( GESTURE_TRAINER ) #if defined( TARGET_X360 ) #include #elif defined( TARGET_DURANGO ) #include #elif defined( TARGET_ORBIS ) #include "SK_iisu/PS4/include/HumanTrackingCTypes.h" #include "Kinect/PS4/commontypes.h" #else #error unsuported platform #endif #else // defined( GESTURE_TRAINER ) #if defined( _XBOX ) || defined( ITF_X360 ) #include #elif defined( DURANGO ) || defined( ITF_DURANGO ) #include #elif defined( __ORBIS__ ) || defined( ITF_ORBIS ) #include "SK_iisu/PS4/include/HumanTrackingCTypes.h" #include "Kinect/PS4/commontypes.h" #elif defined ( WIN32 ) ||defined( ITF_WIN32) #include #else #error unsuported platform #endif #endif #include #include #include #include #if defined( __ORBIS__ ) || defined( TARGET_ORBIS ) || defined( ITF_ORBIS ) #else #pragma warning(disable : 4793) #endif #include "VelocityGrid.h" #if defined( DURANGO ) || defined( TARGET_DURANGO ) using namespace Microsoft::Xbox::Input::Nui; using namespace DirectX; #elif defined ( WIN32 ) ||defined( TARGET_WIN32) using namespace Microsoft::Win32::Input::Nui; using namespace DirectX; #endif inline FLOAT MAX( FLOAT v1, FLOAT v2 ) { return ( v1 ) >= ( v2 ) ? ( v1 ) : ( v2 ); } inline FLOAT MIN( FLOAT v1, FLOAT v2 ) { return ( v1 ) <= ( v2 ) ? ( v1 ) : ( v2 ); } inline FLOAT CLAMP( FLOAT value, FLOAT min, FLOAT max ) { return MIN( ( max ), MAX( ( value ), ( min ) ) ); } struct JointsLine { GESTURE_JOINT_INDEX StartJoint; GESTURE_JOINT_INDEX EndJoint; float Priority; }; #if defined(WIN32) || defined(WIN64) // Define the lines in the skeleton to "render" into the velocity field; don't have to be actual bones const JointsLine g_BoneLines[] = { // Head { GESTURE_JOINT_HEAD, GESTURE_JOINT_SPINE_SHOULDER, 0.5f }, // Top of head to top of neck // Right arm // { GESTURE_JOINT_SPINE_SHOULDER, GESTURE_JOINT_SHOULDER_RIGHT }, // Neck bottom to right shoulder internal { GESTURE_JOINT_SHOULDER_RIGHT, GESTURE_JOINT_ELBOW_RIGHT, 1.0f }, // Right shoulder internal to right elbow { GESTURE_JOINT_ELBOW_RIGHT, GESTURE_JOINT_WRIST_RIGHT, 1.7f}, // Right elbow to right wrist { GESTURE_JOINT_WRIST_RIGHT, GESTURE_JOINT_HAND_RIGHT, 0.3f }, // Right wrist to right hand // Left arm // { GESTURE_JOINT_SPINE_SHOULDER, GESTURE_JOINT_SHOULDER_LEFT }, // Neck bottom to left shoulder internal { GESTURE_JOINT_SHOULDER_LEFT, GESTURE_JOINT_ELBOW_LEFT, 1.0f}, // Left shoulder internal to left elbow { GESTURE_JOINT_ELBOW_LEFT, GESTURE_JOINT_WRIST_LEFT, 1.7f}, // Left elbow to left wrist { GESTURE_JOINT_WRIST_LEFT, GESTURE_JOINT_HAND_LEFT, 0.3f }, // Left wrist to left hand // Right leg and foot { GESTURE_JOINT_HIP_RIGHT, GESTURE_JOINT_KNEE_RIGHT, 1.0f }, // Right hip internal to right knee { GESTURE_JOINT_KNEE_RIGHT, GESTURE_JOINT_ANKLE_RIGHT, 1.5f }, // Right knee to right ankle { GESTURE_JOINT_ANKLE_RIGHT, GESTURE_JOINT_FOOT_RIGHT, 0.5f }, // Left ankle to left foot // Left leg and foot { GESTURE_JOINT_HIP_LEFT, GESTURE_JOINT_KNEE_LEFT, 1.0f }, // Left hip internal to left knee { GESTURE_JOINT_KNEE_LEFT, GESTURE_JOINT_ANKLE_LEFT, 1.5f}, // Left knee to left ankle { GESTURE_JOINT_ANKLE_LEFT, GESTURE_JOINT_FOOT_LEFT, 0.5f }, // Left ankle to left foot // Spine // { GESTURE_JOINT_SPINE_SHOULDER, GESTURE_JOINT_SPINE_MID}, // Neck bottom to spine // { GESTURE_JOINT_SPINE_MID, GESTURE_JOINT_SPINE_BASE }, // Spine to hip center // Hips // { GESTURE_JOINT_HIP_RIGHT, GESTURE_JOINT_SPINE_BASE }, // Right hip to hip center // { GESTURE_JOINT_SPINE_BASE, GESTURE_JOINT_HIP_LEFT}, // Hip center to left hip { GESTURE_JOINT_SHOULDER_LEFT, GESTURE_JOINT_SHOULDER_RIGHT, 0.5f }, { GESTURE_JOINT_HIP_LEFT, GESTURE_JOINT_HIP_RIGHT, 0.5f }, { GESTURE_JOINT_SHOULDER_LEFT, GESTURE_JOINT_HIP_LEFT, 0.5f }, { GESTURE_JOINT_SHOULDER_RIGHT, GESTURE_JOINT_HIP_RIGHT, 0.5f }, }; #elif defined( _XBOX ) || defined( TARGET_X360 ) || defined( ITF_X360 ) // Define the lines in the skeleton to "render" into the velocity field; don't have to be actual bones const JointsLine g_BoneLines[] = { // Head { GESTURE_JOINT_HEAD, GESTURE_JOINT_SPINE_SHOULDER, 0.5f }, // Top of head to top of neck // Right arm // { GESTURE_JOINT_SPINE_SHOULDER, GESTURE_JOINT_SHOULDER_RIGHT }, // Neck bottom to right shoulder internal { GESTURE_JOINT_SHOULDER_RIGHT, GESTURE_JOINT_ELBOW_RIGHT, 1.0f }, // Right shoulder internal to right elbow { GESTURE_JOINT_ELBOW_RIGHT, GESTURE_JOINT_WRIST_RIGHT, 1.7f}, // Right elbow to right wrist { GESTURE_JOINT_WRIST_RIGHT, GESTURE_JOINT_HAND_RIGHT, 0.3f }, // Right wrist to right hand // Left arm // { GESTURE_JOINT_SPINE_SHOULDER, GESTURE_JOINT_SHOULDER_LEFT }, // Neck bottom to left shoulder internal { GESTURE_JOINT_SHOULDER_LEFT, GESTURE_JOINT_ELBOW_LEFT, 1.0f}, // Left shoulder internal to left elbow { GESTURE_JOINT_ELBOW_LEFT, GESTURE_JOINT_WRIST_LEFT, 1.7f}, // Left elbow to left wrist { GESTURE_JOINT_WRIST_LEFT, GESTURE_JOINT_HAND_LEFT, 0.3f }, // Left wrist to left hand // Right leg and foot { GESTURE_JOINT_HIP_RIGHT, GESTURE_JOINT_KNEE_RIGHT, 1.0f }, // Right hip internal to right knee { GESTURE_JOINT_KNEE_RIGHT, GESTURE_JOINT_ANKLE_RIGHT, 1.5f }, // Right knee to right ankle { GESTURE_JOINT_ANKLE_RIGHT, GESTURE_JOINT_FOOT_RIGHT, 0.5f }, // Left ankle to left foot // Left leg and foot { GESTURE_JOINT_HIP_LEFT, GESTURE_JOINT_KNEE_LEFT, 1.0f }, // Left hip internal to left knee { GESTURE_JOINT_KNEE_LEFT, GESTURE_JOINT_ANKLE_LEFT, 1.5f}, // Left knee to left ankle { GESTURE_JOINT_ANKLE_LEFT, GESTURE_JOINT_FOOT_LEFT, 0.5f }, // Left ankle to left foot // Spine // { GESTURE_JOINT_SPINE_SHOULDER, GESTURE_JOINT_SPINE_MID}, // Neck bottom to spine // { GESTURE_JOINT_SPINE_MID, GESTURE_JOINT_SPINE_BASE }, // Spine to hip center // Hips // { GESTURE_JOINT_HIP_RIGHT, GESTURE_JOINT_SPINE_BASE }, // Right hip to hip center // { GESTURE_JOINT_SPINE_BASE, GESTURE_JOINT_HIP_LEFT}, // Hip center to left hip { GESTURE_JOINT_SHOULDER_LEFT, GESTURE_JOINT_SHOULDER_RIGHT, 0.5f }, { GESTURE_JOINT_HIP_LEFT, GESTURE_JOINT_HIP_RIGHT, 0.5f }, { GESTURE_JOINT_SHOULDER_LEFT, GESTURE_JOINT_HIP_LEFT, 0.5f }, { GESTURE_JOINT_SHOULDER_RIGHT, GESTURE_JOINT_HIP_RIGHT, 0.5f }, }; #elif defined( DURANGO ) || defined( TARGET_DURANGO ) || defined( ITF_DURANGO ) // Define the lines in the skeleton to "render" into the velocity field; don't have to be actual bones const JointsLine g_BoneLines[] = { // Head { GESTURE_JOINT_HEAD, GESTURE_JOINT_NECK, 0.5f }, // Top of head to top of neck // Right arm // { NUI_SKELETON_JOINT_SHOULDER_CENTER, NUI_SKELETON_JOINT_SHOULDER_RIGHT }, // Neck bottom to right shoulder internal { GESTURE_JOINT_SHOULDER_RIGHT, GESTURE_JOINT_ELBOW_RIGHT, 1.0f }, // Right shoulder internal to right elbow { GESTURE_JOINT_ELBOW_RIGHT, GESTURE_JOINT_WRIST_RIGHT, 1.7f}, // Right elbow to right wrist { GESTURE_JOINT_WRIST_RIGHT, GESTURE_JOINT_HAND_RIGHT, 0.3f }, // Right wrist to right hand // Left arm // { NUI_SKELETON_JOINT_SHOULDER_CENTER, NUI_SKELETON_JOINT_SHOULDER_LEFT }, // Neck bottom to left shoulder internal { GESTURE_JOINT_SHOULDER_LEFT, GESTURE_JOINT_ELBOW_LEFT, 1.0f}, // Left shoulder internal to left elbow { GESTURE_JOINT_ELBOW_LEFT, GESTURE_JOINT_WRIST_LEFT, 1.7f}, // Left elbow to left wrist { GESTURE_JOINT_WRIST_LEFT, GESTURE_JOINT_HAND_LEFT, 0.3f }, // Left wrist to left hand // Right leg and foot { GESTURE_JOINT_HIP_RIGHT, GESTURE_JOINT_KNEE_RIGHT, 1.0f }, // Right hip internal to right knee { GESTURE_JOINT_KNEE_RIGHT, GESTURE_JOINT_ANKLE_RIGHT, 1.5f }, // Right knee to right ankle { GESTURE_JOINT_ANKLE_RIGHT, GESTURE_JOINT_FOOT_RIGHT, 0.5f }, // Left ankle to left foot // Left leg and foot { GESTURE_JOINT_HIP_LEFT, GESTURE_JOINT_KNEE_LEFT, 1.0f }, // Left hip internal to left knee { GESTURE_JOINT_KNEE_LEFT, GESTURE_JOINT_ANKLE_LEFT, 1.5f}, // Left knee to left ankle { GESTURE_JOINT_ANKLE_LEFT, GESTURE_JOINT_FOOT_LEFT, 0.5f }, // Left ankle to left foot // Spine // { NUI_SKELETON_JOINT_SHOULDER_CENTER, NUI_SKELETON_JOINT_SPINE}, // Neck bottom to spine // { NUI_SKELETON_JOINT_SPINE, NUI_SKELETON_JOINT_HIP_CENTER }, // Spine to hip center // Hips // { NUI_SKELETON_JOINT_HIP_RIGHT, NUI_SKELETON_JOINT_HIP_CENTER }, // Right hip to hip center // { NUI_SKELETON_JOINT_HIP_CENTER, NUI_SKELETON_JOINT_HIP_LEFT}, // Hip center to left hip { GESTURE_JOINT_SHOULDER_LEFT, GESTURE_JOINT_SHOULDER_RIGHT, 0.5f }, { GESTURE_JOINT_HIP_LEFT, GESTURE_JOINT_HIP_RIGHT, 0.5f }, { GESTURE_JOINT_SHOULDER_LEFT, GESTURE_JOINT_HIP_LEFT, 0.5f }, { GESTURE_JOINT_SHOULDER_RIGHT, GESTURE_JOINT_HIP_RIGHT, 0.5f }, }; #elif defined( __ORBIS__ ) || defined( TARGET_ORBIS ) || defined( ITF_ORBIS ) // Define the lines in the skeleton to "render" into the velocity field; don't have to be actual bones const JointsLine g_BoneLines[] = { // Head { GESTURE_JOINT_HEAD, GESTURE_JOINT_SPINE_SHOULDER, 0.5f }, // Top of head to top of neck // Right arm // { GESTURE_JOINT_SPINE_SHOULDER, GESTURE_JOINT_SHOULDER_RIGHT }, // Neck bottom to right shoulder internal { GESTURE_JOINT_SHOULDER_RIGHT, GESTURE_JOINT_ELBOW_RIGHT, 1.0f }, // Right shoulder internal to right elbow { GESTURE_JOINT_ELBOW_RIGHT, GESTURE_JOINT_WRIST_RIGHT, 1.7f}, // Right elbow to right wrist { GESTURE_JOINT_WRIST_RIGHT, GESTURE_JOINT_HAND_RIGHT, 0.3f }, // Right wrist to right hand // Left arm // { GESTURE_JOINT_SPINE_SHOULDER, GESTURE_JOINT_SHOULDER_LEFT }, // Neck bottom to left shoulder internal { GESTURE_JOINT_SHOULDER_LEFT, GESTURE_JOINT_ELBOW_LEFT, 1.0f}, // Left shoulder internal to left elbow { GESTURE_JOINT_ELBOW_LEFT, GESTURE_JOINT_WRIST_LEFT, 1.7f}, // Left elbow to left wrist { GESTURE_JOINT_WRIST_LEFT, GESTURE_JOINT_HAND_LEFT, 0.3f }, // Left wrist to left hand // Right leg and foot { GESTURE_JOINT_HIP_RIGHT, GESTURE_JOINT_KNEE_RIGHT, 1.0f }, // Right hip internal to right knee { GESTURE_JOINT_KNEE_RIGHT, GESTURE_JOINT_ANKLE_RIGHT, 1.5f }, // Right knee to right ankle { GESTURE_JOINT_ANKLE_RIGHT, GESTURE_JOINT_FOOT_RIGHT, 0.5f }, // Left ankle to left foot // Left leg and foot { GESTURE_JOINT_HIP_LEFT, GESTURE_JOINT_KNEE_LEFT, 1.0f }, // Left hip internal to left knee { GESTURE_JOINT_KNEE_LEFT, GESTURE_JOINT_ANKLE_LEFT, 1.5f}, // Left knee to left ankle { GESTURE_JOINT_ANKLE_LEFT, GESTURE_JOINT_FOOT_LEFT, 0.5f }, // Left ankle to left foot // Spine // { GESTURE_JOINT_SPINE_SHOULDER, GESTURE_JOINT_SPINE_MID}, // Neck bottom to spine // { GESTURE_JOINT_SPINE_MID, GESTURE_JOINT_SPINE_BASE }, // Spine to hip center // Hips // { GESTURE_JOINT_HIP_RIGHT, GESTURE_JOINT_SPINE_BASE }, // Right hip to hip center // { GESTURE_JOINT_SPINE_BASE, GESTURE_JOINT_HIP_LEFT}, // Hip center to left hip { GESTURE_JOINT_SHOULDER_LEFT, GESTURE_JOINT_SHOULDER_RIGHT, 0.5f }, { GESTURE_JOINT_HIP_LEFT, GESTURE_JOINT_HIP_RIGHT, 0.5f }, { GESTURE_JOINT_SHOULDER_LEFT, GESTURE_JOINT_HIP_LEFT, 0.5f }, { GESTURE_JOINT_SHOULDER_RIGHT, GESTURE_JOINT_HIP_RIGHT, 0.5f }, }; #else #error unsuported platform #endif const UINT g_uNumBoneLines = sizeof( g_BoneLines ) / sizeof( JointsLine ); namespace KinectGesture { // VelocityGridCalculate(...) //========================== //************************************ HRESULT VelocityGridCalculate( __out VelocityGridFromSkeleton* pResult, const VelocityGridFromSkeleton* pPrevious, const GESTURE_SKELETON_TYPE* pSkeletonData, const float deltaSeconds ) { //in a normal update pPrevious == pResult so careful... if ( pPrevious == NULL ) { #ifdef _XBOX XMemSet( &( pResult->mGrid.mVelocity ), 0, sizeof( float ) * KINECT_GESTURE_VELOCITY_GRID_X * KINECT_GESTURE_VELOCITY_GRID_Y * 2 ); #else memset( &( pResult->mGrid.mVelocity ), 0, sizeof( float ) * KINECT_GESTURE_VELOCITY_GRID_X * KINECT_GESTURE_VELOCITY_GRID_Y * 2 ); #endif return S_OK; } static float fDamping = 0.7f; static float fStep = 0.25f; static float fScale = 0.1f * fStep; //idea: for each bone, get velocity as difference between m_SkeletonFrame and m_SkeletonFramePrev, // decide which one of the 9 regions it contributes to, // and add velocity. //also, add some relaxation to each region once a frame to decrease velocities. const GESTURE_SKELETON_TYPE* pSkeletonDataPrev = &pPrevious->mSkeletonDataPrev; //m_SkeletonOpticalFlow[iSkeletonIndex].Dampen(damping); for(int i=0; i<3; i++) for(int j=0; j<3; j++) for(int k=0; k<2; k++) pResult->mGrid.mVelocity[i][j][k] = pPrevious->mGrid.mVelocity[i][j][k] * fDamping; XMVECTOR center = GESTURE_GET_POS( pSkeletonData ); XMVECTOR jointvel[GESTURE_JOINT_COUNT]; for ( UINT i = 0; i < GESTURE_JOINT_COUNT; i++ ) { jointvel[i] = XMVectorZero(); if ( GESTURE_GET_JOINT_TRACKING( pSkeletonData, i ) == GESTURE_JOINT_NOT_TRACKED) continue; if ( GESTURE_GET_JOINT_TRACKING( pSkeletonDataPrev, i ) == GESTURE_JOINT_NOT_TRACKED) continue; jointvel[i] = XMVectorSubtract( GESTURE_GET_JOINT_POS( pSkeletonData, i ), GESTURE_GET_JOINT_POS( pSkeletonDataPrev, i ) ); } for ( UINT line = 0; line < g_uNumBoneLines; line++ ) { int i = g_BoneLines[line].StartJoint; int j = g_BoneLines[line].EndJoint; if ( GESTURE_GET_JOINT_TRACKING( pSkeletonData, i ) == GESTURE_JOINT_NOT_TRACKED) continue; if ( GESTURE_GET_JOINT_TRACKING( pSkeletonData, j ) == GESTURE_JOINT_NOT_TRACKED) continue; if ( GESTURE_GET_JOINT_TRACKING( pSkeletonDataPrev, i ) == GESTURE_JOINT_NOT_TRACKED) continue; if ( GESTURE_GET_JOINT_TRACKING( pSkeletonDataPrev, j ) == GESTURE_JOINT_NOT_TRACKED) continue; float scale2 = fScale * g_BoneLines[line].Priority; for(float f = 0.0f; f<= 1.0f; f += fStep) { XMVECTOR pos = XMVectorLerp( GESTURE_GET_JOINT_POS( pSkeletonData, i ), GESTURE_GET_JOINT_POS( pSkeletonData, j ), f ); XMVECTOR vel = XMVectorLerp( jointvel[i], jointvel[j], f ); XMVECTOR relPos = XMVectorSubtract(pos, center); float x = XMVectorGetX(relPos); float y = XMVectorGetY(relPos); int cellx = (int)(CLAMP((x - -1.0f) * ((float)KINECT_GESTURE_VELOCITY_GRID_X / 2.0f), 0, ((float)KINECT_GESTURE_VELOCITY_GRID_X - 0.51f))); int celly = (int)(CLAMP((1.0f - y) * ((float)KINECT_GESTURE_VELOCITY_GRID_Y / 2.0f), 0, ((float)KINECT_GESTURE_VELOCITY_GRID_Y - 0.51f))); vel = XMVectorMultiply(vel, XMVectorSet(scale2, -scale2, 0.0f, 0.0f)); pResult->mGrid.mVelocity[cellx][celly][0] += XMVectorGetX( vel ); pResult->mGrid.mVelocity[cellx][celly][1] += XMVectorGetY( vel ); } } // for ( UINT i = 0; i < NUI_SKELETON_POSITION_COUNT; i++ ) // { // if (pSkeletonData->eSkeletonPositionTrackingState[i] == NUI_SKELETON_POSITION_NOT_TRACKED) continue; // if (pPrevious->mSkeletonDataPrev.eSkeletonPositionTrackingState[i] == NUI_SKELETON_POSITION_NOT_TRACKED) continue; // // XMVECTOR relPos = XMVectorSubtract(pSkeletonData->SkeletonPositions[i], pos); // XMVECTOR vel = XMVectorSubtract(pSkeletonData->SkeletonPositions[i], // pPrevious->mSkeletonDataPrev.SkeletonPositions[i]); // float x = XMVectorGetX(relPos); // float y = XMVectorGetY(relPos); // int cellx = CLAMP((int)((x - -1.0f) * (3.0f/2.0f)), 0, 2); // int celly = CLAMP((int)((1.0f - y) * (3.0f/2.0f)), 0, 2); // // vel = XMVectorMultiply(vel, XMVectorSet(scale, -scale, scale, 0.0f)); // pResult->mVelocity[cellx][celly] = XMVectorAdd(pResult->mVelocity[cellx][celly], vel); // } pResult->mSkeletonDataPrev = *pSkeletonData; return S_OK; } }