HoloLens 2 で指のジェスチャー判定を行うコード解説

本日はHoloLens 2の表現探求枠です。

昨日指のジェスチャー判定を子なうコードと使い方を紹介しました。

redhologerbera.hatenablog.com

今回はコードの解説を行います。

〇コード全文

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using Microsoft.MixedReality.Toolkit;
using Microsoft.MixedReality.Toolkit.Input;
using Microsoft.MixedReality.Toolkit.Utilities;
using UnityEngine.Events;


public class FingerDitected : MonoBehaviour
{
    [SerializeField,Header("Target")]
    Handedness handType;
    
    [SerializeField,Range(0,90)]
    private float indexThreshold = 5;

    [SerializeField,Range(0,90)]
    private float middleThreshold = 5;

    [SerializeField] 
    UnityEvent IndexFingerDetectEvent;

    [SerializeField] 
    UnityEvent IndexFingerLostEvent;

    [SerializeField] 
    UnityEvent MiddleFingerDetectEvent;

    [SerializeField] 
    UnityEvent MiddleFingerLostEvent;

    private bool? handdetected;


    void Update()
    {
        handdetected = HandJointUtils.FindHand(handType)?.TryGetJoint(TrackedHandJoint.Palm, out MixedRealityPose PalmPose);
        if (handdetected != null&& handdetected==true)
        {
            if (indexfingerDetected())
            {
                IndexFingerDetectEvent.Invoke();
            }
            else
            {
                IndexFingerLostEvent.Invoke();
            }

            if (middlefingerDetected())
            {
                MiddleFingerDetectEvent.Invoke();
            }
            else
            {
                MiddleFingerLostEvent.Invoke();
            }   
        }
    }
    
    private bool indexfingerDetected()
    {
        var jointedHand = HandJointUtils.FindHand(handType);
        if (jointedHand.TryGetJoint(TrackedHandJoint.Palm,out MixedRealityPose PalmPose))
        {
            //各関節のpose
            MixedRealityPose indexTipPose,indexDistalPose,IndexKnucklePose,indexMiddlePose; 
            if(jointedHand.TryGetJoint(TrackedHandJoint.IndexTip,out indexTipPose)&& jointedHand.TryGetJoint(TrackedHandJoint.IndexDistalJoint,out indexDistalPose)&&jointedHand.TryGetJoint(TrackedHandJoint.IndexMiddleJoint,out indexMiddlePose)&& jointedHand.TryGetJoint(TrackedHandJoint.IndexKnuckle,out IndexKnucklePose))
            {
                Vector3 f1 = IndexKnucklePose.Position - PalmPose.Position;
                Vector3 f2 = indexMiddlePose.Position - IndexKnucklePose.Position;
                Vector3 f3 = indexDistalPose.Position - indexMiddlePose.Position;
                Vector3 f4 = indexTipPose.Position - indexDistalPose.Position;
                
            float c = Vector3.Angle(PalmPose.Position, f1);
            float d = Vector3.Angle(f1, f2);
            float e = Vector3.Angle(f2, f3);
            float f = Vector3.Angle(f3, f4);

            float aba = (Mathf.Abs(d) + Mathf.Abs(e) + Mathf.Abs(f)) / 3;

                if (aba < indexThreshold)
                {
                    return true;
                }
            }
        }
        return false;
    }

    private bool middlefingerDetected()
    {
        var jointedHand = HandJointUtils.FindHand(handType);
        if (jointedHand.TryGetJoint(TrackedHandJoint.Palm, out MixedRealityPose PalmPose))
        {
            MixedRealityPose middleTipsPose,middleDistalPose, middleKnucklePose,middleMiddlePose;
            if (jointedHand.TryGetJoint(TrackedHandJoint.MiddleTip, out middleTipsPose) &&
                jointedHand.TryGetJoint(TrackedHandJoint.MiddleDistalJoint, out middleDistalPose) &&
                jointedHand.TryGetJoint(TrackedHandJoint.MiddleKnuckle, out middleKnucklePose)&&jointedHand.TryGetJoint(TrackedHandJoint.MiddleMiddleJoint,out middleMiddlePose))
            {
                Vector3 f1= middleKnucklePose.Position - PalmPose.Position;
                Vector3 f2 = middleMiddlePose.Position - middleKnucklePose.Position;
                Vector3 f3 = middleDistalPose.Position - middleMiddlePose.Position;
                Vector3 f4 = middleTipsPose.Position - middleDistalPose.Position;
                
                float c = Vector3.Angle(PalmPose.Position, f1);
                float d = Vector3.Angle(f1, f2);
                float e = Vector3.Angle(f2, f3);
                float f = Vector3.Angle(f3, f4);

                float aba = (Mathf.Abs(d) + Mathf.Abs(e) + Mathf.Abs(f)) / 3;

                if (aba < middleThreshold)
                {
                    return true;
                }
                
            }
        }

        return false;
    }
}

〇Update関数

  void Update()
    {
         handdetected = HandJointUtils.FindHand(handType)?.TryGetJoint(TrackedHandJoint.Palm, out MixedRealityPose PalmPose);
        if ( handdetected != null&&  handdetected==true)
        {
            if (indexfingerDetected())
            {
                IndexFingerDetectEvent.Invoke();
            }
            else
            {
                IndexFingerLostEvent.Invoke();
            }

            if (middlefingerDetected())
            {
                MiddleFingerDetectEvent.Invoke();
            }
            else
            {
                MiddleFingerLostEvent.Invoke();
            }   
        }
    }

ここではif文で条件分けを行いTrueの場合、対応するイベントが発火します。

[handdetected]はboolのnull許容型で定義しています。　これはMRTKを用いた場合手を検知した場合のみシーン内に出現するため、HandTrackingを検知していない場合Nullが発生するためです。

  handdetected = HandJointUtils.FindHand(handType)?.TryGetJoint(TrackedHandJoint.Palm, out MixedRealityPose PalmPose);

始めに毎フレームごとに対象のHandTrackingが存在するか調べます。

  if ( handdetected != null&&  handdetected==true)
        {
            ...
        }

HandTrackingが存在し、HandTrackingが使用できる場合次の計算が行われます。

   if (indexfingerDetected())
            {
                IndexFingerDetectEvent.Invoke();
            }
            else
            {
                IndexFingerLostEvent.Invoke();
            }

            if (middlefingerDetected())
            {
                MiddleFingerDetectEvent.Invoke();
            }
            else
            {
                MiddleFingerLostEvent.Invoke();
            }   
        }

ここではindexfingerDetected()、middlefingerDetected()の帰り値を基に指の検知状態を判断しそれぞれ指が伸びている場合、曲げられている場合のイベントを発火します。

〇indexfingerDetected()、middlefingerDetected()

    private bool indexfingerDetected()
    {
        var jointedHand = HandJointUtils.FindHand(handType);
        if (jointedHand.TryGetJoint(TrackedHandJoint.Palm,out MixedRealityPose PalmPose))
        {
            //各関節のpose
            MixedRealityPose indexTipPose,indexDistalPose,IndexKnucklePose,indexMiddlePose; 
            if(jointedHand.TryGetJoint(TrackedHandJoint.IndexTip,out indexTipPose)&& jointedHand.TryGetJoint(TrackedHandJoint.IndexDistalJoint,out indexDistalPose)&&jointedHand.TryGetJoint(TrackedHandJoint.IndexMiddleJoint,out indexMiddlePose)&& jointedHand.TryGetJoint(TrackedHandJoint.IndexKnuckle,out IndexKnucklePose))
            {
                Vector3 f1 = IndexKnucklePose.Position - PalmPose.Position;
                Vector3 f2 = indexMiddlePose.Position - IndexKnucklePose.Position;
                Vector3 f3 = indexDistalPose.Position - indexMiddlePose.Position;
                Vector3 f4 = indexTipPose.Position - indexDistalPose.Position;
                
            float c = Vector3.Angle(PalmPose.Position, f1);
            float d = Vector3.Angle(f1, f2);
            float e = Vector3.Angle(f2, f3);
            float f = Vector3.Angle(f3, f4);

            float aba = (Mathf.Abs(d) + Mathf.Abs(e) + Mathf.Abs(f)) / 3;

                if (aba < indexThreshold)
                {
                    return true;
                }
            }
        }
        return false;
    }

    var jointedHand = HandJointUtils.FindHand(handType);
        if (jointedHand.TryGetJoint(TrackedHandJoint.Palm,out MixedRealityPose PalmPose))
        {
           ...
        }

jointedHandには、プロパティで選択したhandTypeに応じて指定した左手、右手がトラッキングされます。

        MixedRealityPose indexTipPose,indexDistalPose,IndexKnucklePose,indexMiddlePose; 
            if(jointedHand.TryGetJoint(TrackedHandJoint.IndexTip,out indexTipPose)&& jointedHand.TryGetJoint(TrackedHandJoint.IndexDistalJoint,out indexDistalPose)&&jointedHand.TryGetJoint(TrackedHandJoint.IndexMiddleJoint,out indexMiddlePose)&& jointedHand.TryGetJoint(TrackedHandJoint.IndexKnuckle,out IndexKnucklePose))
            {
                Vector3 f1 = IndexKnucklePose.Position - PalmPose.Position;
                Vector3 f2 = indexMiddlePose.Position - IndexKnucklePose.Position;
                Vector3 f3 = indexDistalPose.Position - indexMiddlePose.Position;
                Vector3 f4 = indexTipPose.Position - indexDistalPose.Position;
                
            float c = Vector3.Angle(PalmPose.Position, f1);
            float d = Vector3.Angle(f1, f2);
            float e = Vector3.Angle(f2, f3);
            float f = Vector3.Angle(f3, f4);

            float aba = (Mathf.Abs(d) + Mathf.Abs(e) + Mathf.Abs(f)) / 3;

                if (aba < indexThreshold)
                {
                    return true;
                }
            }

if文では人差し指、中指の各関節の位置情報が取得されます。

finger(f1~f4)は各指の関節をPalmの子オブジェクトに変換しています。これは元々各関節の位置情報がワールド座標で産出されるためです。

この語Vector3.Angle(A,B)で各関節間の角度を取得します。

docs.unity3d.com

この値がそのまま指の曲げ角に該当します。　これの平均値が変数indexThreshold,middleThresholdで指定した値より下回っている場合（角度が０に近いほど）指が伸びていると判断してTrueを返しています。

以上になります。