HoloLens 2 で指のジェスチャー判定を行う

本日はHoloLens 2の表現調査の記事です。

HoloLens 2ではHandTrackingによってユーザーの手を取得できます。

これはMixedRealityPoseとして各関節ごとに取得してアプリ内でデータを扱うことができます。

今回はオリジナルでジェスチャーを作成する一環として、指が伸びているのか、曲げられているのかを検知できる機能を作成しました。

今回は紹介記事です。明日の記事で解説を行います。

〇実機

今回は人差し指、中指を検知し、指が伸ばされていると各色のボールが出現するようにしました。

youtu.be

〇コード

次のようなスクリプトを書きました。

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using Microsoft.MixedReality.Toolkit;
using Microsoft.MixedReality.Toolkit.Input;
using Microsoft.MixedReality.Toolkit.Utilities;
using UnityEngine.Events;


public class FingerDitected : MonoBehaviour
{
    [SerializeField,Header("Target")]
    Handedness handType;
    
    [SerializeField,Range(0,90)]
    private float indexThreshold = 5;

    [SerializeField,Range(0,90)]
    private float middleThreshold = 5;

    [SerializeField] 
    UnityEvent IndexFingerDetectEvent;

    [SerializeField] 
    UnityEvent IndexFingerLostEvent;

    [SerializeField] 
    UnityEvent MiddleFingerDetectEvent;

    [SerializeField] 
    UnityEvent MiddleFingerLostEvent;

    private bool? handdetected;


    void Update()
    {
        handdetected = HandJointUtils.FindHand(handType)?.TryGetJoint(TrackedHandJoint.Palm, out MixedRealityPose PalmPose);
        if (handdetected != null&& handdetected==true)
        {
            if (indexfingerDetected())
            {
                IndexFingerDetectEvent.Invoke();
            }
            else
            {
                IndexFingerLostEvent.Invoke();
            }

            if (middlefingerDetected())
            {
                MiddleFingerDetectEvent.Invoke();
            }
            else
            {
                MiddleFingerLostEvent.Invoke();
            }   
        }
    }
    
    private bool indexfingerDetected()
    {
        var jointedHand = HandJointUtils.FindHand(handType);
        if (jointedHand.TryGetJoint(TrackedHandJoint.Palm,out MixedRealityPose PalmPose))
        {
            //各関節のpose
            MixedRealityPose indexTipPose,indexDistalPose,IndexKnucklePose,indexMiddlePose; 
            if(jointedHand.TryGetJoint(TrackedHandJoint.IndexTip,out indexTipPose)&& jointedHand.TryGetJoint(TrackedHandJoint.IndexDistalJoint,out indexDistalPose)&&jointedHand.TryGetJoint(TrackedHandJoint.IndexMiddleJoint,out indexMiddlePose)&& jointedHand.TryGetJoint(TrackedHandJoint.IndexKnuckle,out IndexKnucklePose))
            {
                Vector3 f1 = IndexKnucklePose.Position - PalmPose.Position;
                Vector3 f2 = indexMiddlePose.Position - IndexKnucklePose.Position;
                Vector3 f3 = indexDistalPose.Position - indexMiddlePose.Position;
                Vector3 f4 = indexTipPose.Position - indexDistalPose.Position;
                
            float c = Vector3.Angle(PalmPose.Position, f1);
            float d = Vector3.Angle(f1, f2);
            float e = Vector3.Angle(f2, f3);
            float f = Vector3.Angle(f3, f4);

            float aba = (Mathf.Abs(d) + Mathf.Abs(e) + Mathf.Abs(f)) / 3;

                if (aba < indexThreshold)
                {
                    return true;
                }
            }
        }
        return false;
    }

    private bool middlefingerDetected()
    {
        var jointedHand = HandJointUtils.FindHand(handType);
        if (jointedHand.TryGetJoint(TrackedHandJoint.Palm, out MixedRealityPose PalmPose))
        {
            MixedRealityPose middleTipsPose,middleDistalPose, middleKnucklePose,middleMiddlePose;
            if (jointedHand.TryGetJoint(TrackedHandJoint.MiddleTip, out middleTipsPose) &&
                jointedHand.TryGetJoint(TrackedHandJoint.MiddleDistalJoint, out middleDistalPose) &&
                jointedHand.TryGetJoint(TrackedHandJoint.MiddleKnuckle, out middleKnucklePose)&&jointedHand.TryGetJoint(TrackedHandJoint.MiddleMiddleJoint,out middleMiddlePose))
            {
                Vector3 f1= middleKnucklePose.Position - PalmPose.Position;
                Vector3 f2 = middleMiddlePose.Position - middleKnucklePose.Position;
                Vector3 f3 = middleDistalPose.Position - middleMiddlePose.Position;
                Vector3 f4 = middleTipsPose.Position - middleDistalPose.Position;
                
                float c = Vector3.Angle(PalmPose.Position, f1);
                float d = Vector3.Angle(f1, f2);
                float e = Vector3.Angle(f2, f3);
                float f = Vector3.Angle(f3, f4);

                float aba = (Mathf.Abs(d) + Mathf.Abs(e) + Mathf.Abs(f)) / 3;

                if (aba < middleThreshold)
                {
                    return true;
                }
                
            }
        }

        return false;
    }
}