このイベントは関西のxRコミュニティ大阪駆動開発が主催のHoloLensイベントで、毎年行われているイベントが今年は新型コロナウィルス感染症の影響を受けオンラインの形で行われました。
本日は７日目の取り組みになります。
〇ボッチソン『HoloLensで片手で操作できる未来感あるUI』
今回筆者が取り組むテーマは『HoloLensで片手で操作できる未来感あるUI』です。
これはHoloLens 2で使用できるHandTrackingを用いたオリジナルのUIを作成するプロダクトで、MRTKで提供されているHandMenuなどデフォルトのHandUIは片手操作ができないという門外があります。
様々な人が今後HoloLensを使っていく中で片手で操作できるUIというものを作りたいと思いテーマとして選びました。
〇指検知の改良
今回は指検知の改良を行います。
二日目に作成した指の曲げ伸び状態の検知の機能ですが、バグが起こっていましたのでリファクタリングもかねてコードを修正しました。
●指検知とイベントの分離
前回までは指検知を行う[FingerGestureDetect]コンポーネントないでAnimationのトリガーを呼び出し検知後のイベントも行っていましたが、今回指を検知する機能とそれを用いてイベントを行う２つのコンポーネントに分けました。
以下のスクリプトを作成しました。
using System;
using System.Collections;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using UnityEngine;
using UnityEngine.Events;
[DisallowMultipleComponent,RequireComponent(typeof(FingerGestureDetect))]
public class FingerGesture : MonoBehaviour
{
[SerializeField]
FingerGestureDetect fingerDetect;
[SerializeField,Header("WaitTime")]
public float SelectWaitTime;
[Header("FingerStatus")]
public bool thumbActive, indexActive, middleActive, ringAndPinkyActive=false;
[SerializeField,Header("Event")]
UnityEvent firstInventory;
[SerializeField]
UnityEvent SecondInventory, ThirdInventory,LostInventry;
[SerializeField] private UnityEvent IndexOn, MiddleOn, RingOn, IndexOff, MiddleOff, RingOff;
[SerializeField]
private bool eventstatus1, eventstatus2, eventstatus3 = false;
private bool setFirst,setSecond,setThird = true;
public void fingerStatus(string name)
{
switch (name)
{
case "thumb":
thumbActive = true;
break;
case "index":
indexActive = true;
break;
case "middle":
middleActive = true;
break;
case "ringandpinky":
ringAndPinkyActive = true;
break;
case "thumblost":
thumbActive = false;
break;
case "indexlost":
indexActive = false;
break;
case "middlelost":
middleActive = false;
break;
case "ringandpinkylost":
ringAndPinkyActive = false;
break;
}
}
// Start is called before the first frame update
void Start()
{
}
private void Update()
{
if (!indexActive&&!middleActive&&!ringAndPinkyActive)
{
Debug.Log("4");
// LostInventry.Invoke();
if (eventstatus1)
{
IndexOff.Invoke();
}
if (eventstatus2)
{
MiddleOff.Invoke();
}
if (eventstatus3)
{
MiddleOff.Invoke();
}
}
if (indexActive&&!middleActive&&!ringAndPinkyActive)
{
if (setFirst)
{
Debug.Log("1");
setFirst = !setFirst;
eventstatus1 = true;
fingerEvent("index");
}
}
else
{
if (!setFirst )
{
setFirst = false;
if (eventstatus1)
{
eventstatus1 = false;
}
}
}
if (indexActive && middleActive && !ringAndPinkyActive)
{
if (setSecond)
{
Debug.Log("2");
setSecond = !setSecond;
eventstatus2 = true;
fingerEvent("middle");
}
}
else
{
if (!setSecond)
{
setSecond = true;
if (eventstatus2)
{
eventstatus2 = false;
}
}
}
if (indexActive&&middleActive&&ringAndPinkyActive)
{
if (setThird)
{
Debug.Log("3");
setThird = !setThird;
eventstatus3 = true;
fingerEvent("ring");
}
}
else
{
if (!setThird)
{
setThird = true;
if (eventstatus3)
{
eventstatus3 = false;
}
}
}
}
public void fingerEvent(string st)
{
switch (st)
{
case "index":
IndexOn.Invoke();
if (eventstatus2)
{
MiddleOff.Invoke();
}
if (eventstatus3)
{
RingOff.Invoke();
}
break;
case"middle":
MiddleOn.Invoke();
if (eventstatus1)
{
IndexOff.Invoke();
}
if (eventstatus3)
{
RingOff.Invoke();
}
break;
case "ring":
if (eventstatus1)
{
IndexOff.Invoke();
}
if (eventstatus2)
{
MiddleOff.Invoke();
}
RingOn.Invoke();
break;
}
}
private void Reset()
{
fingerDetect = this.GetComponent<FingerGestureDetect>();
}
}
この[FingerGesture]を[FingerGestureDetect]がアタッチされているオブジェクトにアタッチします。
[DisallowMultipleComponent,RequireComponent(typeof(FingerGestureDetect))]
public class FingerGesture : MonoBehaviour
{
...
}
[FingerGesture]コンポーネントにはDisallowMultipleComponentとRequireComponent(typeof(FingerGestureDetect))属性を付けています。
RequireComponent(typeof(FingerGestureDetect))は[FingerGesture]コンポーネントをゲームオブジェクトにアタッチした際に[FingerGestureDetect]コンポーネントがアタッチされていない場合自動的にアタッチされます。
またDisallowMultipleComponentの属性により一つのゲームオブジェクトに[FingerGesture]コンポーネントが一つ以上アタッチできないようになりました。
①[FingerGestureDetect]コンポーネントの[On（各指名）FingerDetect()]に[FingerGesture]コンポーネントの[FingerGesture.fingerStatus()]を設定し、引数に各指名(人差し指ならIndex)を設定します。
反対に[FingerGestureDetect]コンポーネントの[On（各指名）FingerLost()]に[FingerGesture]コンポーネントの[FingerGesture.fingerStatus()]を設定し、引数に各指名+lost(人差し指ならIndexlost)を設定します。
これで指が伸びている間は[FingerGesture]の対応する指のbool型にチェックが付くようになります。
以上で指の検知とそれによって実行されるイベントを２つのコンポーネントに分離する仕組みが完成しました。
〇コード全文
●FingerGestureDetect
指の曲げ伸びの検知を行います。
using System;
using UnityEngine;
using Microsoft.MixedReality.Toolkit.Input;
using Microsoft.MixedReality.Toolkit.Utilities;
using Microsoft.MixedReality.Toolkit.Utilities.Solvers;
using UnityEngine.Events;
public class FingerGestureDetect : MonoBehaviour
{
[SerializeField,Header("Target"),Tooltip("Tracked Hands")]
Handedness handType;
[SerializeField,Range(0,90),Header("Threshold")]
private float indexThreshold = 5;
[SerializeField,Range(0,90)]
private float middleThreshold,ringAndPinkyFingerThreshpld, thumbThreshold;
[SerializeField, Range(0, 90)]
private float facingThreshold;
[SerializeField,Header("Index"),Header("Events")]
UnityEvent OnIndexFingerDetect;
[SerializeField]
UnityEvent OnIndexFingerLost;
[SerializeField,Header("Middle")]
UnityEvent OnMiddleFingerDetect;
[SerializeField]
UnityEvent OnMiddleFingerLost;
[SerializeField, Header("Ring And Pinky")]
UnityEvent OnRingAndPinkyDetect;
[SerializeField]
UnityEvent OnRingAndPinkyLost;
[SerializeField,Header("Thumb"),]
UnityEvent OnThumbFingerDetect;
[SerializeField]
UnityEvent OnThumbFingerLost;
[SerializeField, Header("FirstEvent")]
UnityEvent OnIndexFingerFirst;
[SerializeField]
UnityEvent OnMiddleFingerFirst,OnRingAndPinkyFingerFirst;
[SerializeField, Header("FirstLostEvent")]
UnityEvent OnIndexLostFirst;
[SerializeField] UnityEvent OnMiddleLostFirst,OnRingAndPinkyLostFirst;
bool? handdetected;
bool indexFirstfinger,middleFirstfinger,ringandPinkyFirstfinger,indexFirstlostfinger,middleFirstLostfinger,ringandPinkyFirstLostfinger;
// Update is called once per frame
void Update()
{
handdetected = HandJointUtils.FindHand(handType)?.TryGetJoint(TrackedHandJoint.Palm, out MixedRealityPose PalmPose);
if ( handdetected != null && handdetected==true )
{
if (indexfingerDetected()&& HanddirDetected())
{
Debug.Log("index:Open");
OnIndexFingerDetect.Invoke();
if (indexFirstfinger)
{
indexFirstfinger = false;
OnIndexFingerFirst.Invoke();
}
}
else
{
Debug.Log("index:Close");
OnIndexFingerLost.Invoke();
if (!indexFirstfinger)
{
indexFirstfinger = true;
OnIndexLostFirst.Invoke();
}
}
if (middlefingerDetected()&& HanddirDetected())
{
Debug.Log("middle:Open");
OnMiddleFingerDetect.Invoke();
if (middleFirstfinger)
{
middleFirstfinger = false;
OnMiddleFingerFirst.Invoke();
}
}
else
{
Debug.Log("middle:Close");
OnMiddleFingerLost.Invoke();
if (!middleFirstfinger)
{
middleFirstfinger = true;
OnMiddleLostFirst.Invoke();
}
}
if (RingAndPinkyFingerDetected()&& HanddirDetected())
{
Debug.Log("RingAndPinky:Open");
OnRingAndPinkyDetect.Invoke();
if (ringandPinkyFirstfinger)
{
ringandPinkyFirstfinger = false;
OnRingAndPinkyFingerFirst.Invoke();
}
}
else
{
OnRingAndPinkyLost.Invoke();
if (!ringandPinkyFirstfinger)
{
ringandPinkyFirstfinger = true;
//ringandPinkyFirstLostfinger = false;
OnRingAndPinkyLostFirst.Invoke();
}
}
}
}
private bool HanddirDetected()
{
var jointedHand = HandJointUtils.FindHand(handType);
if (jointedHand.TryGetJoint(TrackedHandJoint.Palm,out MixedRealityPose palmPose))
{
if ( facingThreshold<Vector3.Angle(palmPose.Up, CameraCache.Main.transform.forward) )
{
// Debug.Log(palmPose.Up);
return true;
}
}
return false;
}
private bool indexfingerDetected()
{
var jointedHand = HandJointUtils.FindHand(handType);
if (jointedHand.TryGetJoint(TrackedHandJoint.Palm,out MixedRealityPose PalmPose))
{
//各関節のpose
MixedRealityPose indexTipPose,indexDistalPose,IndexKnucklePose,indexMiddlePose;
if(jointedHand.TryGetJoint(TrackedHandJoint.IndexTip,out indexTipPose)&& jointedHand.TryGetJoint(TrackedHandJoint.IndexDistalJoint,out indexDistalPose)&&jointedHand.TryGetJoint(TrackedHandJoint.IndexMiddleJoint,out indexMiddlePose)&& jointedHand.TryGetJoint(TrackedHandJoint.IndexKnuckle,out IndexKnucklePose))
{
Vector3 finger1 = IndexKnucklePose.Position - PalmPose.Position;
Vector3 finger2 = indexMiddlePose.Position - IndexKnucklePose.Position;
Vector3 finger3 = indexDistalPose.Position - indexMiddlePose.Position;
Vector3 finger4 = indexTipPose.Position - indexDistalPose.Position;
float c = Vector3.Angle(PalmPose.Position, finger1);
float d = Vector3.Angle(finger1, finger2);
float e = Vector3.Angle(finger2, finger3);
float f = Vector3.Angle(finger3, finger4);
float aba = (Mathf.Abs(d) + Mathf.Abs(e) + Mathf.Abs(f)) / 3;
if (aba < indexThreshold)
{
return true;
}
}
}
return false;
}
private bool middlefingerDetected()
{
var jointedHand = HandJointUtils.FindHand(handType);
if (jointedHand.TryGetJoint(TrackedHandJoint.Palm, out MixedRealityPose PalmPose))
{
MixedRealityPose middleTipsPose,middleDistalPose, middleKnucklePose,middleMiddlePose;
if (jointedHand.TryGetJoint(TrackedHandJoint.MiddleTip, out middleTipsPose) &&
jointedHand.TryGetJoint(TrackedHandJoint.MiddleDistalJoint, out middleDistalPose) &&
jointedHand.TryGetJoint(TrackedHandJoint.MiddleKnuckle, out middleKnucklePose)&&jointedHand.TryGetJoint(TrackedHandJoint.MiddleMiddleJoint,out middleMiddlePose))
{
Vector3 finger1 = middleKnucklePose.Position - PalmPose.Position;
Vector3 finger2 = middleMiddlePose.Position - middleKnucklePose.Position;
Vector3 finger3 = middleDistalPose.Position - middleMiddlePose.Position;
Vector3 finger4 = middleTipsPose.Position - middleDistalPose.Position;
float c = Vector3.Angle(PalmPose.Position, finger1);
float d = Vector3.Angle(finger1, finger2);
float e = Vector3.Angle(finger2, finger3);
float f = Vector3.Angle(finger3, finger4);
float aba = (Mathf.Abs(d) + Mathf.Abs(e) + Mathf.Abs(f)) / 3;
if (aba < middleThreshold)
{
return true;
}
}
}
return false;
}
private bool RingAndPinkyFingerDetected()
{
//※メモ HoloLens2の場合実装的には薬指、小指それぞれの判定は可能だが、実機の場合特に小指単体のトラッキング精度に問題があるため薬指、小指はひとまとめで一つの指として検知する。
var jointedHand =HandJointUtils.FindHand(handType);
if (jointedHand.TryGetJoint(TrackedHandJoint.Palm,out MixedRealityPose PalmPose))
{
MixedRealityPose PinkyTipPose,
PinkyDistalPose,
PinkyMiddlePose,
PinkyKnucklePose,
RingTipPose,
RingDistalPose,
RingMiddlePose,
RingKnucklePose;
if (jointedHand.TryGetJoint(TrackedHandJoint.RingTip,out RingTipPose)&& jointedHand.TryGetJoint(TrackedHandJoint.RingMiddleJoint ,out RingMiddlePose)&&jointedHand.TryGetJoint(TrackedHandJoint.RingDistalJoint ,out RingDistalPose)
&& jointedHand.TryGetJoint(TrackedHandJoint.RingKnuckle ,out RingKnucklePose))
{
Vector3 finger1 = RingKnucklePose.Position - PalmPose.Position;
Vector3 finger2 = RingMiddlePose.Position - RingKnucklePose.Position;
Vector3 finger3 = RingDistalPose.Position - RingMiddlePose.Position;
Vector3 finger4 = RingTipPose.Position - RingDistalPose.Position;
float c = Vector3.Angle(PalmPose.Position, finger1);
float d = Vector3.Angle(finger1, finger2);
float e = Vector3.Angle(finger2, finger3);
float f = Vector3.Angle(finger3, finger4);
float aba = (Mathf.Abs(d) + Mathf.Abs(e) + Mathf.Abs(f)) / 3;
Debug.Log(aba);
if (aba < ringAndPinkyFingerThreshpld)
{
return true;
}
}
}
return false;
}
private bool ThumbDetected()
{
//※メモ 親指は他の指と異なり特殊でまっすぐ伸びているか？の判定＋親指の向きで判定する。
var jointedHand = HandJointUtils.FindHand(handType);
//Get TargetHamd
if (jointedHand.TryGetJoint(TrackedHandJoint.Palm, out MixedRealityPose PalmPose))
{
MixedRealityPose ThumbTipPose, ThumbDistalPose, ThumbProximalPose;
//Get finger Joint Deta
if (jointedHand.TryGetJoint(TrackedHandJoint.ThumbTip, out ThumbTipPose) &&
jointedHand.TryGetJoint(TrackedHandJoint.ThumbDistalJoint, out ThumbDistalPose) &&
jointedHand.TryGetJoint(TrackedHandJoint.ThumbProximalJoint ,out ThumbProximalPose))
{
Vector3 finger1 = ThumbProximalPose.Position - PalmPose.Position;
Vector3 finger2 = ThumbDistalPose.Position - ThumbProximalPose.Position;
Vector3 finger3 = ThumbTipPose.Position - ThumbProximalPose.Position;
float c = Vector3.Angle(PalmPose.Position, finger1);
float d = Vector3.Angle(finger1, finger2);
float e = Vector3.Angle(finger2, finger3);
float aba = (Mathf.Abs(d) + Mathf.Abs(e)/ 2);
if (aba < thumbThreshold)
{
return true;
}
}
}
return false;
}
private void Reset()
{
if (this.GetComponent<SolverHandler>() != null)
{
handType = this.GetComponent<SolverHandler>().TrackedHandness;
}
}
}
●FingerGesture
[FingerGestureDetect]から手の検知状態を受けとります。
using System;
using System.Collections;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using UnityEngine;
using UnityEngine.Events;
[DisallowMultipleComponent,RequireComponent(typeof(FingerGestureDetect))]
public class FingerGesture : MonoBehaviour
{
[SerializeField]
FingerGestureDetect fingerDetect;
[SerializeField,Header("WaitTime")]
public float SelectWaitTime;
[Header("FingerStatus")]
public bool thumbActive, indexActive, middleActive, ringAndPinkyActive=false;
[SerializeField,Header("Event")]
UnityEvent firstInventory;
[SerializeField]
UnityEvent SecondInventory, ThirdInventory,LostInventry;
[SerializeField] private UnityEvent IndexOn, MiddleOn, RingOn, IndexOff, MiddleOff, RingOff;
[SerializeField]
private bool eventstatus1, eventstatus2, eventstatus3 = false;
private bool setFirst,setSecond,setThird = true;
public void fingerStatus(string name)
{
switch (name)
{
case "thumb":
thumbActive = true;
break;
case "index":
indexActive = true;
break;
case "middle":
middleActive = true;
break;
case "ringandpinky":
ringAndPinkyActive = true;
break;
case "thumblost":
thumbActive = false;
break;
case "indexlost":
indexActive = false;
break;
case "middlelost":
middleActive = false;
break;
case "ringandpinkylost":
ringAndPinkyActive = false;
break;
}
}
// Start is called before the first frame update
void Start()
{
}
private void Update()
{
if (!indexActive&&!middleActive&&!ringAndPinkyActive)
{
Debug.Log("4");
// LostInventry.Invoke();
if (eventstatus1)
{
IndexOff.Invoke();
}
if (eventstatus2)
{
MiddleOff.Invoke();
}
if (eventstatus3)
{
MiddleOff.Invoke();
}
}
if (indexActive&&!middleActive&&!ringAndPinkyActive)
{
if (setFirst)
{
Debug.Log("1");
setFirst = !setFirst;
eventstatus1 = true;
fingerEvent("index");
}
}
else
{
if (!setFirst )
{
setFirst = false;
if (eventstatus1)
{
eventstatus1 = false;
}
}
}
if (indexActive && middleActive && !ringAndPinkyActive)
{
if (setSecond)
{
Debug.Log("2");
setSecond = !setSecond;
eventstatus2 = true;
fingerEvent("middle");
}
}
else
{
if (!setSecond)
{
setSecond = true;
if (eventstatus2)
{
eventstatus2 = false;
}
}
}
if (indexActive&&middleActive&&ringAndPinkyActive)
{
if (setThird)
{
Debug.Log("3");
setThird = !setThird;
eventstatus3 = true;
fingerEvent("ring");
}
}
else
{
if (!setThird)
{
setThird = true;
if (eventstatus3)
{
eventstatus3 = false;
}
}
}
}
public void fingerEvent(string st)
{
switch (st)
{
case "index":
IndexOn.Invoke();
if (eventstatus2)
{
MiddleOff.Invoke();
}
if (eventstatus3)
{
RingOff.Invoke();
}
break;
case"middle":
MiddleOn.Invoke();
if (eventstatus1)
{
IndexOff.Invoke();
}
if (eventstatus3)
{
RingOff.Invoke();
}
break;
case "ring":
if (eventstatus1)
{
IndexOff.Invoke();
}
if (eventstatus2)
{
MiddleOff.Invoke();
}
RingOn.Invoke();
break;
}
}
private void Reset()
{
fingerDetect = this.GetComponent<FingerGestureDetect>();
}
}
