using System; using System.Collections; using System.Collections.Generic; using Leap.Unity.Attributes; using UnityEngine; using UnityEngine.VR; namespace Leap.Unity { public class LeapVRTemporalWarping : MonoBehaviour { public float TweenImageWarping { get { return this.tweenImageWarping; } set { this.tweenImageWarping = Mathf.Clamp01(value); } } public float TweenRotationalWarping { get { return this.tweenRotationalWarping; } set { this.tweenRotationalWarping = Mathf.Clamp01(value); } } public float TweenPositionalWarping { get { return this.tweenPositionalWarping; } set { this.tweenPositionalWarping = Mathf.Clamp01(value); } } public LeapVRTemporalWarping.SyncMode TemporalSyncMode { get { return this.syncMode; } set { this.syncMode = value; } } public float RewindAdjust { get { return (float)this.warpingAdjustment; } } public bool TryGetWarpedTransform(LeapVRTemporalWarping.WarpedAnchor anchor, out Vector3 rewoundPosition, out Quaternion rewoundRotation, long leapTime) { if (this._headTransform == null) { rewoundPosition = Vector3.one; rewoundRotation = Quaternion.identity; return false; } LeapVRTemporalWarping.TransformData transformData = this.transformAtTime(leapTime - (long)(this.warpingAdjustment * 1000)); if (this._trackingAnchor == null) { rewoundRotation = transformData.localRotation; rewoundPosition = transformData.localPosition + rewoundRotation * Vector3.forward * this.deviceInfo.focalPlaneOffset; } else { rewoundRotation = this._trackingAnchor.rotation * transformData.localRotation; rewoundPosition = this._trackingAnchor.TransformPoint(transformData.localPosition) + rewoundRotation * Vector3.forward * this.deviceInfo.focalPlaneOffset; } switch (anchor) { case LeapVRTemporalWarping.WarpedAnchor.CENTER: break; case LeapVRTemporalWarping.WarpedAnchor.LEFT: rewoundPosition += rewoundRotation * Vector3.left * this.deviceInfo.baseline * 0.5f; break; case LeapVRTemporalWarping.WarpedAnchor.RIGHT: rewoundPosition += rewoundRotation * Vector3.right * this.deviceInfo.baseline * 0.5f; break; default: throw new Exception("Unexpected Rewind Type " + anchor); } return true; } public bool TryGetWarpedTransform(LeapVRTemporalWarping.WarpedAnchor anchor, out Vector3 rewoundPosition, out Quaternion rewoundRotation) { long timestamp = this.provider.CurrentFrame.Timestamp; if (this.TryGetWarpedTransform(anchor, out rewoundPosition, out rewoundRotation, timestamp)) { return true; } rewoundPosition = Vector3.zero; rewoundRotation = Quaternion.identity; return false; } public void ManualyUpdateTemporalWarping() { if (this._trackingAnchor == null) { this.updateHistory(this._headTransform.position, this._headTransform.rotation); this.updateTemporalWarping(this._headTransform.position, this._headTransform.rotation); } else { this.updateHistory(this._trackingAnchor.InverseTransformPoint(this._headTransform.position), Quaternion.Inverse(this._trackingAnchor.rotation) * this._headTransform.rotation); } } protected void Start() { if (this.provider.IsConnected()) { this.deviceInfo = this.provider.GetDeviceInfo(); this._shouldSetLocalPosition = true; LeapVRCameraControl.OnValidCameraParams += this.onValidCameraParams; if (this.deviceInfo.type == LeapDeviceType.Invalid) { Debug.LogWarning("Invalid Leap Device -> enabled = false"); base.enabled = false; return; } } else { base.StartCoroutine(this.waitForConnection()); Controller leapController = this.provider.GetLeapController(); leapController.Device += this.OnDevice; } } private IEnumerator waitForConnection() { while (!this.provider.IsConnected()) { yield return null; } LeapVRCameraControl.OnValidCameraParams -= this.onValidCameraParams; LeapVRCameraControl.OnValidCameraParams += this.onValidCameraParams; yield break; } protected void OnDevice(object sender, DeviceEventArgs args) { this.deviceInfo = this.provider.GetDeviceInfo(); this._shouldSetLocalPosition = true; if (this.deviceInfo.type == LeapDeviceType.Invalid) { Debug.LogWarning("Invalid Leap Device -> enabled = false"); base.enabled = false; return; } LeapVRCameraControl.OnValidCameraParams -= this.onValidCameraParams; LeapVRCameraControl.OnValidCameraParams += this.onValidCameraParams; } protected void OnEnable() { if (this.deviceInfo.type != LeapDeviceType.Invalid) { LeapVRCameraControl.OnValidCameraParams -= this.onValidCameraParams; LeapVRCameraControl.OnValidCameraParams += this.onValidCameraParams; } } protected void OnDisable() { LeapVRCameraControl.OnValidCameraParams -= this.onValidCameraParams; } protected void OnDestroy() { LeapVRCameraControl.OnValidCameraParams -= this.onValidCameraParams; } protected void Update() { if (this._shouldSetLocalPosition) { base.transform.localPosition = base.transform.forward * this.deviceInfo.focalPlaneOffset; this._shouldSetLocalPosition = false; } if (Input.GetKeyDown(this.recenter) && VRSettings.enabled && VRDevice.isPresent) { InputTracking.Recenter(); } if (this.allowManualTimeAlignment && (this.unlockHold == KeyCode.None || Input.GetKey(this.unlockHold))) { if (Input.GetKeyDown(this.moreRewind)) { this.warpingAdjustment++; } if (Input.GetKeyDown(this.lessRewind)) { this.warpingAdjustment--; } } } protected void LateUpdate() { if (VRSettings.enabled) { this.updateTemporalWarping(InputTracking.GetLocalPosition(VRNode.CenterEye), InputTracking.GetLocalRotation(VRNode.CenterEye)); } } private void onValidCameraParams(LeapVRCameraControl.CameraParams cameraParams) { this._projectionMatrix = cameraParams.ProjectionMatrix; if (VRSettings.enabled) { if (this.provider != null) { this.updateHistory(InputTracking.GetLocalPosition(VRNode.CenterEye), InputTracking.GetLocalRotation(VRNode.CenterEye)); } if (this.syncMode == LeapVRTemporalWarping.SyncMode.LOW_LATENCY) { this.updateTemporalWarping(InputTracking.GetLocalPosition(VRNode.CenterEye), InputTracking.GetLocalRotation(VRNode.CenterEye)); } } } private void updateHistory(Vector3 currLocalPosition, Quaternion currLocalRotation) { long num = this.provider.GetLeapController().Now(); this._history.Add(new LeapVRTemporalWarping.TransformData { leapTime = num, localPosition = currLocalPosition, localRotation = currLocalRotation }); while (this._history.Count > 0 && 200000L < num - this._history[0].leapTime) { this._history.RemoveAt(0); } } private void updateTemporalWarping(Vector3 currLocalPosition, Quaternion currLocalRotation) { if (this._trackingAnchor == null || this.provider.GetLeapController() == null) { return; } Vector3 a = this._trackingAnchor.TransformPoint(currLocalPosition); Quaternion quaternion = this._trackingAnchor.rotation * currLocalRotation; long time = this.provider.CurrentFrame.Timestamp - (long)(this.warpingAdjustment * 1000); LeapVRTemporalWarping.TransformData transformData = this.transformAtTime(time); Vector3 b = this._trackingAnchor.TransformPoint(transformData.localPosition); Quaternion b2 = this._trackingAnchor.rotation * transformData.localRotation; Quaternion rhs = Quaternion.Slerp(quaternion, b2, this.tweenImageWarping); Quaternion q = Quaternion.Inverse(quaternion) * rhs; q = Quaternion.Euler(q.eulerAngles.x, q.eulerAngles.y, -q.eulerAngles.z); Matrix4x4 value = this._projectionMatrix * Matrix4x4.TRS(Vector3.zero, q, Vector3.one) * this._projectionMatrix.inverse; Shader.SetGlobalMatrix("_LeapGlobalWarpedOffset", value); base.transform.position = Vector3.Lerp(a, b, this.tweenPositionalWarping); base.transform.rotation = Quaternion.Slerp(quaternion, b2, this.tweenRotationalWarping); base.transform.position += base.transform.forward * this.deviceInfo.focalPlaneOffset; } private LeapVRTemporalWarping.TransformData transformAtTime(long time) { if (this._history.Count == 0) { return new LeapVRTemporalWarping.TransformData { leapTime = 0L, localPosition = Vector3.zero, localRotation = Quaternion.identity }; } if (this._history[0].leapTime >= time) { return this._history[0]; } int num = 1; while (num < this._history.Count && this._history[num].leapTime <= time) { num++; } if (num >= this._history.Count) { return this._history[this._history.Count - 1]; } return LeapVRTemporalWarping.TransformData.Lerp(this._history[num - 1], this._history[num], time); } private const long MAX_LATENCY = 200000L; [AutoFind(AutoFindLocations.All)] [SerializeField] private LeapServiceProvider provider; [Tooltip("The transform that represents the head object.")] [SerializeField] private Transform _headTransform; [Tooltip("The transform that is the anchor that tracking movement is relative to. Can be null if head motion is in world space.")] [SerializeField] private Transform _trackingAnchor; [Tooltip("Key to recenter the VR tracking space.")] [SerializeField] private KeyCode recenter = KeyCode.R; [Tooltip("Allows smooth enabling or disabling of the Image-Warping feature. Usually should match rotation warping.")] [Range(0f, 1f)] [SerializeField] private float tweenImageWarping; [Tooltip("Allows smooth enabling or disabling of the Rotational warping of Leap Space. Usually should match image warping.")] [Range(0f, 1f)] [SerializeField] private float tweenRotationalWarping; [Tooltip("Allows smooth enabling or disabling of the Positional warping of Leap Space. Usually should be disabled when using image warping.")] [Range(0f, 1f)] [SerializeField] private float tweenPositionalWarping; [Tooltip("Controls when this script synchronizes the time warp of images. Use LowLatency for AR, and SyncWithHands for VR.")] [SerializeField] private LeapVRTemporalWarping.SyncMode syncMode; [Tooltip("Allow manual adjustment of the rewind time.")] [SerializeField] private bool allowManualTimeAlignment; [Tooltip("Timestamps and other uncertanties can lead to sub-optimal alignment, this value can be tuned to get desired alignment.")] [SerializeField] private int warpingAdjustment = 60; [SerializeField] private KeyCode unlockHold = KeyCode.RightShift; [SerializeField] private KeyCode moreRewind = KeyCode.LeftArrow; [SerializeField] private KeyCode lessRewind = KeyCode.RightArrow; private LeapDeviceInfo deviceInfo; private Matrix4x4 _projectionMatrix; private List _history = new List(); private bool _shouldSetLocalPosition; public enum WarpedAnchor { CENTER, LEFT, RIGHT } public enum SyncMode { SYNC_WITH_HANDS, LOW_LATENCY } protected struct TransformData { public static LeapVRTemporalWarping.TransformData Lerp(LeapVRTemporalWarping.TransformData from, LeapVRTemporalWarping.TransformData to, long time) { if (from.leapTime == to.leapTime) { return from; } float t = (float)(time - from.leapTime) / (float)(to.leapTime - from.leapTime); return new LeapVRTemporalWarping.TransformData { leapTime = time, localPosition = Vector3.Lerp(from.localPosition, to.localPosition, t), localRotation = Quaternion.Slerp(from.localRotation, to.localRotation, t) }; } public long leapTime; public Vector3 localPosition; public Quaternion localRotation; } } }