LeapVRTemporalWarping.cs 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405
  1. using System;
  2. using System.Collections;
  3. using System.Collections.Generic;
  4. using Leap.Unity.Attributes;
  5. using UnityEngine;
  6. using UnityEngine.VR;
  7. namespace Leap.Unity
  8. {
  9. public class LeapVRTemporalWarping : MonoBehaviour
  10. {
  11. public float TweenImageWarping
  12. {
  13. get
  14. {
  15. return this.tweenImageWarping;
  16. }
  17. set
  18. {
  19. this.tweenImageWarping = Mathf.Clamp01(value);
  20. }
  21. }
  22. public float TweenRotationalWarping
  23. {
  24. get
  25. {
  26. return this.tweenRotationalWarping;
  27. }
  28. set
  29. {
  30. this.tweenRotationalWarping = Mathf.Clamp01(value);
  31. }
  32. }
  33. public float TweenPositionalWarping
  34. {
  35. get
  36. {
  37. return this.tweenPositionalWarping;
  38. }
  39. set
  40. {
  41. this.tweenPositionalWarping = Mathf.Clamp01(value);
  42. }
  43. }
  44. public LeapVRTemporalWarping.SyncMode TemporalSyncMode
  45. {
  46. get
  47. {
  48. return this.syncMode;
  49. }
  50. set
  51. {
  52. this.syncMode = value;
  53. }
  54. }
  55. public float RewindAdjust
  56. {
  57. get
  58. {
  59. return (float)this.warpingAdjustment;
  60. }
  61. }
  62. public bool TryGetWarpedTransform(LeapVRTemporalWarping.WarpedAnchor anchor, out Vector3 rewoundPosition, out Quaternion rewoundRotation, long leapTime)
  63. {
  64. if (this._headTransform == null)
  65. {
  66. rewoundPosition = Vector3.one;
  67. rewoundRotation = Quaternion.identity;
  68. return false;
  69. }
  70. LeapVRTemporalWarping.TransformData transformData = this.transformAtTime(leapTime - (long)(this.warpingAdjustment * 1000));
  71. if (this._trackingAnchor == null)
  72. {
  73. rewoundRotation = transformData.localRotation;
  74. rewoundPosition = transformData.localPosition + rewoundRotation * Vector3.forward * this.deviceInfo.focalPlaneOffset;
  75. }
  76. else
  77. {
  78. rewoundRotation = this._trackingAnchor.rotation * transformData.localRotation;
  79. rewoundPosition = this._trackingAnchor.TransformPoint(transformData.localPosition) + rewoundRotation * Vector3.forward * this.deviceInfo.focalPlaneOffset;
  80. }
  81. switch (anchor)
  82. {
  83. case LeapVRTemporalWarping.WarpedAnchor.CENTER:
  84. break;
  85. case LeapVRTemporalWarping.WarpedAnchor.LEFT:
  86. rewoundPosition += rewoundRotation * Vector3.left * this.deviceInfo.baseline * 0.5f;
  87. break;
  88. case LeapVRTemporalWarping.WarpedAnchor.RIGHT:
  89. rewoundPosition += rewoundRotation * Vector3.right * this.deviceInfo.baseline * 0.5f;
  90. break;
  91. default:
  92. throw new Exception("Unexpected Rewind Type " + anchor);
  93. }
  94. return true;
  95. }
  96. public bool TryGetWarpedTransform(LeapVRTemporalWarping.WarpedAnchor anchor, out Vector3 rewoundPosition, out Quaternion rewoundRotation)
  97. {
  98. long timestamp = this.provider.CurrentFrame.Timestamp;
  99. if (this.TryGetWarpedTransform(anchor, out rewoundPosition, out rewoundRotation, timestamp))
  100. {
  101. return true;
  102. }
  103. rewoundPosition = Vector3.zero;
  104. rewoundRotation = Quaternion.identity;
  105. return false;
  106. }
  107. public void ManualyUpdateTemporalWarping()
  108. {
  109. if (this._trackingAnchor == null)
  110. {
  111. this.updateHistory(this._headTransform.position, this._headTransform.rotation);
  112. this.updateTemporalWarping(this._headTransform.position, this._headTransform.rotation);
  113. }
  114. else
  115. {
  116. this.updateHistory(this._trackingAnchor.InverseTransformPoint(this._headTransform.position), Quaternion.Inverse(this._trackingAnchor.rotation) * this._headTransform.rotation);
  117. }
  118. }
  119. protected void Start()
  120. {
  121. if (this.provider.IsConnected())
  122. {
  123. this.deviceInfo = this.provider.GetDeviceInfo();
  124. this._shouldSetLocalPosition = true;
  125. LeapVRCameraControl.OnValidCameraParams += this.onValidCameraParams;
  126. if (this.deviceInfo.type == LeapDeviceType.Invalid)
  127. {
  128. Debug.LogWarning("Invalid Leap Device -> enabled = false");
  129. base.enabled = false;
  130. return;
  131. }
  132. }
  133. else
  134. {
  135. base.StartCoroutine(this.waitForConnection());
  136. Controller leapController = this.provider.GetLeapController();
  137. leapController.Device += this.OnDevice;
  138. }
  139. }
  140. private IEnumerator waitForConnection()
  141. {
  142. while (!this.provider.IsConnected())
  143. {
  144. yield return null;
  145. }
  146. LeapVRCameraControl.OnValidCameraParams -= this.onValidCameraParams;
  147. LeapVRCameraControl.OnValidCameraParams += this.onValidCameraParams;
  148. yield break;
  149. }
  150. protected void OnDevice(object sender, DeviceEventArgs args)
  151. {
  152. this.deviceInfo = this.provider.GetDeviceInfo();
  153. this._shouldSetLocalPosition = true;
  154. if (this.deviceInfo.type == LeapDeviceType.Invalid)
  155. {
  156. Debug.LogWarning("Invalid Leap Device -> enabled = false");
  157. base.enabled = false;
  158. return;
  159. }
  160. LeapVRCameraControl.OnValidCameraParams -= this.onValidCameraParams;
  161. LeapVRCameraControl.OnValidCameraParams += this.onValidCameraParams;
  162. }
  163. protected void OnEnable()
  164. {
  165. if (this.deviceInfo.type != LeapDeviceType.Invalid)
  166. {
  167. LeapVRCameraControl.OnValidCameraParams -= this.onValidCameraParams;
  168. LeapVRCameraControl.OnValidCameraParams += this.onValidCameraParams;
  169. }
  170. }
  171. protected void OnDisable()
  172. {
  173. LeapVRCameraControl.OnValidCameraParams -= this.onValidCameraParams;
  174. }
  175. protected void OnDestroy()
  176. {
  177. LeapVRCameraControl.OnValidCameraParams -= this.onValidCameraParams;
  178. }
  179. protected void Update()
  180. {
  181. if (this._shouldSetLocalPosition)
  182. {
  183. base.transform.localPosition = base.transform.forward * this.deviceInfo.focalPlaneOffset;
  184. this._shouldSetLocalPosition = false;
  185. }
  186. if (Input.GetKeyDown(this.recenter) && VRSettings.enabled && VRDevice.isPresent)
  187. {
  188. InputTracking.Recenter();
  189. }
  190. if (this.allowManualTimeAlignment && (this.unlockHold == KeyCode.None || Input.GetKey(this.unlockHold)))
  191. {
  192. if (Input.GetKeyDown(this.moreRewind))
  193. {
  194. this.warpingAdjustment++;
  195. }
  196. if (Input.GetKeyDown(this.lessRewind))
  197. {
  198. this.warpingAdjustment--;
  199. }
  200. }
  201. }
  202. protected void LateUpdate()
  203. {
  204. if (VRSettings.enabled)
  205. {
  206. this.updateTemporalWarping(InputTracking.GetLocalPosition(VRNode.CenterEye), InputTracking.GetLocalRotation(VRNode.CenterEye));
  207. }
  208. }
  209. private void onValidCameraParams(LeapVRCameraControl.CameraParams cameraParams)
  210. {
  211. this._projectionMatrix = cameraParams.ProjectionMatrix;
  212. if (VRSettings.enabled)
  213. {
  214. if (this.provider != null)
  215. {
  216. this.updateHistory(InputTracking.GetLocalPosition(VRNode.CenterEye), InputTracking.GetLocalRotation(VRNode.CenterEye));
  217. }
  218. if (this.syncMode == LeapVRTemporalWarping.SyncMode.LOW_LATENCY)
  219. {
  220. this.updateTemporalWarping(InputTracking.GetLocalPosition(VRNode.CenterEye), InputTracking.GetLocalRotation(VRNode.CenterEye));
  221. }
  222. }
  223. }
  224. private void updateHistory(Vector3 currLocalPosition, Quaternion currLocalRotation)
  225. {
  226. long num = this.provider.GetLeapController().Now();
  227. this._history.Add(new LeapVRTemporalWarping.TransformData
  228. {
  229. leapTime = num,
  230. localPosition = currLocalPosition,
  231. localRotation = currLocalRotation
  232. });
  233. while (this._history.Count > 0 && 200000L < num - this._history[0].leapTime)
  234. {
  235. this._history.RemoveAt(0);
  236. }
  237. }
  238. private void updateTemporalWarping(Vector3 currLocalPosition, Quaternion currLocalRotation)
  239. {
  240. if (this._trackingAnchor == null || this.provider.GetLeapController() == null)
  241. {
  242. return;
  243. }
  244. Vector3 a = this._trackingAnchor.TransformPoint(currLocalPosition);
  245. Quaternion quaternion = this._trackingAnchor.rotation * currLocalRotation;
  246. long time = this.provider.CurrentFrame.Timestamp - (long)(this.warpingAdjustment * 1000);
  247. LeapVRTemporalWarping.TransformData transformData = this.transformAtTime(time);
  248. Vector3 b = this._trackingAnchor.TransformPoint(transformData.localPosition);
  249. Quaternion b2 = this._trackingAnchor.rotation * transformData.localRotation;
  250. Quaternion rhs = Quaternion.Slerp(quaternion, b2, this.tweenImageWarping);
  251. Quaternion q = Quaternion.Inverse(quaternion) * rhs;
  252. q = Quaternion.Euler(q.eulerAngles.x, q.eulerAngles.y, -q.eulerAngles.z);
  253. Matrix4x4 value = this._projectionMatrix * Matrix4x4.TRS(Vector3.zero, q, Vector3.one) * this._projectionMatrix.inverse;
  254. Shader.SetGlobalMatrix("_LeapGlobalWarpedOffset", value);
  255. base.transform.position = Vector3.Lerp(a, b, this.tweenPositionalWarping);
  256. base.transform.rotation = Quaternion.Slerp(quaternion, b2, this.tweenRotationalWarping);
  257. base.transform.position += base.transform.forward * this.deviceInfo.focalPlaneOffset;
  258. }
  259. private LeapVRTemporalWarping.TransformData transformAtTime(long time)
  260. {
  261. if (this._history.Count == 0)
  262. {
  263. return new LeapVRTemporalWarping.TransformData
  264. {
  265. leapTime = 0L,
  266. localPosition = Vector3.zero,
  267. localRotation = Quaternion.identity
  268. };
  269. }
  270. if (this._history[0].leapTime >= time)
  271. {
  272. return this._history[0];
  273. }
  274. int num = 1;
  275. while (num < this._history.Count && this._history[num].leapTime <= time)
  276. {
  277. num++;
  278. }
  279. if (num >= this._history.Count)
  280. {
  281. return this._history[this._history.Count - 1];
  282. }
  283. return LeapVRTemporalWarping.TransformData.Lerp(this._history[num - 1], this._history[num], time);
  284. }
  285. private const long MAX_LATENCY = 200000L;
  286. [AutoFind(AutoFindLocations.All)]
  287. [SerializeField]
  288. private LeapServiceProvider provider;
  289. [Tooltip("The transform that represents the head object.")]
  290. [SerializeField]
  291. private Transform _headTransform;
  292. [Tooltip("The transform that is the anchor that tracking movement is relative to. Can be null if head motion is in world space.")]
  293. [SerializeField]
  294. private Transform _trackingAnchor;
  295. [Tooltip("Key to recenter the VR tracking space.")]
  296. [SerializeField]
  297. private KeyCode recenter = KeyCode.R;
  298. [Tooltip("Allows smooth enabling or disabling of the Image-Warping feature. Usually should match rotation warping.")]
  299. [Range(0f, 1f)]
  300. [SerializeField]
  301. private float tweenImageWarping;
  302. [Tooltip("Allows smooth enabling or disabling of the Rotational warping of Leap Space. Usually should match image warping.")]
  303. [Range(0f, 1f)]
  304. [SerializeField]
  305. private float tweenRotationalWarping;
  306. [Tooltip("Allows smooth enabling or disabling of the Positional warping of Leap Space. Usually should be disabled when using image warping.")]
  307. [Range(0f, 1f)]
  308. [SerializeField]
  309. private float tweenPositionalWarping;
  310. [Tooltip("Controls when this script synchronizes the time warp of images. Use LowLatency for AR, and SyncWithHands for VR.")]
  311. [SerializeField]
  312. private LeapVRTemporalWarping.SyncMode syncMode;
  313. [Tooltip("Allow manual adjustment of the rewind time.")]
  314. [SerializeField]
  315. private bool allowManualTimeAlignment;
  316. [Tooltip("Timestamps and other uncertanties can lead to sub-optimal alignment, this value can be tuned to get desired alignment.")]
  317. [SerializeField]
  318. private int warpingAdjustment = 60;
  319. [SerializeField]
  320. private KeyCode unlockHold = KeyCode.RightShift;
  321. [SerializeField]
  322. private KeyCode moreRewind = KeyCode.LeftArrow;
  323. [SerializeField]
  324. private KeyCode lessRewind = KeyCode.RightArrow;
  325. private LeapDeviceInfo deviceInfo;
  326. private Matrix4x4 _projectionMatrix;
  327. private List<LeapVRTemporalWarping.TransformData> _history = new List<LeapVRTemporalWarping.TransformData>();
  328. private bool _shouldSetLocalPosition;
  329. public enum WarpedAnchor
  330. {
  331. CENTER,
  332. LEFT,
  333. RIGHT
  334. }
  335. public enum SyncMode
  336. {
  337. SYNC_WITH_HANDS,
  338. LOW_LATENCY
  339. }
  340. protected struct TransformData
  341. {
  342. public static LeapVRTemporalWarping.TransformData Lerp(LeapVRTemporalWarping.TransformData from, LeapVRTemporalWarping.TransformData to, long time)
  343. {
  344. if (from.leapTime == to.leapTime)
  345. {
  346. return from;
  347. }
  348. float t = (float)(time - from.leapTime) / (float)(to.leapTime - from.leapTime);
  349. return new LeapVRTemporalWarping.TransformData
  350. {
  351. leapTime = time,
  352. localPosition = Vector3.Lerp(from.localPosition, to.localPosition, t),
  353. localRotation = Quaternion.Slerp(from.localRotation, to.localRotation, t)
  354. };
  355. }
  356. public long leapTime;
  357. public Vector3 localPosition;
  358. public Quaternion localRotation;
  359. }
  360. }
  361. }