LeapImageRetriever.cs 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440
  1. using System;
  2. using System.Collections;
  3. using Leap.Unity.Attributes;
  4. using UnityEngine;
  5. using UnityEngine.Serialization;
  6. namespace Leap.Unity
  7. {
  8. [RequireComponent(typeof(Camera))]
  9. public class LeapImageRetriever : MonoBehaviour
  10. {
  11. public LeapImageRetriever.EyeTextureData TextureData
  12. {
  13. get
  14. {
  15. return this._eyeTextureData;
  16. }
  17. }
  18. private void Start()
  19. {
  20. if (this._provider == null)
  21. {
  22. Debug.LogWarning("Cannot use LeapImageRetriever if there is no LeapProvider!");
  23. base.enabled = false;
  24. return;
  25. }
  26. this.ApplyGammaCorrectionValues();
  27. this.ApplyCameraProjectionValues(base.GetComponent<Camera>());
  28. }
  29. private void OnEnable()
  30. {
  31. Controller leapController = this._provider.GetLeapController();
  32. if (leapController != null)
  33. {
  34. this.onController(leapController);
  35. }
  36. else
  37. {
  38. base.StartCoroutine(this.waitForController());
  39. }
  40. LeapVRCameraControl.OnLeftPreRender += this.ApplyCameraProjectionValues;
  41. LeapVRCameraControl.OnRightPreRender += this.ApplyCameraProjectionValues;
  42. }
  43. private void OnDisable()
  44. {
  45. base.StopAllCoroutines();
  46. Controller leapController = this._provider.GetLeapController();
  47. if (leapController != null)
  48. {
  49. this._provider.GetLeapController().DistortionChange -= new EventHandler<DistortionEventArgs>(this.onDistortionChange);
  50. }
  51. LeapVRCameraControl.OnLeftPreRender -= this.ApplyCameraProjectionValues;
  52. LeapVRCameraControl.OnRightPreRender -= this.ApplyCameraProjectionValues;
  53. }
  54. private void OnDestroy()
  55. {
  56. base.StopAllCoroutines();
  57. Controller leapController = this._provider.GetLeapController();
  58. if (leapController != null)
  59. {
  60. this._provider.GetLeapController().DistortionChange -= new EventHandler<DistortionEventArgs>(this.onDistortionChange);
  61. }
  62. }
  63. private void OnPreRender()
  64. {
  65. if (this.imagesEnabled)
  66. {
  67. Controller leapController = this._provider.GetLeapController();
  68. long num = leapController.Now();
  69. while (!this._requestedImage.IsComplete)
  70. {
  71. if (leapController.Now() - num > this.ImageTimeout)
  72. {
  73. break;
  74. }
  75. }
  76. if (this._requestedImage.IsComplete)
  77. {
  78. if (this._eyeTextureData.CheckStale(this._requestedImage, this._requestedImage))
  79. {
  80. this._eyeTextureData.Reconstruct(this._requestedImage, this._requestedImage);
  81. }
  82. this._eyeTextureData.UpdateTextures(this._requestedImage, this._requestedImage);
  83. }
  84. else if (!this.checkingImageState)
  85. {
  86. base.StartCoroutine(this.checkImageMode());
  87. }
  88. }
  89. }
  90. private void Update()
  91. {
  92. if (this.imagesEnabled)
  93. {
  94. Frame currentFrame = this._provider.CurrentFrame;
  95. Controller leapController = this._provider.GetLeapController();
  96. this._requestedImage = leapController.RequestImages(currentFrame.Id, Image.ImageType.DEFAULT);
  97. }
  98. else if (!this.checkingImageState)
  99. {
  100. base.StartCoroutine(this.checkImageMode());
  101. }
  102. }
  103. private IEnumerator waitForController()
  104. {
  105. Controller controller = null;
  106. do
  107. {
  108. controller = this._provider.GetLeapController();
  109. yield return null;
  110. }
  111. while (controller == null);
  112. this.onController(controller);
  113. yield break;
  114. }
  115. private IEnumerator checkImageMode()
  116. {
  117. this.checkingImageState = true;
  118. yield return new WaitForSeconds(2f);
  119. this._provider.GetLeapController().Config.Get<int>("images_mode", delegate(int enabled)
  120. {
  121. this.imagesEnabled = (enabled != 0);
  122. this.checkingImageState = false;
  123. });
  124. yield break;
  125. }
  126. private void onController(Controller controller)
  127. {
  128. controller.DistortionChange += new EventHandler<DistortionEventArgs>(this.onDistortionChange);
  129. controller.Connect += delegate(object A_1, ConnectionEventArgs A_2)
  130. {
  131. this._provider.GetLeapController().Config.Get<int>("images_mode", delegate(int enabled)
  132. {
  133. this.imagesEnabled = (enabled != 0);
  134. });
  135. };
  136. if (!this.checkingImageState)
  137. {
  138. base.StartCoroutine(this.checkImageMode());
  139. }
  140. }
  141. public void ApplyGammaCorrectionValues()
  142. {
  143. float value = 1f;
  144. if (QualitySettings.activeColorSpace != ColorSpace.Linear)
  145. {
  146. value = -Mathf.Log10(Mathf.GammaToLinearSpace(0.1f));
  147. }
  148. Shader.SetGlobalFloat("_LeapGlobalColorSpaceGamma", value);
  149. Shader.SetGlobalFloat("_LeapGlobalGammaCorrectionExponent", 1f / this._gammaCorrection);
  150. }
  151. public void ApplyCameraProjectionValues(Camera camera)
  152. {
  153. Shader.SetGlobalVector("_LeapGlobalProjection", new Vector4
  154. {
  155. x = camera.projectionMatrix[0, 2],
  156. y = 0f,
  157. z = camera.projectionMatrix[0, 0],
  158. w = camera.projectionMatrix[1, 1]
  159. });
  160. }
  161. private void onDistortionChange(object sender, LeapEventArgs args)
  162. {
  163. this._eyeTextureData.MarkStale();
  164. }
  165. public const string GLOBAL_COLOR_SPACE_GAMMA_NAME = "_LeapGlobalColorSpaceGamma";
  166. public const string GLOBAL_GAMMA_CORRECTION_EXPONENT_NAME = "_LeapGlobalGammaCorrectionExponent";
  167. public const string GLOBAL_CAMERA_PROJECTION_NAME = "_LeapGlobalProjection";
  168. public const int IMAGE_WARNING_WAIT = 10;
  169. public const int LEFT_IMAGE_INDEX = 0;
  170. public const int RIGHT_IMAGE_INDEX = 1;
  171. public const float IMAGE_SETTING_POLL_RATE = 2f;
  172. [AutoFind(AutoFindLocations.All)]
  173. [SerializeField]
  174. private LeapServiceProvider _provider;
  175. [SerializeField]
  176. [FormerlySerializedAs("gammaCorrection")]
  177. private float _gammaCorrection = 1f;
  178. [MinValue(0f)]
  179. [SerializeField]
  180. protected long ImageTimeout = 9000L;
  181. private LeapImageRetriever.EyeTextureData _eyeTextureData = new LeapImageRetriever.EyeTextureData();
  182. protected Image _requestedImage = new Image();
  183. protected bool imagesEnabled = true;
  184. private bool checkingImageState;
  185. public class LeapTextureData
  186. {
  187. public Texture2D CombinedTexture
  188. {
  189. get
  190. {
  191. return this._combinedTexture;
  192. }
  193. }
  194. public bool CheckStale(Image image)
  195. {
  196. return this._combinedTexture == null || this._intermediateArray == null || (image.Width != this._combinedTexture.width || image.Height * 2 != this._combinedTexture.height) || this._combinedTexture.format != this.getTextureFormat(image);
  197. }
  198. public void Reconstruct(Image image, string globalShaderName, string pixelSizeName)
  199. {
  200. int width = image.Width;
  201. int num = image.Height * 2;
  202. TextureFormat textureFormat = this.getTextureFormat(image);
  203. if (this._combinedTexture != null)
  204. {
  205. UnityEngine.Object.DestroyImmediate(this._combinedTexture);
  206. }
  207. this._combinedTexture = new Texture2D(width, num, textureFormat, false, true);
  208. this._combinedTexture.wrapMode = TextureWrapMode.Clamp;
  209. this._combinedTexture.filterMode = FilterMode.Bilinear;
  210. this._combinedTexture.name = globalShaderName;
  211. this._combinedTexture.hideFlags = HideFlags.DontSave;
  212. this._intermediateArray = new byte[width * num * this.bytesPerPixel(textureFormat)];
  213. Shader.SetGlobalTexture(globalShaderName, this._combinedTexture);
  214. Shader.SetGlobalVector(pixelSizeName, new Vector2(1f / (float)image.Width, 1f / (float)image.Height));
  215. }
  216. public void UpdateTexture(Image image)
  217. {
  218. Array.Copy(image.Data, 0, this._intermediateArray, 0, this._intermediateArray.Length);
  219. this._combinedTexture.LoadRawTextureData(this._intermediateArray);
  220. this._combinedTexture.Apply();
  221. }
  222. private TextureFormat getTextureFormat(Image image)
  223. {
  224. switch (image.Format)
  225. {
  226. case Image.FormatType.INFRARED:
  227. return TextureFormat.Alpha8;
  228. case Image.FormatType.IBRG:
  229. case (Image.FormatType)4:
  230. return TextureFormat.RGBA32;
  231. }
  232. throw new Exception("Unexpected image format " + image.Format + "!");
  233. }
  234. private int bytesPerPixel(TextureFormat format)
  235. {
  236. switch (format)
  237. {
  238. case TextureFormat.Alpha8:
  239. return 1;
  240. default:
  241. if (format != TextureFormat.BGRA32)
  242. {
  243. throw new Exception("Unexpected texture format " + format);
  244. }
  245. break;
  246. case TextureFormat.RGBA32:
  247. case TextureFormat.ARGB32:
  248. break;
  249. }
  250. return 4;
  251. }
  252. private Texture2D _combinedTexture;
  253. private byte[] _intermediateArray;
  254. }
  255. public class LeapDistortionData
  256. {
  257. public Texture2D CombinedTexture
  258. {
  259. get
  260. {
  261. return this._combinedTexture;
  262. }
  263. }
  264. public bool CheckStale()
  265. {
  266. return this._combinedTexture == null;
  267. }
  268. public void Reconstruct(Image image, string shaderName)
  269. {
  270. int num = image.DistortionWidth / 2;
  271. int num2 = image.DistortionHeight * 2;
  272. if (this._combinedTexture != null)
  273. {
  274. UnityEngine.Object.DestroyImmediate(this._combinedTexture);
  275. }
  276. Color32[] array = new Color32[num * num2];
  277. this._combinedTexture = new Texture2D(num, num2, TextureFormat.RGBA32, false, true);
  278. this._combinedTexture.filterMode = FilterMode.Bilinear;
  279. this._combinedTexture.wrapMode = TextureWrapMode.Clamp;
  280. this._combinedTexture.hideFlags = HideFlags.DontSave;
  281. this.addDistortionData(image, array, 0);
  282. this._combinedTexture.SetPixels32(array);
  283. this._combinedTexture.Apply();
  284. Shader.SetGlobalTexture(shaderName, this._combinedTexture);
  285. }
  286. private void addDistortionData(Image image, Color32[] colors, int startIndex)
  287. {
  288. float[] distortion = image.Distortion;
  289. for (int i = 0; i < distortion.Length; i += 2)
  290. {
  291. byte r;
  292. byte g;
  293. this.encodeFloat(distortion[i], out r, out g);
  294. byte b;
  295. byte a;
  296. this.encodeFloat(distortion[i + 1], out b, out a);
  297. colors[i / 2 + startIndex] = new Color32(r, g, b, a);
  298. }
  299. }
  300. private void encodeFloat(float value, out byte byte0, out byte byte1)
  301. {
  302. value = (value + 0.6f) / 2.3f;
  303. float num = value;
  304. float num2 = value * 255f;
  305. num -= (float)((int)num);
  306. num2 -= (float)((int)num2);
  307. num -= 0.003921569f * num2;
  308. byte0 = (byte)(num * 256f);
  309. byte1 = (byte)(num2 * 256f);
  310. }
  311. private Texture2D _combinedTexture;
  312. }
  313. public class EyeTextureData
  314. {
  315. public EyeTextureData()
  316. {
  317. this.BrightTexture = new LeapImageRetriever.LeapTextureData();
  318. this.RawTexture = new LeapImageRetriever.LeapTextureData();
  319. this.Distortion = new LeapImageRetriever.LeapDistortionData();
  320. }
  321. public static void ResetGlobalShaderValues()
  322. {
  323. Texture2D texture2D = new Texture2D(1, 1, TextureFormat.ARGB32, false, false);
  324. texture2D.name = "EmptyTexture";
  325. texture2D.hideFlags = HideFlags.DontSave;
  326. texture2D.SetPixel(0, 0, new Color(0f, 0f, 0f, 0f));
  327. Shader.SetGlobalTexture("_LeapGlobalBrightnessTexture", texture2D);
  328. Shader.SetGlobalTexture("_LeapGlobalRawTexture", texture2D);
  329. Shader.SetGlobalTexture("_LeapGlobalDistortion", texture2D);
  330. }
  331. public bool CheckStale(Image bright, Image raw)
  332. {
  333. return this.BrightTexture.CheckStale(bright) || this.RawTexture.CheckStale(raw) || this.Distortion.CheckStale() || this._isStale;
  334. }
  335. public void MarkStale()
  336. {
  337. this._isStale = true;
  338. }
  339. public void Reconstruct(Image bright, Image raw)
  340. {
  341. this.BrightTexture.Reconstruct(bright, "_LeapGlobalBrightnessTexture", "_LeapGlobalBrightnessPixelSize");
  342. this.RawTexture.Reconstruct(raw, "_LeapGlobalRawTexture", "_LeapGlobalRawPixelSize");
  343. this.Distortion.Reconstruct(raw, "_LeapGlobalDistortion");
  344. Image.FormatType format = raw.Format;
  345. if (format != Image.FormatType.INFRARED)
  346. {
  347. if (format != (Image.FormatType)4)
  348. {
  349. Debug.LogWarning("Unexpected format type " + raw.Format);
  350. }
  351. else
  352. {
  353. Shader.DisableKeyword("LEAP_FORMAT_IR");
  354. Shader.EnableKeyword("LEAP_FORMAT_RGB");
  355. }
  356. }
  357. else
  358. {
  359. Shader.DisableKeyword("LEAP_FORMAT_RGB");
  360. Shader.EnableKeyword("LEAP_FORMAT_IR");
  361. }
  362. this._isStale = false;
  363. }
  364. public void UpdateTextures(Image bright, Image raw)
  365. {
  366. this.BrightTexture.UpdateTexture(bright);
  367. this.RawTexture.UpdateTexture(raw);
  368. }
  369. private const string IR_SHADER_VARIANT_NAME = "LEAP_FORMAT_IR";
  370. private const string RGB_SHADER_VARIANT_NAME = "LEAP_FORMAT_RGB";
  371. private const string GLOBAL_BRIGHT_TEXTURE_NAME = "_LeapGlobalBrightnessTexture";
  372. private const string GLOBAL_RAW_TEXTURE_NAME = "_LeapGlobalRawTexture";
  373. private const string GLOBAL_DISTORTION_TEXTURE_NAME = "_LeapGlobalDistortion";
  374. private const string GLOBAL_BRIGHT_PIXEL_SIZE_NAME = "_LeapGlobalBrightnessPixelSize";
  375. private const string GLOBAL_RAW_PIXEL_SIZE_NAME = "_LeapGlobalRawPixelSize";
  376. public readonly LeapImageRetriever.LeapTextureData BrightTexture;
  377. public readonly LeapImageRetriever.LeapTextureData RawTexture;
  378. public readonly LeapImageRetriever.LeapDistortionData Distortion;
  379. private bool _isStale;
  380. }
  381. }
  382. }