Resampler.cs 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482
  1. using System;
  2. using System.Collections.Generic;
  3. using UnityEngine;
  4. using UnityEngine.Events;
  5. namespace RenderHeads.Media.AVProVideo
  6. {
  7. public class Resampler
  8. {
  9. public Resampler(MediaPlayer player, string name, int bufferSize = 2, Resampler.ResampleMode resampleMode = Resampler.ResampleMode.LINEAR)
  10. {
  11. this._bufferSize = Mathf.Max(2, bufferSize);
  12. if (player.Info != null)
  13. {
  14. this._elapsedTimeSinceBase = (float)this._bufferSize / player.Info.GetVideoFrameRate();
  15. }
  16. player.Events.AddListener(new UnityAction<MediaPlayer, MediaPlayerEvent.EventType, ErrorCode>(this.OnVideoEvent));
  17. this._mediaPlayer = player;
  18. Shader shader = Shader.Find("AVProVideo/BlendFrames");
  19. if (shader != null)
  20. {
  21. this._blendMat = new Material(shader);
  22. this._propT = Shader.PropertyToID("_t");
  23. this._propAfterTex = Shader.PropertyToID("_AfterTex");
  24. }
  25. else
  26. {
  27. Debug.LogError("[AVProVideo] Failed to find BlendFrames shader");
  28. }
  29. this._resampleMode = resampleMode;
  30. this._name = name;
  31. Debug.Log("[AVProVideo] Resampler " + this._name + " started");
  32. }
  33. public int DroppedFrames
  34. {
  35. get
  36. {
  37. return this._droppedFrames;
  38. }
  39. }
  40. public int FrameDisplayedTimer
  41. {
  42. get
  43. {
  44. return this._frameDisplayedTimer;
  45. }
  46. }
  47. public long BaseTimestamp
  48. {
  49. get
  50. {
  51. return this._baseTimestamp;
  52. }
  53. set
  54. {
  55. this._baseTimestamp = value;
  56. }
  57. }
  58. public float ElapsedTimeSinceBase
  59. {
  60. get
  61. {
  62. return this._elapsedTimeSinceBase;
  63. }
  64. set
  65. {
  66. this._elapsedTimeSinceBase = value;
  67. }
  68. }
  69. public float LastT { get; private set; }
  70. public long TextureTimeStamp { get; private set; }
  71. public void OnVideoEvent(MediaPlayer mp, MediaPlayerEvent.EventType et, ErrorCode errorCode)
  72. {
  73. if (et != MediaPlayerEvent.EventType.MetaDataReady)
  74. {
  75. if (et == MediaPlayerEvent.EventType.Closing)
  76. {
  77. this.Reset();
  78. }
  79. }
  80. else
  81. {
  82. this._elapsedTimeSinceBase = (float)this._bufferSize / this._mediaPlayer.Info.GetVideoFrameRate();
  83. }
  84. }
  85. public Texture[] OutputTexture
  86. {
  87. get
  88. {
  89. return this._outputTexture;
  90. }
  91. }
  92. public void Reset()
  93. {
  94. this._lastTimeStamp = -1L;
  95. this._baseTimestamp = 0L;
  96. this.InvalidateBuffer();
  97. }
  98. public void Release()
  99. {
  100. this.ReleaseRenderTextures();
  101. if (this._blendMat != null)
  102. {
  103. UnityEngine.Object.Destroy(this._blendMat);
  104. }
  105. }
  106. private void ReleaseRenderTextures()
  107. {
  108. for (int i = 0; i < this._buffer.Count; i++)
  109. {
  110. for (int j = 0; j < this._buffer[i].Length; j++)
  111. {
  112. if (this._buffer[i][j].texture != null)
  113. {
  114. RenderTexture.ReleaseTemporary(this._buffer[i][j].texture);
  115. this._buffer[i][j].texture = null;
  116. }
  117. }
  118. if (this._outputTexture != null && this._outputTexture[i] != null)
  119. {
  120. RenderTexture.ReleaseTemporary(this._outputTexture[i]);
  121. }
  122. }
  123. this._outputTexture = null;
  124. }
  125. private void ConstructRenderTextures()
  126. {
  127. this.ReleaseRenderTextures();
  128. this._buffer.Clear();
  129. this._outputTexture = new RenderTexture[this._mediaPlayer.TextureProducer.GetTextureCount()];
  130. for (int i = 0; i < this._mediaPlayer.TextureProducer.GetTextureCount(); i++)
  131. {
  132. Texture texture = this._mediaPlayer.TextureProducer.GetTexture(i);
  133. this._buffer.Add(new Resampler.TimestampedRenderTexture[this._bufferSize]);
  134. for (int j = 0; j < this._bufferSize; j++)
  135. {
  136. this._buffer[i][j] = new Resampler.TimestampedRenderTexture();
  137. }
  138. for (int k = 0; k < this._buffer[i].Length; k++)
  139. {
  140. this._buffer[i][k].texture = RenderTexture.GetTemporary(texture.width, texture.height, 0);
  141. this._buffer[i][k].timestamp = 0L;
  142. this._buffer[i][k].used = false;
  143. }
  144. this._outputTexture[i] = RenderTexture.GetTemporary(texture.width, texture.height, 0);
  145. }
  146. }
  147. private bool CheckRenderTexturesValid()
  148. {
  149. for (int i = 0; i < this._mediaPlayer.TextureProducer.GetTextureCount(); i++)
  150. {
  151. Texture texture = this._mediaPlayer.TextureProducer.GetTexture(i);
  152. for (int j = 0; j < this._buffer.Count; j++)
  153. {
  154. if (this._buffer[i][j].texture == null || this._buffer[i][j].texture.width != texture.width || this._buffer[i][j].texture.height != texture.height)
  155. {
  156. return false;
  157. }
  158. }
  159. if (this._outputTexture == null || this._outputTexture[i] == null || this._outputTexture[i].width != texture.width || this._outputTexture[i].height != texture.height)
  160. {
  161. return false;
  162. }
  163. }
  164. return true;
  165. }
  166. private int FindBeforeFrameIndex(int frameIdx)
  167. {
  168. if (frameIdx >= this._buffer.Count)
  169. {
  170. return -1;
  171. }
  172. int num = -1;
  173. float num2 = float.MaxValue;
  174. int num3 = -1;
  175. float num4 = float.MaxValue;
  176. for (int i = 0; i < this._buffer[frameIdx].Length; i++)
  177. {
  178. if (this._buffer[frameIdx][i].used)
  179. {
  180. float num5 = (float)(this._buffer[frameIdx][i].timestamp - this._baseTimestamp) / 10000000f;
  181. if (num5 < num4)
  182. {
  183. num3 = i;
  184. num4 = num5;
  185. }
  186. float num6 = this._elapsedTimeSinceBase - num5;
  187. if (num6 >= 0f && num6 < num2)
  188. {
  189. num2 = num6;
  190. num = i;
  191. }
  192. }
  193. }
  194. if (num >= 0)
  195. {
  196. return num;
  197. }
  198. if (num3 < 0)
  199. {
  200. return -1;
  201. }
  202. return num3;
  203. }
  204. private int findClosestFrame(int frameIdx)
  205. {
  206. if (frameIdx >= this._buffer.Count)
  207. {
  208. return -1;
  209. }
  210. int result = -1;
  211. float num = float.MaxValue;
  212. for (int i = 0; i < this._buffer[frameIdx].Length; i++)
  213. {
  214. if (this._buffer[frameIdx][i].used)
  215. {
  216. float num2 = (float)(this._buffer[frameIdx][i].timestamp - this._baseTimestamp) / 10000000f;
  217. float num3 = Mathf.Abs(this._elapsedTimeSinceBase - num2);
  218. if (num3 < num)
  219. {
  220. result = i;
  221. num = num3;
  222. }
  223. }
  224. }
  225. return result;
  226. }
  227. private void PointUpdate()
  228. {
  229. for (int i = 0; i < this._buffer.Count; i++)
  230. {
  231. int num = this.findClosestFrame(i);
  232. if (num >= 0)
  233. {
  234. this._outputTexture[i].DiscardContents();
  235. Graphics.Blit(this._buffer[i][num].texture, this._outputTexture[i]);
  236. this._currentDisplayedTimestamp = this._buffer[i][num].timestamp;
  237. }
  238. }
  239. }
  240. private void SampleFrame(int frameIdx, int bufferIdx)
  241. {
  242. this._outputTexture[bufferIdx].DiscardContents();
  243. Graphics.Blit(this._buffer[bufferIdx][frameIdx].texture, this._outputTexture[bufferIdx]);
  244. this.TextureTimeStamp = this._buffer[bufferIdx][frameIdx].timestamp;
  245. this._currentDisplayedTimestamp = this._buffer[bufferIdx][frameIdx].timestamp;
  246. }
  247. private void SampleFrames(int bufferIdx, int frameIdx1, int frameIdx2, float t)
  248. {
  249. this._blendMat.SetFloat(this._propT, t);
  250. this._blendMat.SetTexture(this._propAfterTex, this._buffer[bufferIdx][frameIdx2].texture);
  251. this._outputTexture[bufferIdx].DiscardContents();
  252. Graphics.Blit(this._buffer[bufferIdx][frameIdx1].texture, this._outputTexture[bufferIdx], this._blendMat);
  253. this.TextureTimeStamp = (long)Mathf.Lerp((float)this._buffer[bufferIdx][frameIdx1].timestamp, (float)this._buffer[bufferIdx][frameIdx2].timestamp, t);
  254. this._currentDisplayedTimestamp = this._buffer[bufferIdx][frameIdx1].timestamp;
  255. }
  256. private void LinearUpdate()
  257. {
  258. for (int i = 0; i < this._buffer.Count; i++)
  259. {
  260. int num = this.FindBeforeFrameIndex(i);
  261. if (num >= 0)
  262. {
  263. float num2 = (float)(this._buffer[i][num].timestamp - this._baseTimestamp) / 10000000f;
  264. if (num2 > this._elapsedTimeSinceBase)
  265. {
  266. this.SampleFrame(num, i);
  267. this.LastT = -1f;
  268. }
  269. else
  270. {
  271. int num3 = (num + 1) % this._buffer[i].Length;
  272. float num4 = (float)(this._buffer[i][num3].timestamp - this._baseTimestamp) / 10000000f;
  273. if (num4 < num2)
  274. {
  275. this.SampleFrame(num, i);
  276. this.LastT = 2f;
  277. }
  278. else
  279. {
  280. float num5 = num4 - num2;
  281. float num6 = (this._elapsedTimeSinceBase - num2) / num5;
  282. this.SampleFrames(i, num, num3, num6);
  283. this.LastT = num6;
  284. }
  285. }
  286. }
  287. }
  288. }
  289. private void InvalidateBuffer()
  290. {
  291. this._elapsedTimeSinceBase = (float)(this._bufferSize / 2) / this._mediaPlayer.Info.GetVideoFrameRate();
  292. for (int i = 0; i < this._buffer.Count; i++)
  293. {
  294. for (int j = 0; j < this._buffer[i].Length; j++)
  295. {
  296. this._buffer[i][j].used = false;
  297. }
  298. }
  299. this._start = (this._end = 0);
  300. }
  301. public void Update()
  302. {
  303. if (this._mediaPlayer.TextureProducer == null)
  304. {
  305. return;
  306. }
  307. if (this._mediaPlayer.TextureProducer == null || this._mediaPlayer.TextureProducer.GetTexture(0) == null)
  308. {
  309. return;
  310. }
  311. if (!this.CheckRenderTexturesValid())
  312. {
  313. this.ConstructRenderTextures();
  314. }
  315. long textureTimeStamp = this._mediaPlayer.TextureProducer.GetTextureTimeStamp();
  316. if (textureTimeStamp != this._lastTimeStamp)
  317. {
  318. float num = Mathf.Abs((float)(textureTimeStamp - this._lastTimeStamp));
  319. float num2 = 10000000f * (1f / this._mediaPlayer.Info.GetVideoFrameRate());
  320. if (num > num2 * 1.1f && num < num2 * 3.1f)
  321. {
  322. this._droppedFrames += (int)((double)((num - num2) / num2) + 0.5);
  323. }
  324. this._lastTimeStamp = textureTimeStamp;
  325. }
  326. long textureTimeStamp2 = this._mediaPlayer.TextureProducer.GetTextureTimeStamp();
  327. bool flag = !this._mediaPlayer.Control.IsSeeking();
  328. if (this._start != this._end || this._buffer[0][this._end].used)
  329. {
  330. int num3 = (this._end + this._buffer[0].Length - 1) % this._buffer[0].Length;
  331. if (textureTimeStamp2 == this._buffer[0][num3].timestamp)
  332. {
  333. flag = false;
  334. }
  335. }
  336. if (flag)
  337. {
  338. if (this._start == this._end && !this._buffer[0][this._end].used)
  339. {
  340. this._baseTimestamp = textureTimeStamp2;
  341. }
  342. if (this._end == this._start && this._buffer[0][this._end].used)
  343. {
  344. this._start = (this._start + 1) % this._buffer[0].Length;
  345. }
  346. for (int i = 0; i < this._mediaPlayer.TextureProducer.GetTextureCount(); i++)
  347. {
  348. Texture texture = this._mediaPlayer.TextureProducer.GetTexture(i);
  349. this._buffer[i][this._end].texture.DiscardContents();
  350. Graphics.Blit(texture, this._buffer[i][this._end].texture);
  351. this._buffer[i][this._end].timestamp = textureTimeStamp2;
  352. this._buffer[i][this._end].used = true;
  353. }
  354. this._end = (this._end + 1) % this._buffer[0].Length;
  355. }
  356. bool flag2 = this._start != this._end || !this._buffer[0][this._end].used;
  357. if (flag2)
  358. {
  359. for (int j = 0; j < this._buffer.Count; j++)
  360. {
  361. this._outputTexture[j].DiscardContents();
  362. Graphics.Blit(this._buffer[j][this._start].texture, this._outputTexture[j]);
  363. this._currentDisplayedTimestamp = this._buffer[j][this._start].timestamp;
  364. }
  365. }
  366. if (this._mediaPlayer.Control.IsPaused())
  367. {
  368. this.InvalidateBuffer();
  369. }
  370. if (flag2)
  371. {
  372. return;
  373. }
  374. if (this._mediaPlayer.Control.IsPlaying() && !this._mediaPlayer.Control.IsFinished())
  375. {
  376. long num4 = this._buffer[0][(this._start + this._bufferSize / 2) % this._bufferSize].timestamp - this._baseTimestamp;
  377. double num5 = (double)Mathf.Abs((float)((double)this._elapsedTimeSinceBase * 10000000.0) - (float)num4);
  378. double num6 = (double)((float)(this._buffer[0].Length / 2) / this._mediaPlayer.Info.GetVideoFrameRate() * 10000000f);
  379. if (num5 > num6)
  380. {
  381. this._elapsedTimeSinceBase = (float)num4 / 10000000f;
  382. }
  383. if (this._resampleMode == Resampler.ResampleMode.POINT)
  384. {
  385. this.PointUpdate();
  386. }
  387. else if (this._resampleMode == Resampler.ResampleMode.LINEAR)
  388. {
  389. this.LinearUpdate();
  390. }
  391. this._elapsedTimeSinceBase += Time.unscaledDeltaTime;
  392. }
  393. }
  394. public void UpdateTimestamp()
  395. {
  396. if (this._lastDisplayedTimestamp != this._currentDisplayedTimestamp)
  397. {
  398. this._lastDisplayedTimestamp = this._currentDisplayedTimestamp;
  399. this._frameDisplayedTimer = 0;
  400. }
  401. this._frameDisplayedTimer++;
  402. }
  403. private List<Resampler.TimestampedRenderTexture[]> _buffer = new List<Resampler.TimestampedRenderTexture[]>();
  404. private MediaPlayer _mediaPlayer;
  405. private RenderTexture[] _outputTexture;
  406. private int _start;
  407. private int _end;
  408. private int _bufferSize;
  409. private long _baseTimestamp;
  410. private float _elapsedTimeSinceBase;
  411. private Material _blendMat;
  412. private Resampler.ResampleMode _resampleMode;
  413. private string _name = string.Empty;
  414. private long _lastTimeStamp = -1L;
  415. private int _droppedFrames;
  416. private long _lastDisplayedTimestamp;
  417. private int _frameDisplayedTimer;
  418. private long _currentDisplayedTimestamp;
  419. private const string ShaderPropT = "_t";
  420. private const string ShaderPropAftertex = "_AfterTex";
  421. private int _propAfterTex;
  422. private int _propT;
  423. private class TimestampedRenderTexture
  424. {
  425. public RenderTexture texture;
  426. public long timestamp;
  427. public bool used;
  428. }
  429. public enum ResampleMode
  430. {
  431. POINT,
  432. LINEAR
  433. }
  434. }
  435. }