#if UNITY_EDITOR using UnityEditor; using UnityEditorInternal; #endif using UnityEngine; using UnityEngine.Playables; using UnityEngine.Timeline; namespace FT.Timeline { public class LeviathanVideoBehaviour : PlayableBehaviour { public TextAsset videoOverride; public bool syncDecodeFirstFrame = true; public bool fastDecode = true; /// 使用多线程解码并由 Timeline 对齐帧(WebGL 等平台会降级);编辑器预览在勾选本项时同样走多线程轨。 public bool useMultithreadedDecode = true; /// 注入;多线程时 CatchUp 与 Lockstep 二选一。 public LeviathanVideoTimelineMtDriveMode timelineMultithreadedDriveMode = LeviathanVideoTimelineMtDriveMode.DecodeThreadCatchUp; /// 注入;仅在 Timeline 多线程驱动路径下由 TryPrepareDecoder 写入 Decoder。 public int timelineMultithreadedWaitMs = 150; /// 注入,用于在运行时匹配 读取 clip.clipIn public PlayableAsset sourcePlayableAsset; /// 掉帧/大步时间时,用顺序 追赶的最大帧数;超过则走 Seek 擦洗。 private const int MaxSequentialCatchUpFrames = 32; private bool _prepared; private long _lastScrubFrame = -1; private LeviathanVideoDecoderBase _decoderPausedForTimeline; public override void OnBehaviourPlay(Playable playable, FrameData info) { ReleaseTimelineDecoderClock(); _prepared = false; _lastScrubFrame = -1; } public override void OnPlayableDestroy(Playable playable) { ReleaseTimelineDecoderClock(); base.OnPlayableDestroy(playable); } public override void ProcessFrame(Playable playable, FrameData info, object playerData) { var decoder = playerData as LeviathanVideoDecoderBase; if (decoder == null) return; if (!decoder.isActiveAndEnabled) { if (decoder.IsDecoderValid && decoder.IsMultithreadedDecodeActive) decoder.Pause(); return; } if (info.effectiveWeight <= 0.001f) { if (decoder.IsDecoderValid && decoder.IsMultithreadedDecodeActive) decoder.Pause(); return; } if (_prepared && decoder.IsPlaying && !decoder.IsDecoderValid) { _prepared = false; _lastScrubFrame = -1; } if (!_prepared) { if (!TryPrepareDecoder(playable, decoder)) return; _prepared = true; } if (decoder.FrameInterval <= 0 || decoder.FrameCount <= 0) return; double videoTime = GetSourceVideoTimeSeconds(playable, decoder) + playable.GetTime(); long frame = (long)(videoTime / decoder.FrameInterval) + 1; frame = ClampFrame(frame, decoder.FrameCount); if (decoder.IsMultithreadedDecodeActive && (decoder.TimelineMultithreadedExternalClockActive || decoder.TimelineMultithreadedCatchUpModeActive)) { if (frame == _lastScrubFrame) { // CatchUp 模式:不解码线程暂停,让预解码继续运行(与直接 Update 路径一致) // Lockstep 模式:暂停以保持精确帧对齐 if (!decoder.TimelineMultithreadedCatchUpModeActive) decoder.PauseTimelineMultithreadedHold(); return; } // CatchUp 模式:只处理大 gap(需要 Seek)的情况,小 gap 由 Update 路径自然追赶 // 这样避免每拍都调用沉重的 TimelineCatchUpPresentToFrame(8-13ms) if (decoder.TimelineMultithreadedCatchUpModeActive) { // 用解码器实际显示的帧号计算 gap long currentFrame = decoder.PlayFrameIndex >= 1 ? decoder.PlayFrameIndex : 1; long gap = frame - currentFrame; const int catchUpSeekIfForwardGapAtLeast = 20; if (_lastScrubFrame < 0 || gap < 0 || gap >= catchUpSeekIfForwardGapAtLeast) { // 大 gap 或回跳:需要 Seek if (RunTimelineMultithreadedPresent(decoder, frame)) _lastScrubFrame = frame; RequestTimelinePreviewRepaint(); } else { // 小 gap:只更新 _lastScrubFrame,让 Update 路径的 deltaTime 积累自然追赶 // Update 循环最多消费 2 帧/拍,逐步缩小 gap _lastScrubFrame = frame; RequestTimelinePreviewRepaint(); } return; } // Lockstep 模式:保持原有逻辑 if (RunTimelineMultithreadedPresent(decoder, frame)) _lastScrubFrame = frame; RequestTimelinePreviewRepaint(); return; } if (decoder.IsMultithreadedDecodeActive) { decoder.Resume(); return; } if (frame == _lastScrubFrame) return; DriveDecoderToTimelineFrame(decoder, frame); } private bool WantMultithreadedTimelineDecode() { #if UNITY_WEBGL && !UNITY_EDITOR return false; #else return useMultithreadedDecode; #endif } /// /// 运行态 Timeline 多线程:Sync/CatchUp 期间 Defer GPU,结束后统一 , /// 避免画面拖到 才上屏(减轻与 Timeline 不同步感)。 /// /// CatchUp 时同 ;Lockstep 为 Sync 返回值。 private static bool RunTimelineMultithreadedPresent(LeviathanVideoDecoderBase decoder, long targetFrame) { if (Application.isPlaying) decoder.SetTimelineMultithreadedDeferGpuApply(true); bool ok; try { if (decoder.TimelineMultithreadedCatchUpModeActive) ok = decoder.TimelineCatchUpPresentToFrame(targetFrame); else ok = decoder.SyncMultithreadedToTimelineFrame(targetFrame); } finally { if (Application.isPlaying) { decoder.SetTimelineMultithreadedDeferGpuApply(false); decoder.FlushDeferredTimelineDisplay(); } } return ok; } /// /// 单线程 Timeline:优先顺序解码(免 Seek);小范围落后用多步 ; /// 运行态合并延迟 GPU Apply 降低每帧同步开销。 /// private void DriveDecoderToTimelineFrame(LeviathanVideoDecoderBase decoder, long frame) { bool deferGpuApply = Application.isPlaying; void FinishScrub(bool applyImmediately) { decoder.ScrubToFrameAndDisplay(frame, applyImmediately); if (deferGpuApply) decoder.FlushDeferredTimelineDisplay(); _lastScrubFrame = frame; RequestTimelinePreviewRepaint(); } if (_lastScrubFrame < 0 || frame < _lastScrubFrame) { FinishScrub(!deferGpuApply); return; } long ahead = frame - _lastScrubFrame; if (ahead > MaxSequentialCatchUpFrames) { FinishScrub(!deferGpuApply); return; } bool applyNow = !deferGpuApply; for (long i = 0; i < ahead; i++) { if (!decoder.TryAdvanceOneFrameForTimeline(applyNow)) { FinishScrub(!deferGpuApply); return; } } if (deferGpuApply) decoder.FlushDeferredTimelineDisplay(); _lastScrubFrame = frame; RequestTimelinePreviewRepaint(); } /// /// 源视频时间轴上的基准时刻(秒)= Timeline ;完整时刻还要加上片段本地已播放时间(见 )。 /// private double GetSourceVideoTimeSeconds(Playable clipPlayable, LeviathanVideoDecoderBase trackBinding) { double clipIn = 0; if (sourcePlayableAsset != null && trackBinding != null && TryResolveTimelineClipIn(clipPlayable, sourcePlayableAsset, trackBinding, out var resolved)) clipIn = resolved; return clipIn; } /// /// 在 Timeline 窗口向左拖片段左缘(或 Inspector 里 Clip In)即可跳过源视频前若干秒,等价于这里的 clip.clipIn。 /// private static bool TryResolveTimelineClipIn(Playable clipPlayable, PlayableAsset asset, LeviathanVideoDecoderBase boundDecoder, out double clipInSeconds) { clipInSeconds = 0; if (asset == null || boundDecoder == null) return false; var graph = clipPlayable.GetGraph(); if (!graph.IsValid()) return false; var resolver = graph.GetResolver(); if (resolver is not PlayableDirector director || director.playableAsset is not TimelineAsset timeline) return false; double dTime = director.time; foreach (var track in timeline.GetOutputTracks()) { if (track is not LeviathanVideoTrack) continue; if (director.GetGenericBinding(track) as LeviathanVideoDecoderBase != boundDecoder) continue; TimelineClip singleCandidate = null; int assetClipCount = 0; foreach (var clip in track.GetClips()) { if (clip.asset != asset) continue; assetClipCount++; singleCandidate = clip; double clipEnd = clip.start + clip.duration; if (dTime + 1e-3 >= clip.start && dTime < clipEnd + 1e-3) { clipInSeconds = clip.clipIn; return true; } } // Director 时间与片段边界浮点误差导致未命中时:该轨上仅一段引用本资源则回退 if (assetClipCount == 1 && singleCandidate != null) { clipInSeconds = singleCandidate.clipIn; return true; } } return false; } private bool TryPrepareDecoder(Playable clipPlayable, LeviathanVideoDecoderBase decoder) { if (!decoder.isActiveAndEnabled) return false; if (decoder.IsPlaying && !decoder.IsDecoderValid) decoder.ForceStopForTimeline(); #if UNITY_EDITOR if (!Application.isPlaying && decoder.IsPlaying) decoder.ForceStopForTimeline(); #endif var source = videoOverride != null ? videoOverride : decoder.bytesFile; if (source == null) return false; double sourceStartSeconds = GetSourceVideoTimeSeconds(clipPlayable, decoder); bool needInit = decoder.bytesFile != source || !decoder.IsPlaying; int beginFrameIndex = needInit ? ComputeBeginFrameIndexFromSourceStartSeconds(source, sourceStartSeconds) : 1; decoder.SetLooping(false); decoder.SetTimelineMultithreadedExternalClock(false); decoder.SetTimelineMultithreadedCatchUpMode(false); bool wantMt = WantMultithreadedTimelineDecode(); bool justInitialized = false; if (decoder.bytesFile != source) { if (!decoder.ChangeVideo(source, decoder.AlphaType, wantMt, beginFrameIndex, syncDecodeFirstFrame, fastDecode)) { decoder.ForceStopForTimeline(); return false; } justInitialized = true; } else if (!decoder.IsPlaying) { int ret = decoder.PlayVideo(decoder.bytesFile, decoder.AlphaType, wantMt, beginFrameIndex, syncDecodeFirstFrame, fastDecode); if (ret != 0) { decoder.ForceStopForTimeline(); return false; } justInitialized = true; } if (!decoder.IsDecoderValid) return false; bool catchUp = wantMt && decoder.IsMultithreadedDecodeActive && timelineMultithreadedDriveMode == LeviathanVideoTimelineMtDriveMode.DecodeThreadCatchUp; decoder.SetTimelineMultithreadedCatchUpMode(catchUp); decoder.SetTimelineMultithreadedExternalClock(wantMt && decoder.IsMultithreadedDecodeActive && !catchUp); decoder.TimelineMultithreadedWaitMs = timelineMultithreadedWaitMs; ApplyTimelineCatchUpParams(decoder); if (justInitialized && decoder.FrameInterval > 0 && decoder.FrameCount > 0) { long startFrame = 1; if (sourceStartSeconds > 0.0001) startFrame = (long)(sourceStartSeconds / decoder.FrameInterval) + 1; startFrame = ClampFrame(startFrame, decoder.FrameCount); if (decoder.TimelineMultithreadedExternalClockActive || decoder.TimelineMultithreadedCatchUpModeActive) { RunTimelineMultithreadedPresent(decoder, startFrame); RequestTimelinePreviewRepaint(); } else if (!decoder.IsMultithreadedDecodeActive) { decoder.ScrubToFrameAndDisplay(startFrame); RequestTimelinePreviewRepaint(); } _lastScrubFrame = startFrame; } else if (!needInit && decoder.FrameInterval > 0 && decoder.FrameCount > 0) { long targetFrame = 1; if (sourceStartSeconds > 0.0001) targetFrame = ClampFrame((long)(sourceStartSeconds / decoder.FrameInterval) + 1, decoder.FrameCount); long cur = decoder.PlayFrameIndex < 1 ? 1 : decoder.PlayFrameIndex; if (decoder.TimelineMultithreadedExternalClockActive || decoder.TimelineMultithreadedCatchUpModeActive) { if (AbsLong(cur - targetFrame) > 1) { RunTimelineMultithreadedPresent(decoder, targetFrame); RequestTimelinePreviewRepaint(); _lastScrubFrame = targetFrame; } } else if (!decoder.IsMultithreadedDecodeActive && AbsLong(cur - targetFrame) > 1) { decoder.ScrubToFrameAndDisplay(targetFrame); RequestTimelinePreviewRepaint(); _lastScrubFrame = targetFrame; } } if (Application.isPlaying) { if (decoder.IsMultithreadedDecodeActive) ReleaseTimelineDecoderClock(); else EnsureDecoderPausedForTimelineDrive(decoder); } return true; } /// /// 从片段资源写入 CatchUp 节拍:整帧预算、单次消费上限;与 字段对应。 /// private void ApplyTimelineCatchUpParams(LeviathanVideoDecoderBase decoder) { var maxConsume = 8; var budget = 0; var lowFpsCap = false; if (sourcePlayableAsset is LeviathanVideoPlayableAsset a) { if (a.timelineCatchUpMaxConsumePerCall > 0) maxConsume = a.timelineCatchUpMaxConsumePerCall; budget = a.timelineCatchUpUnityFrameDisplayBudget; lowFpsCap = a.timelineCatchUpConservativeLowFpsBudget; } #if UNITY_ANDROID && !UNITY_EDITOR if (budget <= 0) budget = 6; #else if (budget <= 0) budget = 8; #endif if (lowFpsCap) budget = Mathf.Min(budget, 3); decoder.TimelineCatchUpMaxConsumePerCall = maxConsume; decoder.TimelineCatchUpUnityFrameDisplayBudget = budget; } private static int ComputeBeginFrameIndexFromSourceStartSeconds(TextAsset source, double sourceStartSeconds) { if (source == null || sourceStartSeconds <= 0.0001) return 1; if (!LeviathanVideoDecoderBase.TryProbeVideoMetaFromBytes(source, out _, out var frameCount, out var fi) || fi <= 0 || frameCount <= 0) return 1; long sf = ClampFrame((long)(sourceStartSeconds / fi) + 1, frameCount); if (sf > int.MaxValue) return int.MaxValue; return (int)sf; } private void EnsureDecoderPausedForTimelineDrive(LeviathanVideoDecoderBase decoder) { if (!Application.isPlaying || decoder == null || !decoder.IsDecoderValid) return; if (_decoderPausedForTimeline == decoder) return; ReleaseTimelineDecoderClock(); decoder.Pause(); _decoderPausedForTimeline = decoder; } private void ReleaseTimelineDecoderClock() { if (_decoderPausedForTimeline == null) return; if (Application.isPlaying) _decoderPausedForTimeline.Resume(); _decoderPausedForTimeline = null; } private static void RequestTimelinePreviewRepaint() { #if UNITY_EDITOR if (!Application.isPlaying) { Canvas.ForceUpdateCanvases(); EditorApplication.delayCall += () => InternalEditorUtility.RepaintAllViews(); } #endif } private static long AbsLong(long v) => v < 0 ? -v : v; private static long ClampFrame(long frame, long frameCount) { if (frame < 1) return 1; if (frameCount > 0 && frame > frameCount) return frameCount; return frame; } } }