#if UNITY_EDITOR
using UnityEditor;
using UnityEditorInternal;
#endif
using UnityEngine;
using UnityEngine.Playables;
using UnityEngine.Timeline;
namespace FT.Timeline
{
public class LeviathanVideoBehaviour : PlayableBehaviour
{
public TextAsset videoOverride;
public bool syncDecodeFirstFrame = true;
public bool fastDecode = true;
/// 使用多线程解码并由 Timeline 对齐帧(WebGL 等平台会降级);编辑器预览在勾选本项时同样走多线程轨。
public bool useMultithreadedDecode = true;
/// 由 注入;多线程时 CatchUp 与 Lockstep 二选一。
public LeviathanVideoTimelineMtDriveMode timelineMultithreadedDriveMode = LeviathanVideoTimelineMtDriveMode.DecodeThreadCatchUp;
/// 由 注入;仅在 Timeline 多线程驱动路径下由 TryPrepareDecoder 写入 Decoder。
public int timelineMultithreadedWaitMs = 150;
/// 由 注入,用于在运行时匹配 读取 clip.clipIn。
public PlayableAsset sourcePlayableAsset;
/// 掉帧/大步时间时,用顺序 追赶的最大帧数;超过则走 Seek 擦洗。
private const int MaxSequentialCatchUpFrames = 32;
private bool _prepared;
private long _lastScrubFrame = -1;
private LeviathanVideoDecoderBase _decoderPausedForTimeline;
public override void OnBehaviourPlay(Playable playable, FrameData info)
{
ReleaseTimelineDecoderClock();
_prepared = false;
_lastScrubFrame = -1;
}
public override void OnPlayableDestroy(Playable playable)
{
ReleaseTimelineDecoderClock();
base.OnPlayableDestroy(playable);
}
public override void ProcessFrame(Playable playable, FrameData info, object playerData)
{
var decoder = playerData as LeviathanVideoDecoderBase;
if (decoder == null)
return;
if (!decoder.isActiveAndEnabled)
{
if (decoder.IsDecoderValid && decoder.IsMultithreadedDecodeActive)
decoder.Pause();
return;
}
if (info.effectiveWeight <= 0.001f)
{
if (decoder.IsDecoderValid && decoder.IsMultithreadedDecodeActive)
decoder.Pause();
return;
}
if (_prepared && decoder.IsPlaying && !decoder.IsDecoderValid)
{
_prepared = false;
_lastScrubFrame = -1;
}
if (!_prepared)
{
if (!TryPrepareDecoder(playable, decoder))
return;
_prepared = true;
}
if (decoder.FrameInterval <= 0 || decoder.FrameCount <= 0)
return;
double videoTime = GetSourceVideoTimeSeconds(playable, decoder) + playable.GetTime();
long frame = (long)(videoTime / decoder.FrameInterval) + 1;
frame = ClampFrame(frame, decoder.FrameCount);
if (decoder.IsMultithreadedDecodeActive &&
(decoder.TimelineMultithreadedExternalClockActive || decoder.TimelineMultithreadedCatchUpModeActive))
{
if (frame == _lastScrubFrame)
{
// CatchUp 模式:不解码线程暂停,让预解码继续运行(与直接 Update 路径一致)
// Lockstep 模式:暂停以保持精确帧对齐
if (!decoder.TimelineMultithreadedCatchUpModeActive)
decoder.PauseTimelineMultithreadedHold();
return;
}
// CatchUp 模式:只处理大 gap(需要 Seek)的情况,小 gap 由 Update 路径自然追赶
// 这样避免每拍都调用沉重的 TimelineCatchUpPresentToFrame(8-13ms)
if (decoder.TimelineMultithreadedCatchUpModeActive)
{
// 用解码器实际显示的帧号计算 gap
long currentFrame = decoder.PlayFrameIndex >= 1 ? decoder.PlayFrameIndex : 1;
long gap = frame - currentFrame;
const int catchUpSeekIfForwardGapAtLeast = 20;
if (_lastScrubFrame < 0 || gap < 0 || gap >= catchUpSeekIfForwardGapAtLeast)
{
// 大 gap 或回跳:需要 Seek
if (RunTimelineMultithreadedPresent(decoder, frame))
_lastScrubFrame = frame;
RequestTimelinePreviewRepaint();
}
else
{
// 小 gap:只更新 _lastScrubFrame,让 Update 路径的 deltaTime 积累自然追赶
// Update 循环最多消费 2 帧/拍,逐步缩小 gap
_lastScrubFrame = frame;
RequestTimelinePreviewRepaint();
}
return;
}
// Lockstep 模式:保持原有逻辑
if (RunTimelineMultithreadedPresent(decoder, frame))
_lastScrubFrame = frame;
RequestTimelinePreviewRepaint();
return;
}
if (decoder.IsMultithreadedDecodeActive)
{
decoder.Resume();
return;
}
if (frame == _lastScrubFrame)
return;
DriveDecoderToTimelineFrame(decoder, frame);
}
private bool WantMultithreadedTimelineDecode()
{
#if UNITY_WEBGL && !UNITY_EDITOR
return false;
#else
return useMultithreadedDecode;
#endif
}
///
/// 运行态 Timeline 多线程:Sync/CatchUp 期间 Defer GPU,结束后统一 ,
/// 避免画面拖到 才上屏(减轻与 Timeline 不同步感)。
///
/// CatchUp 时同 ;Lockstep 为 Sync 返回值。
private static bool RunTimelineMultithreadedPresent(LeviathanVideoDecoderBase decoder, long targetFrame)
{
if (Application.isPlaying)
decoder.SetTimelineMultithreadedDeferGpuApply(true);
bool ok;
try
{
if (decoder.TimelineMultithreadedCatchUpModeActive)
ok = decoder.TimelineCatchUpPresentToFrame(targetFrame);
else
ok = decoder.SyncMultithreadedToTimelineFrame(targetFrame);
}
finally
{
if (Application.isPlaying)
{
decoder.SetTimelineMultithreadedDeferGpuApply(false);
decoder.FlushDeferredTimelineDisplay();
}
}
return ok;
}
///
/// 单线程 Timeline:优先顺序解码(免 Seek);小范围落后用多步 ;
/// 运行态合并延迟 GPU Apply 降低每帧同步开销。
///
private void DriveDecoderToTimelineFrame(LeviathanVideoDecoderBase decoder, long frame)
{
bool deferGpuApply = Application.isPlaying;
void FinishScrub(bool applyImmediately)
{
decoder.ScrubToFrameAndDisplay(frame, applyImmediately);
if (deferGpuApply)
decoder.FlushDeferredTimelineDisplay();
_lastScrubFrame = frame;
RequestTimelinePreviewRepaint();
}
if (_lastScrubFrame < 0 || frame < _lastScrubFrame)
{
FinishScrub(!deferGpuApply);
return;
}
long ahead = frame - _lastScrubFrame;
if (ahead > MaxSequentialCatchUpFrames)
{
FinishScrub(!deferGpuApply);
return;
}
bool applyNow = !deferGpuApply;
for (long i = 0; i < ahead; i++)
{
if (!decoder.TryAdvanceOneFrameForTimeline(applyNow))
{
FinishScrub(!deferGpuApply);
return;
}
}
if (deferGpuApply)
decoder.FlushDeferredTimelineDisplay();
_lastScrubFrame = frame;
RequestTimelinePreviewRepaint();
}
///
/// 源视频时间轴上的基准时刻(秒)= Timeline ;完整时刻还要加上片段本地已播放时间(见 )。
///
private double GetSourceVideoTimeSeconds(Playable clipPlayable, LeviathanVideoDecoderBase trackBinding)
{
double clipIn = 0;
if (sourcePlayableAsset != null && trackBinding != null &&
TryResolveTimelineClipIn(clipPlayable, sourcePlayableAsset, trackBinding, out var resolved))
clipIn = resolved;
return clipIn;
}
///
/// 在 Timeline 窗口向左拖片段左缘(或 Inspector 里 Clip In)即可跳过源视频前若干秒,等价于这里的 clip.clipIn。
///
private static bool TryResolveTimelineClipIn(Playable clipPlayable, PlayableAsset asset,
LeviathanVideoDecoderBase boundDecoder, out double clipInSeconds)
{
clipInSeconds = 0;
if (asset == null || boundDecoder == null)
return false;
var graph = clipPlayable.GetGraph();
if (!graph.IsValid())
return false;
var resolver = graph.GetResolver();
if (resolver is not PlayableDirector director || director.playableAsset is not TimelineAsset timeline)
return false;
double dTime = director.time;
foreach (var track in timeline.GetOutputTracks())
{
if (track is not LeviathanVideoTrack)
continue;
if (director.GetGenericBinding(track) as LeviathanVideoDecoderBase != boundDecoder)
continue;
TimelineClip singleCandidate = null;
int assetClipCount = 0;
foreach (var clip in track.GetClips())
{
if (clip.asset != asset)
continue;
assetClipCount++;
singleCandidate = clip;
double clipEnd = clip.start + clip.duration;
if (dTime + 1e-3 >= clip.start && dTime < clipEnd + 1e-3)
{
clipInSeconds = clip.clipIn;
return true;
}
}
// Director 时间与片段边界浮点误差导致未命中时:该轨上仅一段引用本资源则回退
if (assetClipCount == 1 && singleCandidate != null)
{
clipInSeconds = singleCandidate.clipIn;
return true;
}
}
return false;
}
private bool TryPrepareDecoder(Playable clipPlayable, LeviathanVideoDecoderBase decoder)
{
if (!decoder.isActiveAndEnabled)
return false;
if (decoder.IsPlaying && !decoder.IsDecoderValid)
decoder.ForceStopForTimeline();
#if UNITY_EDITOR
if (!Application.isPlaying && decoder.IsPlaying)
decoder.ForceStopForTimeline();
#endif
var source = videoOverride != null ? videoOverride : decoder.bytesFile;
if (source == null)
return false;
double sourceStartSeconds = GetSourceVideoTimeSeconds(clipPlayable, decoder);
bool needInit = decoder.bytesFile != source || !decoder.IsPlaying;
int beginFrameIndex = needInit ? ComputeBeginFrameIndexFromSourceStartSeconds(source, sourceStartSeconds) : 1;
decoder.SetLooping(false);
decoder.SetTimelineMultithreadedExternalClock(false);
decoder.SetTimelineMultithreadedCatchUpMode(false);
bool wantMt = WantMultithreadedTimelineDecode();
bool justInitialized = false;
if (decoder.bytesFile != source)
{
if (!decoder.ChangeVideo(source, decoder.AlphaType, wantMt, beginFrameIndex,
syncDecodeFirstFrame, fastDecode))
{
decoder.ForceStopForTimeline();
return false;
}
justInitialized = true;
}
else if (!decoder.IsPlaying)
{
int ret = decoder.PlayVideo(decoder.bytesFile, decoder.AlphaType, wantMt,
beginFrameIndex, syncDecodeFirstFrame, fastDecode);
if (ret != 0)
{
decoder.ForceStopForTimeline();
return false;
}
justInitialized = true;
}
if (!decoder.IsDecoderValid)
return false;
bool catchUp = wantMt && decoder.IsMultithreadedDecodeActive &&
timelineMultithreadedDriveMode == LeviathanVideoTimelineMtDriveMode.DecodeThreadCatchUp;
decoder.SetTimelineMultithreadedCatchUpMode(catchUp);
decoder.SetTimelineMultithreadedExternalClock(wantMt && decoder.IsMultithreadedDecodeActive && !catchUp);
decoder.TimelineMultithreadedWaitMs = timelineMultithreadedWaitMs;
ApplyTimelineCatchUpParams(decoder);
if (justInitialized && decoder.FrameInterval > 0 && decoder.FrameCount > 0)
{
long startFrame = 1;
if (sourceStartSeconds > 0.0001)
startFrame = (long)(sourceStartSeconds / decoder.FrameInterval) + 1;
startFrame = ClampFrame(startFrame, decoder.FrameCount);
if (decoder.TimelineMultithreadedExternalClockActive || decoder.TimelineMultithreadedCatchUpModeActive)
{
RunTimelineMultithreadedPresent(decoder, startFrame);
RequestTimelinePreviewRepaint();
}
else if (!decoder.IsMultithreadedDecodeActive)
{
decoder.ScrubToFrameAndDisplay(startFrame);
RequestTimelinePreviewRepaint();
}
_lastScrubFrame = startFrame;
}
else if (!needInit && decoder.FrameInterval > 0 && decoder.FrameCount > 0)
{
long targetFrame = 1;
if (sourceStartSeconds > 0.0001)
targetFrame = ClampFrame((long)(sourceStartSeconds / decoder.FrameInterval) + 1, decoder.FrameCount);
long cur = decoder.PlayFrameIndex < 1 ? 1 : decoder.PlayFrameIndex;
if (decoder.TimelineMultithreadedExternalClockActive || decoder.TimelineMultithreadedCatchUpModeActive)
{
if (AbsLong(cur - targetFrame) > 1)
{
RunTimelineMultithreadedPresent(decoder, targetFrame);
RequestTimelinePreviewRepaint();
_lastScrubFrame = targetFrame;
}
}
else if (!decoder.IsMultithreadedDecodeActive && AbsLong(cur - targetFrame) > 1)
{
decoder.ScrubToFrameAndDisplay(targetFrame);
RequestTimelinePreviewRepaint();
_lastScrubFrame = targetFrame;
}
}
if (Application.isPlaying)
{
if (decoder.IsMultithreadedDecodeActive)
ReleaseTimelineDecoderClock();
else
EnsureDecoderPausedForTimelineDrive(decoder);
}
return true;
}
///
/// 从片段资源写入 CatchUp 节拍:整帧预算、单次消费上限;与 字段对应。
///
private void ApplyTimelineCatchUpParams(LeviathanVideoDecoderBase decoder)
{
var maxConsume = 8;
var budget = 0;
var lowFpsCap = false;
if (sourcePlayableAsset is LeviathanVideoPlayableAsset a)
{
if (a.timelineCatchUpMaxConsumePerCall > 0)
maxConsume = a.timelineCatchUpMaxConsumePerCall;
budget = a.timelineCatchUpUnityFrameDisplayBudget;
lowFpsCap = a.timelineCatchUpConservativeLowFpsBudget;
}
#if UNITY_ANDROID && !UNITY_EDITOR
if (budget <= 0)
budget = 6;
#else
if (budget <= 0)
budget = 8;
#endif
if (lowFpsCap)
budget = Mathf.Min(budget, 3);
decoder.TimelineCatchUpMaxConsumePerCall = maxConsume;
decoder.TimelineCatchUpUnityFrameDisplayBudget = budget;
}
private static int ComputeBeginFrameIndexFromSourceStartSeconds(TextAsset source, double sourceStartSeconds)
{
if (source == null || sourceStartSeconds <= 0.0001)
return 1;
if (!LeviathanVideoDecoderBase.TryProbeVideoMetaFromBytes(source, out _, out var frameCount, out var fi) ||
fi <= 0 || frameCount <= 0)
return 1;
long sf = ClampFrame((long)(sourceStartSeconds / fi) + 1, frameCount);
if (sf > int.MaxValue)
return int.MaxValue;
return (int)sf;
}
private void EnsureDecoderPausedForTimelineDrive(LeviathanVideoDecoderBase decoder)
{
if (!Application.isPlaying || decoder == null || !decoder.IsDecoderValid)
return;
if (_decoderPausedForTimeline == decoder)
return;
ReleaseTimelineDecoderClock();
decoder.Pause();
_decoderPausedForTimeline = decoder;
}
private void ReleaseTimelineDecoderClock()
{
if (_decoderPausedForTimeline == null)
return;
if (Application.isPlaying)
_decoderPausedForTimeline.Resume();
_decoderPausedForTimeline = null;
}
private static void RequestTimelinePreviewRepaint()
{
#if UNITY_EDITOR
if (!Application.isPlaying)
{
Canvas.ForceUpdateCanvases();
EditorApplication.delayCall += () => InternalEditorUtility.RepaintAllViews();
}
#endif
}
private static long AbsLong(long v) => v < 0 ? -v : v;
private static long ClampFrame(long frame, long frameCount)
{
if (frame < 1)
return 1;
if (frameCount > 0 && frame > frameCount)
return frameCount;
return frame;
}
}
}