490 lines
20 KiB
C#
490 lines
20 KiB
C#
#if UNITY_EDITOR
|
||
using UnityEditor;
|
||
using UnityEditorInternal;
|
||
#endif
|
||
using UnityEngine;
|
||
using UnityEngine.Playables;
|
||
using UnityEngine.Timeline;
|
||
|
||
namespace FT.Timeline
|
||
{
|
||
public class LeviathanVideoBehaviour : PlayableBehaviour
|
||
{
|
||
public TextAsset videoOverride;
|
||
public bool syncDecodeFirstFrame = true;
|
||
public bool fastDecode = true;
|
||
/// <summary>使用多线程解码并由 Timeline 对齐帧（WebGL 等平台会降级）；编辑器预览在勾选本项时同样走多线程轨。</summary>
|
||
public bool useMultithreadedDecode = true;
|
||
/// <summary>由 <see cref="LeviathanVideoPlayableAsset"/> 注入；多线程时 CatchUp 与 Lockstep 二选一。</summary>
|
||
public LeviathanVideoTimelineMtDriveMode timelineMultithreadedDriveMode = LeviathanVideoTimelineMtDriveMode.DecodeThreadCatchUp;
|
||
/// <summary>由 <see cref="LeviathanVideoPlayableAsset"/> 注入；仅在 Timeline 多线程驱动路径下由 TryPrepareDecoder 写入 Decoder。</summary>
|
||
public int timelineMultithreadedWaitMs = 150;
|
||
/// <summary>由 <see cref="LeviathanVideoPlayableAsset.CreatePlayable"/> 注入，用于在运行时匹配 <see cref="TimelineClip"/> 读取 <c>clip.clipIn</c>。</summary>
|
||
public PlayableAsset sourcePlayableAsset;
|
||
|
||
/// <summary>掉帧/大步时间时，用顺序 <see cref="LeviathanVideoDecoderBase.TryAdvanceOneFrameForTimeline"/> 追赶的最大帧数；超过则走 Seek 擦洗。</summary>
|
||
private const int MaxSequentialCatchUpFrames = 32;
|
||
|
||
private bool _prepared;
|
||
private long _lastScrubFrame = -1;
|
||
private LeviathanVideoDecoderBase _decoderPausedForTimeline;
|
||
|
||
public override void OnBehaviourPlay(Playable playable, FrameData info)
|
||
{
|
||
ReleaseTimelineDecoderClock();
|
||
_prepared = false;
|
||
_lastScrubFrame = -1;
|
||
}
|
||
|
||
public override void OnPlayableDestroy(Playable playable)
|
||
{
|
||
ReleaseTimelineDecoderClock();
|
||
base.OnPlayableDestroy(playable);
|
||
}
|
||
|
||
public override void ProcessFrame(Playable playable, FrameData info, object playerData)
|
||
{
|
||
var decoder = playerData as LeviathanVideoDecoderBase;
|
||
if (decoder == null)
|
||
return;
|
||
|
||
if (!decoder.isActiveAndEnabled)
|
||
{
|
||
if (decoder.IsDecoderValid && decoder.IsMultithreadedDecodeActive)
|
||
decoder.Pause();
|
||
return;
|
||
}
|
||
|
||
if (info.effectiveWeight <= 0.001f)
|
||
{
|
||
if (decoder.IsDecoderValid && decoder.IsMultithreadedDecodeActive)
|
||
decoder.Pause();
|
||
return;
|
||
}
|
||
|
||
if (_prepared && decoder.IsPlaying && !decoder.IsDecoderValid)
|
||
{
|
||
_prepared = false;
|
||
_lastScrubFrame = -1;
|
||
}
|
||
|
||
if (!_prepared)
|
||
{
|
||
if (!TryPrepareDecoder(playable, decoder))
|
||
return;
|
||
_prepared = true;
|
||
}
|
||
|
||
if (decoder.FrameInterval <= 0 || decoder.FrameCount <= 0)
|
||
return;
|
||
|
||
double videoTime = GetSourceVideoTimeSeconds(playable, decoder) + playable.GetTime();
|
||
long frame = (long)(videoTime / decoder.FrameInterval) + 1;
|
||
frame = ClampFrame(frame, decoder.FrameCount);
|
||
|
||
if (decoder.IsMultithreadedDecodeActive &&
|
||
(decoder.TimelineMultithreadedExternalClockActive || decoder.TimelineMultithreadedCatchUpModeActive))
|
||
{
|
||
if (frame == _lastScrubFrame)
|
||
{
|
||
// CatchUp 模式：不解码线程暂停，让预解码继续运行（与直接 Update 路径一致）
|
||
// Lockstep 模式：暂停以保持精确帧对齐
|
||
if (!decoder.TimelineMultithreadedCatchUpModeActive)
|
||
decoder.PauseTimelineMultithreadedHold();
|
||
return;
|
||
}
|
||
|
||
// CatchUp 模式：只处理大 gap（需要 Seek）的情况，小 gap 由 Update 路径自然追赶
|
||
// 这样避免每拍都调用沉重的 TimelineCatchUpPresentToFrame（8-13ms）
|
||
if (decoder.TimelineMultithreadedCatchUpModeActive)
|
||
{
|
||
// 用解码器实际显示的帧号计算 gap
|
||
long currentFrame = decoder.PlayFrameIndex >= 1 ? decoder.PlayFrameIndex : 1;
|
||
long gap = frame - currentFrame;
|
||
const int catchUpSeekIfForwardGapAtLeast = 20;
|
||
if (_lastScrubFrame < 0 || gap < 0 || gap >= catchUpSeekIfForwardGapAtLeast)
|
||
{
|
||
// 大 gap 或回跳：需要 Seek
|
||
if (RunTimelineMultithreadedPresent(decoder, frame))
|
||
_lastScrubFrame = frame;
|
||
RequestTimelinePreviewRepaint();
|
||
}
|
||
else
|
||
{
|
||
// 小 gap：只更新 _lastScrubFrame，让 Update 路径的 deltaTime 积累自然追赶
|
||
// Update 循环最多消费 2 帧/拍，逐步缩小 gap
|
||
_lastScrubFrame = frame;
|
||
RequestTimelinePreviewRepaint();
|
||
}
|
||
return;
|
||
}
|
||
|
||
// Lockstep 模式：保持原有逻辑
|
||
if (RunTimelineMultithreadedPresent(decoder, frame))
|
||
_lastScrubFrame = frame;
|
||
RequestTimelinePreviewRepaint();
|
||
return;
|
||
}
|
||
|
||
if (decoder.IsMultithreadedDecodeActive)
|
||
{
|
||
decoder.Resume();
|
||
return;
|
||
}
|
||
|
||
if (frame == _lastScrubFrame)
|
||
return;
|
||
|
||
DriveDecoderToTimelineFrame(decoder, frame);
|
||
}
|
||
|
||
private bool WantMultithreadedTimelineDecode()
|
||
{
|
||
#if UNITY_WEBGL && !UNITY_EDITOR
|
||
return false;
|
||
#else
|
||
return useMultithreadedDecode;
|
||
#endif
|
||
}
|
||
|
||
/// <summary>
|
||
/// 运行态 Timeline 多线程：Sync/CatchUp 期间 Defer GPU，结束后统一 <see cref="LeviathanVideoDecoderBase.FlushDeferredTimelineDisplay"/>，
|
||
/// 避免画面拖到 <see cref="LeviathanVideoDecoderBase.LateUpdate"/> 才上屏（减轻与 Timeline 不同步感）。
|
||
/// </summary>
|
||
/// <returns>CatchUp 时同 <see cref="LeviathanVideoDecoderBase.TimelineCatchUpPresentToFrame"/>；Lockstep 为 Sync 返回值。</returns>
|
||
private static bool RunTimelineMultithreadedPresent(LeviathanVideoDecoderBase decoder, long targetFrame)
|
||
{
|
||
if (Application.isPlaying)
|
||
decoder.SetTimelineMultithreadedDeferGpuApply(true);
|
||
bool ok;
|
||
try
|
||
{
|
||
if (decoder.TimelineMultithreadedCatchUpModeActive)
|
||
ok = decoder.TimelineCatchUpPresentToFrame(targetFrame);
|
||
else
|
||
ok = decoder.SyncMultithreadedToTimelineFrame(targetFrame);
|
||
}
|
||
finally
|
||
{
|
||
if (Application.isPlaying)
|
||
{
|
||
decoder.SetTimelineMultithreadedDeferGpuApply(false);
|
||
decoder.FlushDeferredTimelineDisplay();
|
||
}
|
||
}
|
||
|
||
return ok;
|
||
}
|
||
|
||
/// <summary>
|
||
/// 单线程 Timeline：优先顺序解码（免 Seek）；小范围落后用多步 <see cref="LeviathanVideoDecoderBase.TryAdvanceOneFrameForTimeline"/>；
|
||
/// 运行态合并延迟 GPU Apply 降低每帧同步开销。
|
||
/// </summary>
|
||
private void DriveDecoderToTimelineFrame(LeviathanVideoDecoderBase decoder, long frame)
|
||
{
|
||
bool deferGpuApply = Application.isPlaying;
|
||
|
||
void FinishScrub(bool applyImmediately)
|
||
{
|
||
decoder.ScrubToFrameAndDisplay(frame, applyImmediately);
|
||
if (deferGpuApply)
|
||
decoder.FlushDeferredTimelineDisplay();
|
||
_lastScrubFrame = frame;
|
||
RequestTimelinePreviewRepaint();
|
||
}
|
||
|
||
if (_lastScrubFrame < 0 || frame < _lastScrubFrame)
|
||
{
|
||
FinishScrub(!deferGpuApply);
|
||
return;
|
||
}
|
||
|
||
long ahead = frame - _lastScrubFrame;
|
||
if (ahead > MaxSequentialCatchUpFrames)
|
||
{
|
||
FinishScrub(!deferGpuApply);
|
||
return;
|
||
}
|
||
|
||
bool applyNow = !deferGpuApply;
|
||
for (long i = 0; i < ahead; i++)
|
||
{
|
||
if (!decoder.TryAdvanceOneFrameForTimeline(applyNow))
|
||
{
|
||
FinishScrub(!deferGpuApply);
|
||
return;
|
||
}
|
||
}
|
||
|
||
if (deferGpuApply)
|
||
decoder.FlushDeferredTimelineDisplay();
|
||
_lastScrubFrame = frame;
|
||
RequestTimelinePreviewRepaint();
|
||
}
|
||
|
||
/// <summary>
|
||
/// 源视频时间轴上的基准时刻（秒）= Timeline <see cref="TimelineClip.clipIn"/>；完整时刻还要加上片段本地已播放时间（见 <see cref="ProcessFrame"/>）。
|
||
/// </summary>
|
||
private double GetSourceVideoTimeSeconds(Playable clipPlayable, LeviathanVideoDecoderBase trackBinding)
|
||
{
|
||
double clipIn = 0;
|
||
if (sourcePlayableAsset != null && trackBinding != null &&
|
||
TryResolveTimelineClipIn(clipPlayable, sourcePlayableAsset, trackBinding, out var resolved))
|
||
clipIn = resolved;
|
||
return clipIn;
|
||
}
|
||
|
||
/// <summary>
|
||
/// 在 Timeline 窗口向左拖片段左缘（或 Inspector 里 Clip In）即可跳过源视频前若干秒，等价于这里的 <c>clip.clipIn</c>。
|
||
/// </summary>
|
||
private static bool TryResolveTimelineClipIn(Playable clipPlayable, PlayableAsset asset,
|
||
LeviathanVideoDecoderBase boundDecoder, out double clipInSeconds)
|
||
{
|
||
clipInSeconds = 0;
|
||
if (asset == null || boundDecoder == null)
|
||
return false;
|
||
var graph = clipPlayable.GetGraph();
|
||
if (!graph.IsValid())
|
||
return false;
|
||
var resolver = graph.GetResolver();
|
||
if (resolver is not PlayableDirector director || director.playableAsset is not TimelineAsset timeline)
|
||
return false;
|
||
|
||
double dTime = director.time;
|
||
foreach (var track in timeline.GetOutputTracks())
|
||
{
|
||
if (track is not LeviathanVideoTrack)
|
||
continue;
|
||
if (director.GetGenericBinding(track) as LeviathanVideoDecoderBase != boundDecoder)
|
||
continue;
|
||
|
||
TimelineClip singleCandidate = null;
|
||
int assetClipCount = 0;
|
||
foreach (var clip in track.GetClips())
|
||
{
|
||
if (clip.asset != asset)
|
||
continue;
|
||
assetClipCount++;
|
||
singleCandidate = clip;
|
||
double clipEnd = clip.start + clip.duration;
|
||
if (dTime + 1e-3 >= clip.start && dTime < clipEnd + 1e-3)
|
||
{
|
||
clipInSeconds = clip.clipIn;
|
||
return true;
|
||
}
|
||
}
|
||
|
||
// Director 时间与片段边界浮点误差导致未命中时：该轨上仅一段引用本资源则回退
|
||
if (assetClipCount == 1 && singleCandidate != null)
|
||
{
|
||
clipInSeconds = singleCandidate.clipIn;
|
||
return true;
|
||
}
|
||
}
|
||
|
||
return false;
|
||
}
|
||
|
||
private bool TryPrepareDecoder(Playable clipPlayable, LeviathanVideoDecoderBase decoder)
|
||
{
|
||
if (!decoder.isActiveAndEnabled)
|
||
return false;
|
||
|
||
if (decoder.IsPlaying && !decoder.IsDecoderValid)
|
||
decoder.ForceStopForTimeline();
|
||
|
||
#if UNITY_EDITOR
|
||
if (!Application.isPlaying && decoder.IsPlaying)
|
||
decoder.ForceStopForTimeline();
|
||
#endif
|
||
|
||
var source = videoOverride != null ? videoOverride : decoder.bytesFile;
|
||
if (source == null)
|
||
return false;
|
||
|
||
double sourceStartSeconds = GetSourceVideoTimeSeconds(clipPlayable, decoder);
|
||
bool needInit = decoder.bytesFile != source || !decoder.IsPlaying;
|
||
int beginFrameIndex = needInit ? ComputeBeginFrameIndexFromSourceStartSeconds(source, sourceStartSeconds) : 1;
|
||
|
||
decoder.SetLooping(false);
|
||
decoder.SetTimelineMultithreadedExternalClock(false);
|
||
decoder.SetTimelineMultithreadedCatchUpMode(false);
|
||
|
||
bool wantMt = WantMultithreadedTimelineDecode();
|
||
|
||
bool justInitialized = false;
|
||
|
||
if (decoder.bytesFile != source)
|
||
{
|
||
if (!decoder.ChangeVideo(source, decoder.AlphaType, wantMt, beginFrameIndex,
|
||
syncDecodeFirstFrame, fastDecode))
|
||
{
|
||
decoder.ForceStopForTimeline();
|
||
return false;
|
||
}
|
||
|
||
justInitialized = true;
|
||
}
|
||
else if (!decoder.IsPlaying)
|
||
{
|
||
int ret = decoder.PlayVideo(decoder.bytesFile, decoder.AlphaType, wantMt,
|
||
beginFrameIndex, syncDecodeFirstFrame, fastDecode);
|
||
if (ret != 0)
|
||
{
|
||
decoder.ForceStopForTimeline();
|
||
return false;
|
||
}
|
||
|
||
justInitialized = true;
|
||
}
|
||
|
||
if (!decoder.IsDecoderValid)
|
||
return false;
|
||
|
||
bool catchUp = wantMt && decoder.IsMultithreadedDecodeActive &&
|
||
timelineMultithreadedDriveMode == LeviathanVideoTimelineMtDriveMode.DecodeThreadCatchUp;
|
||
decoder.SetTimelineMultithreadedCatchUpMode(catchUp);
|
||
decoder.SetTimelineMultithreadedExternalClock(wantMt && decoder.IsMultithreadedDecodeActive && !catchUp);
|
||
decoder.TimelineMultithreadedWaitMs = timelineMultithreadedWaitMs;
|
||
ApplyTimelineCatchUpParams(decoder);
|
||
|
||
if (justInitialized && decoder.FrameInterval > 0 && decoder.FrameCount > 0)
|
||
{
|
||
long startFrame = 1;
|
||
if (sourceStartSeconds > 0.0001)
|
||
startFrame = (long)(sourceStartSeconds / decoder.FrameInterval) + 1;
|
||
startFrame = ClampFrame(startFrame, decoder.FrameCount);
|
||
if (decoder.TimelineMultithreadedExternalClockActive || decoder.TimelineMultithreadedCatchUpModeActive)
|
||
{
|
||
RunTimelineMultithreadedPresent(decoder, startFrame);
|
||
RequestTimelinePreviewRepaint();
|
||
}
|
||
else if (!decoder.IsMultithreadedDecodeActive)
|
||
{
|
||
decoder.ScrubToFrameAndDisplay(startFrame);
|
||
RequestTimelinePreviewRepaint();
|
||
}
|
||
|
||
_lastScrubFrame = startFrame;
|
||
}
|
||
else if (!needInit && decoder.FrameInterval > 0 && decoder.FrameCount > 0)
|
||
{
|
||
long targetFrame = 1;
|
||
if (sourceStartSeconds > 0.0001)
|
||
targetFrame = ClampFrame((long)(sourceStartSeconds / decoder.FrameInterval) + 1, decoder.FrameCount);
|
||
long cur = decoder.PlayFrameIndex < 1 ? 1 : decoder.PlayFrameIndex;
|
||
if (decoder.TimelineMultithreadedExternalClockActive || decoder.TimelineMultithreadedCatchUpModeActive)
|
||
{
|
||
if (AbsLong(cur - targetFrame) > 1)
|
||
{
|
||
RunTimelineMultithreadedPresent(decoder, targetFrame);
|
||
RequestTimelinePreviewRepaint();
|
||
_lastScrubFrame = targetFrame;
|
||
}
|
||
}
|
||
else if (!decoder.IsMultithreadedDecodeActive && AbsLong(cur - targetFrame) > 1)
|
||
{
|
||
decoder.ScrubToFrameAndDisplay(targetFrame);
|
||
RequestTimelinePreviewRepaint();
|
||
_lastScrubFrame = targetFrame;
|
||
}
|
||
}
|
||
|
||
if (Application.isPlaying)
|
||
{
|
||
if (decoder.IsMultithreadedDecodeActive)
|
||
ReleaseTimelineDecoderClock();
|
||
else
|
||
EnsureDecoderPausedForTimelineDrive(decoder);
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
/// <summary>
|
||
/// 从片段资源写入 CatchUp 节拍：整帧预算、单次消费上限；与 <see cref="LeviathanVideoPlayableAsset"/> 字段对应。
|
||
/// </summary>
|
||
private void ApplyTimelineCatchUpParams(LeviathanVideoDecoderBase decoder)
|
||
{
|
||
var maxConsume = 8;
|
||
var budget = 0;
|
||
var lowFpsCap = false;
|
||
if (sourcePlayableAsset is LeviathanVideoPlayableAsset a)
|
||
{
|
||
if (a.timelineCatchUpMaxConsumePerCall > 0)
|
||
maxConsume = a.timelineCatchUpMaxConsumePerCall;
|
||
budget = a.timelineCatchUpUnityFrameDisplayBudget;
|
||
lowFpsCap = a.timelineCatchUpConservativeLowFpsBudget;
|
||
}
|
||
|
||
#if UNITY_ANDROID && !UNITY_EDITOR
|
||
if (budget <= 0)
|
||
budget = 6;
|
||
#else
|
||
if (budget <= 0)
|
||
budget = 8;
|
||
#endif
|
||
if (lowFpsCap)
|
||
budget = Mathf.Min(budget, 3);
|
||
|
||
decoder.TimelineCatchUpMaxConsumePerCall = maxConsume;
|
||
decoder.TimelineCatchUpUnityFrameDisplayBudget = budget;
|
||
}
|
||
|
||
private static int ComputeBeginFrameIndexFromSourceStartSeconds(TextAsset source, double sourceStartSeconds)
|
||
{
|
||
if (source == null || sourceStartSeconds <= 0.0001)
|
||
return 1;
|
||
if (!LeviathanVideoDecoderBase.TryProbeVideoMetaFromBytes(source, out _, out var frameCount, out var fi) ||
|
||
fi <= 0 || frameCount <= 0)
|
||
return 1;
|
||
long sf = ClampFrame((long)(sourceStartSeconds / fi) + 1, frameCount);
|
||
if (sf > int.MaxValue)
|
||
return int.MaxValue;
|
||
return (int)sf;
|
||
}
|
||
|
||
private void EnsureDecoderPausedForTimelineDrive(LeviathanVideoDecoderBase decoder)
|
||
{
|
||
if (!Application.isPlaying || decoder == null || !decoder.IsDecoderValid)
|
||
return;
|
||
if (_decoderPausedForTimeline == decoder)
|
||
return;
|
||
ReleaseTimelineDecoderClock();
|
||
decoder.Pause();
|
||
_decoderPausedForTimeline = decoder;
|
||
}
|
||
|
||
private void ReleaseTimelineDecoderClock()
|
||
{
|
||
if (_decoderPausedForTimeline == null)
|
||
return;
|
||
if (Application.isPlaying)
|
||
_decoderPausedForTimeline.Resume();
|
||
_decoderPausedForTimeline = null;
|
||
}
|
||
|
||
private static void RequestTimelinePreviewRepaint()
|
||
{
|
||
#if UNITY_EDITOR
|
||
if (!Application.isPlaying)
|
||
{
|
||
Canvas.ForceUpdateCanvases();
|
||
EditorApplication.delayCall += () => InternalEditorUtility.RepaintAllViews();
|
||
}
|
||
#endif
|
||
}
|
||
|
||
private static long AbsLong(long v) => v < 0 ? -v : v;
|
||
|
||
private static long ClampFrame(long frame, long frameCount)
|
||
{
|
||
if (frame < 1)
|
||
return 1;
|
||
if (frameCount > 0 && frame > frameCount)
|
||
return frameCount;
|
||
return frame;
|
||
}
|
||
}
|
||
}
|