From 74c96ba5e9cf543c39d092d8096869c4b5cb0e76 Mon Sep 17 00:00:00 2001 From: evgeny Date: Wed, 14 Oct 2026 13:43:23 +0200 Subject: [PATCH] Let desktop codecs read ahead while keeping presentation bounded --- tools/chatgui/src/streamplayer_window.cpp | 33 ++++++++++++++++------- tools/chatgui/tests/test_stream_view.cpp | 2 +- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/tools/chatgui/src/streamplayer_window.cpp b/tools/chatgui/src/streamplayer_window.cpp index 93aaf560..32516aa5 100644 --- a/tools/chatgui/src/streamplayer_window.cpp +++ b/tools/chatgui/src/streamplayer_window.cpp @@ -52,8 +52,9 @@ struct StreamPlayerWindow::Playback { stream_playback_state timing{}; stream_media_status media{}; int64_t position = -1, clockAt = 0, ready = -1, audioDacAt = 0, deviceLead = 0, lastShownPts = -1; + int64_t decoderLead = 0, framePeriod = 33333, lastInputPts = -1, lastDecodedPts = -1; uint64_t audioFrameStart = 0, shown = 0, underruns = 0, endFrames = UINT64_MAX; - bool started = false, ended = false, activeAudio = false, holdAudio = false, resync = false; + bool started = false, presenting = false, videoReady = false, ended = false, activeAudio = false, holdAudio = false, resync = false; ~Playback() { stream_playback_audio_destroy(sound); } uint64_t played(int64_t now) const { @@ -64,6 +65,7 @@ struct StreamPlayerWindow::Playback { } int64_t clock(int64_t now) { if (!started) return 0; + if (!presenting) return position; if (activeAudio && sound && audioAvailable) return std::max(position, stream_playback_audio_position(sound, played(now))); if (holdAudio) return position; @@ -121,7 +123,7 @@ void StreamPlayerWindow::openAudio() { p.deviceLead = std::max(0, m_audio.callbackTimeUs() - now); if (!p.sound || !p.started || p.resync) return; p.audioFrameStart = stream_playback_audio_written(p.sound); p.audioDacAt = m_audio.callbackTimeUs(); - int count = p.holdAudio ? 0 : stream_playback_audio_pull(p.sound, pcm, frames, p.ended ? INT64_MAX : p.ready); + int count = !p.presenting || p.holdAudio ? 0 : stream_playback_audio_pull(p.sound, pcm, frames, p.ended ? INT64_MAX : p.ready); if (count < 0) { p.resync = true; count = 0; } if ((unsigned)count < frames) { stream_playback_audio_silence(p.sound, frames - (unsigned)count); @@ -201,7 +203,8 @@ void StreamPlayerWindow::decode() { { std::lock_guard guard(p.lock); stream_playback_audio_destroy(p.sound); p.sound = nullptr; - p.videos.clear(); p.videoBytes = 0; p.started = false; p.position = p.ready = -1; + p.videos.clear(); p.videoBytes = 0; p.started = p.presenting = p.videoReady = false; p.position = p.ready = -1; + p.decoderLead = 0; p.framePeriod = 33333; p.lastInputPts = p.lastDecodedPts = -1; p.audioDacAt = 0; p.audioFrameStart = 0; p.holdAudio = p.resync = false; p.endFrames = UINT64_MAX; } if (request && !gui_stream_resync(p.id)) DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop playback resync refused"); @@ -235,19 +238,20 @@ void StreamPlayerWindow::decode() { { std::lock_guard guard(p.lock); resync = p.resync; } if (resync) reset(true); observe(); - int64_t position, videoAhead, audioAhead; + int64_t position, audioAhead; bool wantVideo; { std::lock_guard guard(p.lock); int64_t now = nowUs(); position = p.started ? p.clock(now) : -1; - videoAhead = p.videos.size() < 2 ? 80000 : -1; + wantVideo = p.videos.size() < 2; audioAhead = !p.activeAudio || !p.sound || stream_playback_audio_needs_input(p.sound) ? p.deviceLead + 80000 : -1; - if (p.started && p.ready >= 0) { + if (p.presenting && p.ready >= 0) { uint64_t played = p.played(now); - p.holdAudio = !p.videos.empty() && p.videos.front().pts < position - 40000; + p.holdAudio = !p.videoReady || (p.videos.empty() && p.lastDecodedPts < position - p.framePeriod) + || (!p.videos.empty() && p.videos.front().pts < position - 40000); int virtualEmpty = !p.activeAudio && p.position + (now - p.clockAt) * p.adaptation.tempo_milli / 1000 > p.ready; if (!p.activeAudio && p.holdAudio) position = p.position; int starved = p.holdAudio || virtualEmpty || (p.sound && stream_playback_audio_starved(p.sound, played)); - int64_t guardUs = std::max(33333, p.activeAudio ? p.deviceLead + 10000 : 33333); + int64_t guardUs = std::max(p.decoderLead + p.framePeriod, p.activeAudio ? p.deviceLead + 10000 : p.framePeriod); int64_t pending = p.sound ? stream_playback_audio_pending_gain(p.sound, played) : 0; if (stream_playback_clock_update(&p.adaptation, now, std::max(p.position, position), p.ready, guardUs, pending, !p.ended && starved, &p.timing) < 0) p.resync = true; @@ -258,7 +262,7 @@ void StreamPlayerWindow::decode() { } } StreamRecord record; - if (!gui_stream_pull(p.id, record, position, videoAhead, audioAhead)) { + if (!gui_stream_pull(p.id, record, position, wantVideo, audioAhead)) { if (!gui_stream_ready()) { fail(text("Chat service is stopped")); break; } if ((!opened || waitKey) && nowUs() > deadline) { fail(text("Stream connection timed out")); break; } std::this_thread::sleep_for(std::chrono::milliseconds(5)); continue; @@ -290,7 +294,8 @@ void StreamPlayerWindow::decode() { } if (revision != record.format.revision) { avcodec_free_context(&decoder); waitKey = true; revision = record.format.revision; - { std::lock_guard guard(p.lock); p.videos.clear(); p.videoBytes = 0; } + { std::lock_guard guard(p.lock); p.videos.clear(); p.videoBytes = 0; p.videoReady = false; + p.decoderLead = 0; p.framePeriod = 33333; p.lastInputPts = p.lastDecodedPts = -1; } deadline = nowUs() + 10000000; } if (waitKey && !(record.flags & STREAM_MEDIA_KEY)) continue; @@ -338,6 +343,9 @@ void StreamPlayerWindow::decode() { DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop video packet rejected codec=%u bytes=%zu result=%d", record.format.video_codec, size, sent); reset(true); continue; } + { std::lock_guard guard(p.lock); + if (p.lastInputPts >= 0) p.framePeriod = std::clamp((int64_t)record.pts - p.lastInputPts, 1, 1000000); + p.lastInputPts = (int64_t)record.pts; } int received; while ((received = avcodec_receive_frame(decoder, frame)) == 0) { decoded++; @@ -347,6 +355,11 @@ void StreamPlayerWindow::decode() { fail(text("Invalid stream video dimensions")); p.running = false; break; } int64_t pts = frame->best_effort_timestamp == AV_NOPTS_VALUE ? (int64_t)record.pts : frame->best_effort_timestamp; + { std::lock_guard guard(p.lock); + p.decoderLead = std::max(0, p.lastInputPts - pts); p.lastDecodedPts = pts; p.videoReady = true; + if (!p.presenting) { p.presenting = true; p.clockAt = nowUs(); + DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop first decoded frame id=%llu pts=%lld decoder_lead=%lld", + (unsigned long long)p.id, (long long)pts, (long long)p.decoderLead); } } { std::lock_guard guard(p.lock); if (pts < p.clock(nowUs()) - 100000) { discarded++; continue; } } converter = sws_getCachedContext(converter, frame->width, frame->height, (AVPixelFormat)frame->format, frame->width, frame->height, AV_PIX_FMT_RGB24, SWS_BILINEAR, nullptr, nullptr, nullptr); diff --git a/tools/chatgui/tests/test_stream_view.cpp b/tools/chatgui/tests/test_stream_view.cpp index e8147df8..dab21507 100644 --- a/tools/chatgui/tests/test_stream_view.cpp +++ b/tools/chatgui/tests/test_stream_view.cpp @@ -154,7 +154,7 @@ private slots: unsigned records = 0; QElapsedTimer held; held.start(); while (held.elapsed() < 30) { - if (gui_stream_pull(102, event, 100000, 0, 0)) { QVERIFY(event.kind != StreamRecordKind::Gap); records++; } + if (gui_stream_pull(102, event, 100000, false, 0)) { QVERIFY(event.kind != StreamRecordKind::Gap); records++; } QTest::qWait(1); } QVERIFY(records <= 2); /* Накопленные 27 секунд остаются в RS, а не в очереди frontend. */