From 5255133f275109ebac4f306de075e04ee8f1175a Mon Sep 17 00:00:00 2001 From: evgeny Date: Thu, 15 Oct 2026 15:58:33 +0200 Subject: [PATCH] Pace desktop call video independently and coalesce immediate UI publication --- tools/chatgui/src/call_video_engine.cpp | 103 +++++++++--------- tools/chatgui/src/call_video_engine.h | 1 + tools/chatgui/tests/test_call_video_clock.cpp | 1 + 3 files changed, 55 insertions(+), 50 deletions(-) diff --git a/tools/chatgui/src/call_video_engine.cpp b/tools/chatgui/src/call_video_engine.cpp index b22bb814..6c6ba3ad 100644 --- a/tools/chatgui/src/call_video_engine.cpp +++ b/tools/chatgui/src/call_video_engine.cpp @@ -5,6 +5,7 @@ #include "call_video_capture_clock.h" #include "call_camera_rate.h" #include "../../../src/call/call_video_port.h" +#include "../../../src/call/call_video_playout.h" #include "../../../src/call/call_audio.h" #include "debug_config.h" #include "mem.h" @@ -23,7 +24,7 @@ #endif namespace { -int64_t nowUs() { return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); } +int64_t nowUs() { return call_video_playout_now_us(); } void idle() { std::this_thread::sleep_for(std::chrono::milliseconds(5)); } } struct CallVideoEngine::Impl { @@ -38,7 +39,9 @@ struct CallVideoEngine::Impl { uint64_t captured = 0; QImage shown; int64_t shownPts = -1; - uint64_t shownSerial = 0, deliveredSerial = 0; + uint64_t publicationEpoch = 0; + bool publicationPending = false; + int64_t shownAt = 0; int64_t previewPts = -1, previewReportAt = 0, previewMaxUs = 0; unsigned previewFrames = 0; bool previewVisible = false; @@ -60,18 +63,9 @@ CallVideoEngine::CallVideoEngine(QObject* parent) : QObject(parent), m(std::make m_timer.setInterval(20); m_timer.setTimerType(Qt::PreciseTimer); connect(&m_timer, &QTimer::timeout, this, [this] { - QImage image; QString status; bool changed = false, report = false; int64_t shownPts = -1; + QString status; bool report = false; { std::lock_guard guard(m->lock); - if (m->shownSerial != m->deliveredSerial) { - image = m->shown; shownPts = m->shownPts; m->deliveredSerial = m->shownSerial; changed = true; - } if (m->statusDirty) { status = m->status; m->statusDirty = false; report = true; } } - if (changed) { - emit imageChanged(image); - if (!image.isNull() && shownPts >= 0 && shownPts % 1000000 < 34000) - DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "video timing stage=publish call=%llu pts=%lld mono_us=%lld", - (unsigned long long)m->id, (long long)shownPts, (long long)nowUs()); - } if (report) emit statusChanged(status); #ifdef HAVE_STREAM_CAPTURE QVideoFrame preview; bool clear = false; @@ -100,6 +94,24 @@ CallVideoEngine::CallVideoEngine(QObject* parent) : QObject(parent), m(std::make }); } CallVideoEngine::~CallVideoEngine() { stop(); } +// В GUI одновременно не больше одного уведомления; при паузе UI остаётся последний выбранный кадр. +void CallVideoEngine::publishImage(const QImage& image, int64_t pts) { + uint64_t epoch; + { std::lock_guard guard(m->lock); + m->shown = image; m->shownPts = pts; m->shownAt = nowUs(); + if (m->publicationPending) return; + m->publicationPending = true; epoch = m->publicationEpoch; } + QMetaObject::invokeMethod(this, [this, epoch] { + QImage image; int64_t pts, selected; + { std::lock_guard guard(m->lock); + if (epoch != m->publicationEpoch) return; + image = m->shown; pts = m->shownPts; selected = m->shownAt; m->publicationPending = false; } + emit imageChanged(image); + if (!image.isNull() && pts >= 0 && pts % 1000000 < 34000) + DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "video timing stage=publish call=%llu pts=%lld mono_us=%lld ui_wait_us=%lld", + (unsigned long long)m->id, (long long)pts, (long long)nowUs(), (long long)(nowUs() - selected)); + }, Qt::QueuedConnection); +} QList> CallVideoEngine::cameras() { QList> result; #ifdef HAVE_STREAM_CAPTURE @@ -129,7 +141,9 @@ void CallVideoEngine::stop() { if (m->tx.joinable()) m->tx.join(); if (m->rx.joinable()) m->rx.join(); if (m->id) call_video_port_close(m->id); - m->id = 0; m->enabled = false; m->shown = {}; emit imageChanged({}); + { std::lock_guard guard(m->lock); + ++m->publicationEpoch; m->publicationPending = false; m->shown = {}; m->shownPts = -1; } + m->id = 0; m->enabled = false; emit imageChanged({}); } void CallVideoEngine::setVideo(bool enabled, const QByteArray& cameraId, int width, int height, int codec) { closeCamera(); @@ -306,48 +320,22 @@ void CallVideoEngine::transmit() { } void CallVideoEngine::receive() { - struct Picture { int64_t pts; QImage image; }; + struct Picture { call_video_playout_frame timing; QImage image; }; std::deque pictures; CallVideoDecoder decoder; - uint64_t sourceEpoch = 0; int64_t lastShown = -1, report = 0, lastPictureAt = 0; unsigned shown = 0; + call_video_playout playout; call_video_playout_init(&playout, m->id); + int64_t lastPictureAt = 0; uint64_t reportedGeneration = 0, rejectedGeneration = 0; while (m->running) { - int64_t now = nowUs(); call_clock_snapshot clock{}; - call_audio_get_clock(m->id, now, &clock); - if (clock.valid) { - if (sourceEpoch && sourceEpoch != clock.source_epoch) { - DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "video audio clock epoch %llu -> %llu; decoder retained", - (unsigned long long)sourceEpoch, (unsigned long long)clock.source_epoch); - } - sourceEpoch = clock.source_epoch; - QImage selected; - while (!pictures.empty() && pictures.front().pts <= clock.pts_us) { - if (pictures.front().pts > lastShown && pictures.front().pts >= clock.pts_us - 500000) { - selected = pictures.front().image; lastShown = pictures.front().pts; - } - pictures.pop_front(); - } - if (!selected.isNull()) { - std::lock_guard guard(m->lock); m->shown = selected; m->shownPts = lastShown; - ++m->shownSerial; ++shown; lastPictureAt = now; - if (lastShown % 1000000 < 34000) - DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "video timing stage=select call=%llu pts=%lld mono_us=%lld audio_pts=%lld", - (unsigned long long)m->id, (long long)lastShown, (long long)nowUs(), (long long)clock.pts_us); - } - } - // VIDEO_OFF пока отсутствует в протоколе: убираем застывшую картинку после прекращения показа. - if (lastPictureAt && now - lastPictureAt >= 5000000) { - std::lock_guard guard(m->lock); m->shown = {}; ++m->shownSerial; lastPictureAt = 0; - DEBUG_INFO(DEBUG_CATEGORY_CALL, "video playback hidden: no playable frame for 5s"); - } - if (pictures.size() < 16) { + // Один кадр ждёт своего срока, второй позволяет пропустить устаревший показ после паузы CPU. + while (pictures.size() < 2 && m->running) { call_video_frame packet{}; if (call_video_port_pull(m->id, &packet) == 1) { if (packet.format.generation == rejectedGeneration) { u_free((void*)packet.data); continue; } QImage image; const int64_t decodeStarted = nowUs(); if (decoder.decode(packet, image)) { - pictures.push_back({packet.pts_us, image}); + pictures.push_back({{packet.pts_us, packet.received_us}, image}); if (packet.pts_us % 1000000 < 34000) DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "video timing stage=decode call=%llu frame=%llu pts=%lld mono_us=%lld decode_us=%lld queue=%zu", @@ -359,13 +347,28 @@ void CallVideoEngine::receive() { } else { rejectedGeneration = packet.format.generation; call_video_port_decoded(m->id, rejectedGeneration, 0); } - u_free((void*)packet.data); continue; + u_free((void*)packet.data); + } else break; + } + int64_t now = nowUs(); call_clock_snapshot clock{}; + call_audio_get_clock(m->id, now, &clock); + auto action = call_video_playout_poll(&playout, pictures.empty() ? nullptr : &pictures.front().timing, + pictures.size() > 1 ? &pictures[1].timing : nullptr, now, clock.valid, clock.pts_us); + if (action != CALL_VIDEO_PLAYOUT_WAIT) { + if (action == CALL_VIDEO_PLAYOUT_SHOW) { + const auto& f = pictures.front(); publishImage(f.image, f.timing.pts_us); lastPictureAt = now; + if (f.timing.pts_us % 1000000 < 34000) + DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, "video timing stage=select call=%llu pts=%lld mono_us=%lld audio_pts=%lld", + (unsigned long long)m->id, (long long)f.timing.pts_us, (long long)now, (long long)clock.pts_us); } + pictures.pop_front(); + continue; } - if (now - report >= 1000000) { - DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "video playback shown=%u queue=%zu audio_valid=%d audio_pts=%lld video_pts=%lld", - shown, pictures.size(), clock.valid, (long long)clock.pts_us, (long long)lastShown); report = now; + // VIDEO_OFF пока отсутствует в протоколе: убираем застывшую картинку после прекращения показа. + if (lastPictureAt && now - lastPictureAt >= 5000000) { + publishImage({}, -1); lastPictureAt = 0; + DEBUG_INFO(DEBUG_CATEGORY_CALL, "video playback hidden: no playable frame for 5s"); } - idle(); + std::this_thread::sleep_for(std::chrono::microseconds(std::max(100, std::min(5000, playout.deadline_us - nowUs())))); } } diff --git a/tools/chatgui/src/call_video_engine.h b/tools/chatgui/src/call_video_engine.h index babbee74..b2019df2 100644 --- a/tools/chatgui/src/call_video_engine.h +++ b/tools/chatgui/src/call_video_engine.h @@ -25,4 +25,5 @@ private: void closeCamera(); void transmit(); void receive(); + void publishImage(const QImage& image, int64_t pts); }; diff --git a/tools/chatgui/tests/test_call_video_clock.cpp b/tools/chatgui/tests/test_call_video_clock.cpp index 4882a81a..c89062de 100644 --- a/tools/chatgui/tests/test_call_video_clock.cpp +++ b/tools/chatgui/tests/test_call_video_clock.cpp @@ -32,6 +32,7 @@ extern "C" int __wrap_call_video_port_pull(uint64_t id, call_video_frame* out) { if (cursor >= (int)frames.size() || nowUs() - start < cursor * 33333ll) return 0; const auto& f = frames[cursor]; *out = {}; out->call_id = id; out->format = format; out->pts_us = f.pts; out->frame_id = ++cursor; out->flags = f.key ? CALL_VIDEO_KEY : 0; + out->received_us = start + (cursor - 1) * 33333ll; out->size = out->length = f.data.size(); auto* data = static_cast(u_malloc(out->size)); memcpy(data, f.data.constData(), out->size); out->data = data; return 1; }