From 5f772fa97f672f7f15a4ea31539abc329dc01d55 Mon Sep 17 00:00:00 2001 From: evgeny Date: Mon, 19 Oct 2026 17:28:13 +0200 Subject: [PATCH] Drain GPU outputs independently of capture and verify lossless recovery --- tests/test_call_video_playout.c | 6 ++--- tools/chatgui/src/live_video_encoder.cpp | 27 +++++++++++++------ tools/chatgui/src/live_video_encoder.h | 4 ++- tools/chatgui/tests/test_call_video_clock.cpp | 10 ++++--- 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/tests/test_call_video_playout.c b/tests/test_call_video_playout.c index 81ce045e..dbcec0f4 100644 --- a/tests/test_call_video_playout.c +++ b/tests/test_call_video_playout.c @@ -8,7 +8,7 @@ static const int64_t origin = 1000000; static void simulate(unsigned pause_ms, int cpu, int64_t quantum, int accelerated_audio) { struct call_video_playout p; call_video_playout_init(&p, 7); - enum { COUNT = 240 }; + enum { COUNT = 480 }; struct call_video_playout_frame frames[COUNT]; for (int i = 0; i < COUNT; ++i) { int64_t arrival = origin + i * 33333ll; @@ -17,8 +17,8 @@ static void simulate(unsigned pause_ms, int cpu, int64_t quantum, int accelerate frames[i] = (struct call_video_playout_frame){origin + i * 33333ll, arrival}; } unsigned shown = 0, accelerated = 0; - int64_t last_show = 0, recovered_lag = INT64_MAX, audio_pts = origin; - for (int64_t now = origin; now < origin + 12000000 && shown < COUNT; now += quantum) { + int64_t last_show = 0, recovered_lag = INT64_MAX, audio_pts = origin - (accelerated_audio ? 150000 : 0); + for (int64_t now = origin; now < origin + 20000000 && shown < COUNT; now += quantum) { uint32_t rate = accelerated_audio && now >= origin + 1500000 && now < origin + 2500000 ? 1100000 : 1000000; if (now > origin) audio_pts += quantum * rate / 1000000; if (cpu && now >= origin + 1000000 && now < origin + 1000000 + pause_ms * 1000ll) continue; diff --git a/tools/chatgui/src/live_video_encoder.cpp b/tools/chatgui/src/live_video_encoder.cpp index 17afb330..f9f90af4 100644 --- a/tools/chatgui/src/live_video_encoder.cpp +++ b/tools/chatgui/src/live_video_encoder.cpp @@ -49,6 +49,7 @@ LiveVideoEncoder::~LiveVideoEncoder() = default; void LiveVideoEncoder::close() { m = std::make_unique(); } QString LiveVideoEncoder::error() const { return m->error; } QString LiveVideoEncoder::name() const { return m->name; } +unsigned LiveVideoEncoder::pendingFrames() const { return static_cast(m->timestamps.size()); } QString LiveVideoEncoder::codecName(int codec) { static const char* names[] = {"", "H.264", "H.265", "VP9", "AV1", "H.266"}; return codec >= 1 && codec <= 5 ? QString::fromLatin1(names[codec]) : QString(); @@ -373,19 +374,23 @@ bool LiveVideoEncoder::open(const LiveVideoConfig& config, Backend backend) { config.codec, int(backend), qPrintable(m->error)); return false; } -bool LiveVideoEncoder::encode(const QImage& image, uint64_t pts, bool key, QList& output) { +bool LiveVideoEncoder::encode(const QImage& image, uint64_t pts, bool key, QList& output, bool finish) { #ifdef HAVE_FFMPEG - if (!m->codec || !m->frame || image.isNull() || pts > INT64_MAX) { + if (!m->codec || !m->frame || pts > INT64_MAX || (finish && !image.isNull())) { m->error = "Invalid video frame"; DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop encode invalid frame/state"); return false; } QElapsedTimer timer; timer.start(); if (!m->reportTimer.isValid()) m->reportTimer.start(); const auto outputBase = output.size(); + int result = 0; + const bool constrained = m->config.profile != LiveVideoProfile::Automatic; + qint64 preparedUs = 0, submittedUs = 0; + if (!image.isNull()) { QImage canvas(m->config.width, m->config.height, QImage::Format_RGB32); canvas.fill(Qt::black); QSize fitted = image.size().scaled(canvas.size(), Qt::KeepAspectRatio); { QPainter painter(&canvas); painter.drawImage(QRect(QPoint((canvas.width() - fitted.width()) / 2, (canvas.height() - fitted.height()) / 2), fitted), image); } - int result = av_frame_make_writable(m->frame); + result = av_frame_make_writable(m->frame); m->scale = sws_getCachedContext(m->scale, canvas.width(), canvas.height(), AV_PIX_FMT_RGB32, m->frame->width, m->frame->height, static_cast(m->frame->format), SWS_FAST_BILINEAR, nullptr, nullptr, nullptr); if (!m->scale) result = AVERROR(ENOMEM); @@ -412,7 +417,6 @@ bool LiveVideoEncoder::encode(const QImage& image, uint64_t pts, bool key, QList qPrintable(m->name), (unsigned long long)pts, image.width(), image.height(), m->frame->width, m->frame->height, m->frame->format, minimum, maximum, (unsigned long long)(samples ? edges * 1000 / samples : 0)); } - bool constrained = m->config.profile != LiveVideoProfile::Automatic; if (constrained && m->timestamps.size() >= 8) { m->error = "GPU encoder exceeded the bounded input pipeline"; DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "call encoder stalled name=%s pending=%zu pts=%llu", @@ -429,14 +433,15 @@ bool LiveVideoEncoder::encode(const QImage& image, uint64_t pts, bool key, QList if (result >= 0) result = av_hwframe_transfer_data(hardware, m->frame, 0); if (result >= 0) result = av_frame_copy_props(hardware, m->frame); } - const qint64 preparedUs = timer.nsecsElapsed() / 1000; + preparedUs = timer.nsecsElapsed() / 1000; if (result >= 0) result = avcodec_send_frame(m->codec, hardware ? hardware : m->frame); - const qint64 submittedUs = timer.nsecsElapsed() / 1000; + submittedUs = timer.nsecsElapsed() / 1000; if (result >= 0) { m->submittedFrames++; if (constrained) m->timestamps.emplace_back(m->frame->pts, pts); } av_frame_free(&hardware); + } else if (finish) result = avcodec_send_frame(m->codec, nullptr); AVPacket* packet = av_packet_alloc(); if (!packet) result = AVERROR(ENOMEM); while (result >= 0) { @@ -505,7 +510,8 @@ bool LiveVideoEncoder::encode(const QImage& image, uint64_t pts, bool key, QList if (result == AVERROR(EAGAIN) || result == AVERROR_EOF) { // Сводка по этапам отличает конвертацию, синхронный encoder и получение/упаковку пакета. const qint64 totalUs = timer.nsecsElapsed() / 1000; - ++m->inputs; m->outputs += output.size() - outputBase; + if (!image.isNull()) ++m->inputs; + m->outputs += output.size() - outputBase; bool producedKey = false; for (auto i = outputBase; i < output.size(); ++i) if (output[i].key) { ++m->keys; producedKey = true; } m->prepareMax = std::max(m->prepareMax, preparedUs); @@ -535,13 +541,18 @@ bool LiveVideoEncoder::encode(const QImage& image, uint64_t pts, bool key, QList } if (key && output.isEmpty()) DEBUG_DEBUG(DEBUG_CATEGORY_VIDEO, "desktop requested KEY buffered name=%s PTS=%llu", qPrintable(m->name), (unsigned long long)pts); + if (finish && !m->timestamps.empty()) { + m->error = "Encoder did not drain all accepted inputs"; + DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "call encoder drain incomplete name=%s pending=%zu", + qPrintable(m->name), m->timestamps.size()); return false; + } return true; } if (result < 0) { m->error = ffError(result); DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop encode failed name=%s PTS=%llu: %s", qPrintable(m->name), (unsigned long long)pts, qPrintable(m->error)); return false; } return true; #else - Q_UNUSED(image); Q_UNUSED(pts); Q_UNUSED(key); Q_UNUSED(output); return false; + Q_UNUSED(image); Q_UNUSED(pts); Q_UNUSED(key); Q_UNUSED(output); Q_UNUSED(finish); return false; #endif } diff --git a/tools/chatgui/src/live_video_encoder.h b/tools/chatgui/src/live_video_encoder.h index c5703947..20520f10 100644 --- a/tools/chatgui/src/live_video_encoder.h +++ b/tools/chatgui/src/live_video_encoder.h @@ -32,7 +32,9 @@ public: bool canSetMaxQp() const; // Поддерживается изменение потолка без пересоздания encoder. bool setMaxQp(int qp); // 0..51, применяется при следующей подаче кадра; false — ошибка/нет поддержки. void close(); - bool encode(const QImage& image, uint64_t pts, bool key, QList& output); + // Пустое image забирает готовый выход без нового input; finish завершает старый поток перед заменой. + bool encode(const QImage& image, uint64_t pts, bool key, QList& output, bool finish = false); + unsigned pendingFrames() const; QString error() const; QString name() const; static QList available(Backend backend = Backend::Automatic); // Установленные кодеки, без открытия и кодирования. diff --git a/tools/chatgui/tests/test_call_video_clock.cpp b/tools/chatgui/tests/test_call_video_clock.cpp index e9608f6f..014f06f4 100644 --- a/tools/chatgui/tests/test_call_video_clock.cpp +++ b/tools/chatgui/tests/test_call_video_clock.cpp @@ -51,14 +51,16 @@ int main(int argc, char** argv) { if (!encoder.encode(image, 1000000 + i * 33333ll, i == 0, output, completion) || output.size() != 1) return 3; frames.push_back(output.front()); } - CallVideoEngine engine; std::vector published; - QObject::connect(&engine, &CallVideoEngine::imageChanged, [&](const QImage& frame) { + CallVideoEngine engine; engine.setPresentationDriven(true); std::vector published; + QObject::connect(&engine, &CallVideoEngine::imageChanged, [&](const QImage& frame, quint64 serial) { if (!frame.isNull()) published.push_back(nowUs()); + // Подтверждение потребления отдельно от доставки signal, как paintEvent окна. + QTimer::singleShot(5, &engine, [&engine, serial] { engine.imagePresented(serial); }); }); for (int scenario = 0; scenario < 2; ++scenario) { disturbed = scenario != 0; cursor = 0; published.clear(); start = nowUs(); engine.start(7); QElapsedTimer timer; timer.start(); bool uiPaused = false; - while (timer.elapsed() < 3200) { + while (timer.elapsed() < 4000) { if (disturbed && !uiPaused && timer.elapsed() >= 1500) { QThread::msleep(60); uiPaused = true; } app.processEvents(); QThread::msleep(2); } @@ -71,7 +73,7 @@ int main(int argc, char** argv) { "gap_p50_us=%lld gap_p95_us=%lld gap_max_us=%lld audio_epochs=32", disturbed, published.size(), requests.load(), rejects.load(), (long long)(gaps.empty() ? 0 : gaps[gaps.size() / 2]), (long long)(gaps.empty() ? 0 : gaps[gaps.size() * 95 / 100]), (long long)(gaps.empty() ? 0 : gaps.back())); - if (published.size() < (disturbed ? 82u : 85u) || requests || rejects) return 1; + if (published.size() != frames.size() || requests || rejects) return 1; } return 0; }