From 97e20d2f1cb0543ad42591a0a021d832d7fe118a Mon Sep 17 00:00:00 2001 From: evgeny Date: Fri, 16 Oct 2026 18:04:27 +0200 Subject: [PATCH] Keep Qt video frame conversion off the publisher worker --- doc/service_lifecycle.md | 6 +- tools/chatgui/src/stream_publish_engine.cpp | 91 ++++++++++++--------- 2 files changed, 58 insertions(+), 39 deletions(-) diff --git a/doc/service_lifecycle.md b/doc/service_lifecycle.md index 17802ffe..82ce6086 100644 --- a/doc/service_lifecycle.md +++ b/doc/service_lifecycle.md @@ -144,8 +144,10 @@ stats timer и stream handle освобождаются до контекста Desktop `MainWindow` владеет `StreamPublishEngine`, а окно управления лишь отображает его состояние. Закрытие окна сохраняет эфир, новый запуск запрещён до конца FIN drain. -Qt capture callback хранит только последний кадр, video/audio workers выполняют encode/DSP -вне GUI и hardware callbacks. Отдельные поколения источника и encoder позволяют менять +Qt capture callback конвертирует кадр в `QImage` до передачи worker и сохраняет последний +`QVideoFrame` отдельно для preview. Конвертация и освобождение media frame не выполняются +в короткоживущем video worker, поскольку Qt multimedia TLS может задержать его exit на Windows. +Video/audio workers выполняют encode/DSP вне GUI и hardware callbacks. Отдельные поколения источника и encoder позволяют менять видеокодек/bitrate/rotation без повторного выбора экрана в портале. Ошибки Qt capture обрабатываются через queued callback, чтобы не уничтожать backend из собственного callback. Stop запрещает producers, закрывает Qt capture и подписку на аудиозахват. Отдельный join worker diff --git a/tools/chatgui/src/stream_publish_engine.cpp b/tools/chatgui/src/stream_publish_engine.cpp index 66a9e446..e46f6007 100644 --- a/tools/chatgui/src/stream_publish_engine.cpp +++ b/tools/chatgui/src/stream_publish_engine.cpp @@ -102,9 +102,10 @@ struct StreamPublishEngine::Impl { std::unique_ptr cursor; #endif std::unique_ptr window; - QVideoFrame frame; + QImage frame; QVideoFrame previewFrame; QVideoFrame presentedPreview; + int64_t lastImageCaptureAt = 0; #endif }; @@ -231,30 +232,58 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) { #ifdef HAVE_STREAM_CAPTURE uint64_t generation; { std::lock_guard guard(m->lock); generation = ++m->sourceGeneration; - m->backendOrigin = m->backendAnchor = -1; m->capturedFrames = 0; } + m->backendOrigin = m->backendAnchor = -1; m->capturedFrames = 0; m->lastImageCaptureAt = 0; } m->session = std::make_unique(); m->sink = std::make_unique(this); m->session->setVideoSink(m->sink.get()); connect(m->sink.get(), &QVideoSink::videoFrameChanged, this, [this, generation](const QVideoFrame& frame) { if (!frame.isValid()) { DEBUG_TRACE(DEBUG_CATEGORY_VIDEO, "desktop capture sink invalid frame generation=%llu", (unsigned long long)generation); return; } if (!m->running) return; - std::lock_guard guard(m->lock); - if (generation != m->sourceGeneration) return; - if (!m->capturedFrames++) DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop capture first raw frame kind=%d size=%dx%d pixel=%d PTS=%lld", - m->wanted.source.kind, frame.width(), frame.height(), frame.pixelFormat(), (long long)frame.startTime()); - int64_t now = monotonicUs(), capture = frame.startTime(); - if (capture >= 0) { - if (m->backendOrigin < 0) { - m->backendOrigin = capture; m->backendAnchor = now; - DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop capture timing generation=%llu source_PTS=%lld anchor=%lld calibration=arrival-estimate", - (unsigned long long)generation, (long long)capture, (long long)now); + int64_t callbackAt = monotonicUs(), capture = frame.startTime(), framePts = 0; + bool convert = false; + { + std::lock_guard guard(m->lock); + if (generation != m->sourceGeneration) return; + if (!m->capturedFrames++) DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop capture first raw frame kind=%d size=%dx%d pixel=%d PTS=%lld", + m->wanted.source.kind, frame.width(), frame.height(), frame.pixelFormat(), (long long)capture); + if (capture >= 0) { + if (m->backendOrigin < 0) { + m->backendOrigin = capture; m->backendAnchor = callbackAt; + DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop capture timing generation=%llu source_PTS=%lld anchor=%lld calibration=arrival-estimate", + (unsigned long long)generation, (long long)capture, (long long)callbackAt); + } + callbackAt = m->backendAnchor + capture - m->backendOrigin; + } else if (m->backendAnchor < 0) { + m->backendAnchor = callbackAt; + DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "desktop capture has no source PTS; using monotonic callback estimate"); } - now = m->backendAnchor + capture - m->backendOrigin; - } else if (m->backendAnchor < 0) { - m->backendAnchor = now; - DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "desktop capture has no source PTS; using monotonic callback estimate"); + int fps = std::clamp(m->wanted.video.fps, 1, 60); + convert = !m->lastImageCaptureAt || callbackAt - m->lastImageCaptureAt >= 1000000 / fps; + m->previewFrame = frame; + if (!convert) { m->wake.notify_all(); return; } + m->lastImageCaptureAt = callbackAt; + if (m->framePts < 0 && m->wanted.source.id == "portal") m->started = monotonicUs(); + framePts = std::max(1, callbackAt - m->origin); + } + /* QVideoFrame conversion stays on Qt's persistent multimedia callback thread. + * Qt worker-thread storage was preventing the short-lived publisher thread from exiting on Windows. */ + QImage image = frame.toImage(); + if (image.isNull()) { + DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop captured frame conversion failed generation=%llu format=%d size=%dx%d", + (unsigned long long)generation, frame.pixelFormat(), frame.width(), frame.height()); + QMetaObject::invokeMethod(this, [this, generation] { sourceFailed(generation, text("Video frame conversion failed")); }); + return; + } +#if QT_VERSION >= QT_VERSION_CHECK(6, 7, 0) + if (frame.rotation() != QtVideo::Rotation::None) image = image.transformed(QTransform().rotate(static_cast(frame.rotation()))); + if (frame.mirrored()) image = image.transformed(QTransform().scale(-1, 1)); +#else + if (frame.rotationAngle()) image = image.transformed(QTransform().rotate(frame.rotationAngle())); +#endif + { + std::lock_guard guard(m->lock); + if (!m->running || generation != m->sourceGeneration) return; + m->frame = std::move(image); m->framePts = framePts; m->frameGeneration = m->generation; } - if (m->framePts < 0 && m->wanted.source.id == "portal") m->started = monotonicUs(); - m->frame = m->previewFrame = frame; m->framePts = std::max(1, now - m->origin); m->frameGeneration = m->generation; m->wake.notify_all(); }, Qt::DirectConnection); auto failed = [this, generation](const QString& error) { @@ -462,7 +491,7 @@ bool StreamPublishEngine::reconfigure(const StreamPublishConfig& config) { && (config.video.width != previous.video.width || config.video.height != previous.video.height || config.video.fps != previous.video.fps)); { std::lock_guard guard(m->lock); m->wanted = config; m->generation++; m->started = monotonicUs(); m->softwareRequest = 0; #ifdef HAVE_STREAM_CAPTURE - if (!sourceChange) { m->frame = m->previewFrame; m->frameGeneration = m->generation; } + if (!sourceChange) m->frameGeneration = m->generation; #endif } if (sourceChange && !openSource(config)) { sourceFailed(m->generation, text("Could not open the selected video source")); return false; } @@ -654,17 +683,17 @@ void StreamPublishEngine::videoLoop(StreamPublishConfig initial) { int64_t lastPts = -1, lastKey = -1000000, next = 0, report = 0; unsigned count = 0, skipped = 0; while (m->running) { m->videoProgress("waiting-frame"); - StreamPublishConfig config; uint64_t generation; QVideoFrame frame; int64_t capture; + StreamPublishConfig config; uint64_t generation; QImage frame; int64_t capture; { std::unique_lock lock(m->lock); config = m->wanted; generation = m->generation; if (config.audio && !m->lastAudioAt.load(std::memory_order_acquire)) { m->wake.wait_for(lock, std::chrono::milliseconds(5)); continue; } int64_t now = monotonicUs(); if (next > now) { m->wake.wait_for(lock, std::chrono::microseconds(next - now)); continue; } - if (m->frameGeneration == generation) { frame = m->frame; capture = m->framePts; m->frame = {}; } + if (m->frameGeneration == generation) { frame = std::move(m->frame); capture = m->framePts; } else capture = -1; } // Размер кадра точнее QScreen при HiDPI и единственный доступный после выбора Wayland portal. - if (config.source.kind == StreamVideoSource::Screen && frame.isValid() && config.source.size != frame.size()) { + if (config.source.kind == StreamVideoSource::Screen && !frame.isNull() && config.source.size != frame.size()) { QSize size = stream_video_screen_size(frame.size(), config.nativeSize ? 4096 : std::min(config.video.width, config.video.height)); if (size.isEmpty()) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop screen invalid captured size=%dx%d", frame.width(), frame.height()); @@ -685,7 +714,7 @@ void StreamPublishEngine::videoLoop(StreamPublishConfig initial) { m->wake.wait_for(lock, std::chrono::milliseconds(100), [this, generation] { return !m->running || m->generation != generation; }); continue; } - if (config.source.kind == StreamVideoSource::Screen && config.source.size.isEmpty() && !frame.isValid()) { + if (config.source.kind == StreamVideoSource::Screen && config.source.size.isEmpty() && frame.isNull()) { std::unique_lock lock(m->lock); m->wake.wait_for(lock, std::chrono::milliseconds(5)); continue; } if (generation != current) { @@ -748,22 +777,10 @@ void StreamPublishEngine::videoLoop(StreamPublishConfig initial) { } } bool screen = config.source.kind != StreamVideoSource::Camera; - if (frame.isValid()) { + if (!frame.isNull()) { if (!screen && (capture <= lastPts || monotonicUs() - m->origin - capture > 250000)) { skipped++; continue; } - m->videoProgress("convert-frame-to-image"); - lastImage = frame.toImage(); + lastImage = std::move(frame); m->convertedFrames++; - if (lastImage.isNull()) { - DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop captured frame conversion failed format=%d", frame.pixelFormat()); - QMetaObject::invokeMethod(this, [this, generation] { sourceFailed(generation, text("Video frame conversion failed")); }); - continue; - } -#if QT_VERSION >= QT_VERSION_CHECK(6, 7, 0) - if (frame.rotation() != QtVideo::Rotation::None) lastImage = lastImage.transformed(QTransform().rotate(static_cast(frame.rotation()))); - if (frame.mirrored()) lastImage = lastImage.transformed(QTransform().scale(-1, 1)); -#else - if (frame.rotationAngle()) lastImage = lastImage.transformed(QTransform().rotate(frame.rotationAngle())); -#endif } else if (!screen || lastImage.isNull()) { std::unique_lock lock(m->lock); m->wake.wait_for(lock, std::chrono::milliseconds(5)); continue; } else m->repeatedFrames++;