From bf729502a142c15c9cd163e55196ded73d5a0790 Mon Sep 17 00:00:00 2001 From: evgeny Date: Sat, 17 Oct 2026 00:38:29 +0200 Subject: [PATCH] Improve Windows screen capture frame rate --- tools/chatgui/src/stream_publish_engine.cpp | 60 +++++++++++++++------ 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/tools/chatgui/src/stream_publish_engine.cpp b/tools/chatgui/src/stream_publish_engine.cpp index 8440b2ca..a8a6ce9a 100644 --- a/tools/chatgui/src/stream_publish_engine.cpp +++ b/tools/chatgui/src/stream_publish_engine.cpp @@ -76,7 +76,7 @@ struct StreamPublishEngine::Impl { QVideoFrame imageInput; uint64_t imageGeneration = 0; int64_t imagePts = -1; - bool imagePending = false, imageStop = false; + bool imagePending = false, imageStop = false, imageDrawCursor = false; uint64_t id = 0, group = 0, generation = 0, sourceGeneration = 0, confirmedGeneration = 0, frameGeneration = 0, audioSequence = 0; int64_t origin = 0, started = 0, framePts = -1, backendOrigin = -1, backendAnchor = -1; int64_t audioAnchor = -1; @@ -85,6 +85,7 @@ struct StreamPublishEngine::Impl { uint64_t capturedFrames = 0; uint64_t convertedCaptureFrames = 0; int64_t captureReport = 0; + uint64_t captureReportRaw = 0, captureReportConverted = 0, captureReportEncoded = 0, captureReportDelivered = 0; std::atomic videoStage{"stopped"}; std::atomic videoStageAt{0}; std::atomic imageStage{"starting"}; @@ -114,7 +115,7 @@ struct StreamPublishEngine::Impl { #endif std::unique_ptr screenGrabTimer; #if defined(UTUN_STREAM_X11_CURSOR) || defined(UTUN_STREAM_WIN_CURSOR) - std::unique_ptr cursor; + std::shared_ptr cursor; #endif std::unique_ptr window; QImage frame; @@ -326,7 +327,8 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) { { std::lock_guard guard(m->lock); if (!m->running || generation != m->sourceGeneration) return; - m->imageInput = frame; m->imageGeneration = generation; m->imagePts = framePts; m->imagePending = true; + m->imageInput = frame; m->imageGeneration = generation; m->imagePts = framePts; + m->imageDrawCursor = m->wanted.showCursor && cursorSupported(m->wanted.source); m->imagePending = true; } m->imageWake.notify_one(); }, Qt::DirectConnection); @@ -375,9 +377,9 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) { #endif // QScreen даёт выбранный монитор без курсора; накладываем его один раз до preview и кодирования. // Для X11 это также исключает путаницу Qt Multimedia с одноимёнными root. - if (selected && (QGuiApplication::platformName() == "xcb" || cursorSupported(config.source))) { -#if defined(UTUN_STREAM_X11_CURSOR) || defined(UTUN_STREAM_WIN_CURSOR) - m->cursor = std::make_unique(selected); + if (selected && QGuiApplication::platformName() == "xcb") { +#if defined(UTUN_STREAM_X11_CURSOR) + { std::lock_guard guard(m->lock); m->cursor = std::make_shared(selected); } #endif m->screenGrabTimer = std::make_unique(); m->screenGrabTimer->setTimerType(Qt::PreciseTimer); connect(m->screenGrabTimer.get(), &QTimer::timeout, this, [this, generation, screen = QPointer(selected), failed] { @@ -417,6 +419,9 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) { qPrintable(QGuiApplication::platformName()), qPrintable(config.source.name), m->screenGrabTimer->interval(), config.showCursor, cursorSupported(config.source)); } else { +#ifdef UTUN_STREAM_WIN_CURSOR + if (selected) { std::lock_guard guard(m->lock); m->cursor = std::make_shared(selected); } +#endif m->screen = std::make_unique(); m->screen->setScreen(selected); m->session->setScreenCapture(m->screen.get()); @@ -426,6 +431,10 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) { }, Qt::QueuedConnection); connect(m->screen.get(), &QScreenCapture::errorOccurred, this, [failed](QScreenCapture::Error, const QString& error) { failed(error); }, Qt::QueuedConnection); m->screen->start(); +#ifdef UTUN_STREAM_WIN_CURSOR + DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop Windows screen capture backend=QtMultimedia cursor_overlay=frame_worker source=%s", + qPrintable(config.source.name)); +#endif } } else { auto window = config.source.window.value(); @@ -451,13 +460,13 @@ void StreamPublishEngine::closeSource() { std::lock_guard guard(m->lock); ++m->sourceGeneration; m->imagePending = false; m->imageInput = {}; m->frame = {}; m->latestImage = {}; m->previewFrame = {}; m->presentedPreview = {}; m->framePts = -1; +#if defined(UTUN_STREAM_X11_CURSOR) || defined(UTUN_STREAM_WIN_CURSOR) + m->cursor.reset(); +#endif } m->screenGrabTimer.reset(); #ifdef UTUN_STREAM_PORTAL m->portal.reset(); -#endif -#if defined(UTUN_STREAM_X11_CURSOR) || defined(UTUN_STREAM_WIN_CURSOR) - m->cursor.reset(); #endif emit previewVideoChanged(QVariant()); if (m->sink) disconnect(m->sink.get(), nullptr, this, nullptr); @@ -472,13 +481,14 @@ void StreamPublishEngine::closeSource() { void StreamPublishEngine::imageLoop() { #ifdef HAVE_STREAM_CAPTURE for (;;) { - QVideoFrame frame; uint64_t generation; int64_t pts; + QVideoFrame frame; uint64_t generation; int64_t pts; bool drawCursor; m->imageProgress("wait-frame"); { std::unique_lock lock(m->lock); m->imageWake.wait(lock, [this] { return m->imageStop || m->imagePending; }); if (m->imageStop) return; - frame = std::move(m->imageInput); generation = m->imageGeneration; pts = m->imagePts; m->imagePending = false; + frame = std::move(m->imageInput); generation = m->imageGeneration; pts = m->imagePts; + drawCursor = m->imageDrawCursor; m->imagePending = false; } int64_t convertAt = monotonicUs(); m->imageProgress("convert-frame"); @@ -493,6 +503,14 @@ void StreamPublishEngine::imageLoop() { QMetaObject::invokeMethod(this, [this, generation] { sourceFailed(generation, text("Video frame conversion failed")); }); continue; } +#ifdef UTUN_STREAM_WIN_CURSOR + std::shared_ptr cursor; + { std::lock_guard guard(m->lock); cursor = m->cursor; } + if (drawCursor && cursor && !cursor->draw(image)) { + QMetaObject::invokeMethod(this, [this, generation] { sourceFailed(generation, text("Could not capture the mouse cursor")); }); + continue; + } +#endif #if QT_VERSION >= QT_VERSION_CHECK(6, 7, 0) if (frame.rotation() != QtVideo::Rotation::None) image = image.transformed(QTransform().rotate(static_cast(frame.rotation()))); if (frame.mirrored()) image = image.transformed(QTransform().scale(-1, 1)); @@ -534,6 +552,7 @@ bool StreamPublishEngine::start(uint64_t group, const StreamPublishConfig& confi m->origin = m->started = monotonicUs(); m->stats = {}; } m->convertedFrames = m->encodedFrames = m->deliveredFrames = m->repeatedFrames = 0; m->videoProgress("starting"); m->captureReport = 0; + m->captureReportRaw = m->captureReportConverted = m->captureReportEncoded = m->captureReportDelivered = 0; m->id = QRandomGenerator::global()->generate64(); if (!m->id) m->id = 1; m->group = group; m->audio.reset(); m->audioSequence = 0; m->muted = false; stream_media_format format{(uint16_t)config.video.width, (uint16_t)config.video.height, (uint16_t)config.video.rotation, @@ -983,13 +1002,22 @@ void StreamPublishEngine::tick() { sourceFailed(m->generation, text("Timed out waiting for the first video frame")); { std::lock_guard guard(m->lock); if (monotonicUs() - m->captureReport >= 5000000) { - m->captureReport = monotonicUs(); + int64_t reportAt = monotonicUs(); + double elapsed = m->captureReport ? (reportAt - m->captureReport) / 1000000.0 : 0.0; + double rawFps = elapsed > 0 ? (m->capturedFrames - m->captureReportRaw) / elapsed : 0.0; + double convertedFps = elapsed > 0 ? (m->convertedFrames.load() - m->captureReportConverted) / elapsed : 0.0; + double encodedFps = elapsed > 0 ? (m->encodedFrames.load() - m->captureReportEncoded) / elapsed : 0.0; + double deliveredFps = elapsed > 0 ? (m->deliveredFrames.load() - m->captureReportDelivered) / elapsed : 0.0; + m->captureReport = reportAt; m->captureReportRaw = m->capturedFrames; + m->captureReportConverted = m->convertedFrames.load(); m->captureReportEncoded = m->encodedFrames.load(); + m->captureReportDelivered = m->deliveredFrames.load(); DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop capture progress source_generation=%llu video_generation=%llu confirmed=%llu raw_frames=%llu raw_PTS=%lld " - "converted=%llu encoded=%llu delivered=%llu repeated=%llu stage=%s stage_age_ms=%lld", + "raw_fps=%.1f converted=%llu converted_fps=%.1f encoded=%llu encoded_fps=%.1f delivered=%llu delivered_fps=%.1f " + "repeated=%llu stage=%s stage_age_ms=%lld", (unsigned long long)m->sourceGeneration, (unsigned long long)m->generation, (unsigned long long)m->confirmedGeneration, - (unsigned long long)m->capturedFrames, (long long)m->framePts, - (unsigned long long)m->convertedFrames.load(), (unsigned long long)m->encodedFrames.load(), - (unsigned long long)m->deliveredFrames.load(), (unsigned long long)m->repeatedFrames.load(), + (unsigned long long)m->capturedFrames, (long long)m->framePts, rawFps, + (unsigned long long)m->convertedFrames.load(), convertedFps, (unsigned long long)m->encodedFrames.load(), encodedFps, + (unsigned long long)m->deliveredFrames.load(), deliveredFps, (unsigned long long)m->repeatedFrames.load(), m->videoStage.load(), (long long)((monotonicUs() - m->videoStageAt.load()) / 1000)); } } }