From 97c460e2f1fb9d5bac3947fc9f13f825b7c72fab Mon Sep 17 00:00:00 2001 From: evgeny Date: Wed, 14 Oct 2026 11:44:11 +0200 Subject: [PATCH] Observe capture activation raw frames and encoder generations --- tools/chatgui/src/stream_publish_engine.cpp | 23 ++++++++++++++++++++- tools/chatgui/tests/test_stream_publish.cpp | 2 ++ tools/chatgui/transport/stream_bridge.cpp | 2 +- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/tools/chatgui/src/stream_publish_engine.cpp b/tools/chatgui/src/stream_publish_engine.cpp index 4ac33dd7..f04fe10f 100644 --- a/tools/chatgui/src/stream_publish_engine.cpp +++ b/tools/chatgui/src/stream_publish_engine.cpp @@ -45,6 +45,8 @@ struct StreamPublishEngine::Impl { uint64_t id = 0, group = 0, generation = 0, sourceGeneration = 0, confirmedGeneration = 0, frameGeneration = 0, audioSequence = 0; int64_t origin = 0, started = 0, framePts = -1, backendOrigin = -1, backendAnchor = -1; int channels = 2; + uint64_t capturedFrames = 0; + int64_t captureReport = 0; bool probeReady = false, probeDelivered = false, hasGood = false, rollingBack = false; StreamPublishConfig wanted, good; QList codecs; @@ -116,9 +118,12 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) { m->session = std::make_unique(); m->sink = std::make_unique(); m->session->setVideoSink(m->sink.get()); connect(m->sink.get(), &QVideoSink::videoFrameChanged, this, [this, generation](const QVideoFrame& frame) { - if (!m->running || !frame.isValid()) return; + if (!frame.isValid()) { DEBUG_TRACE(DEBUG_CATEGORY_VIDEO, "desktop capture sink invalid frame generation=%llu", (unsigned long long)generation); return; } + if (!m->running) return; std::lock_guard guard(m->lock); if (generation != m->sourceGeneration) return; + if (!m->capturedFrames++) DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop capture first raw frame kind=%d size=%dx%d pixel=%d PTS=%lld", + m->wanted.source.kind, frame.width(), frame.height(), frame.pixelFormat(), (long long)frame.startTime()); int64_t now = monotonicUs(), capture = frame.startTime(); if (capture >= 0) { if (m->backendOrigin < 0) { @@ -152,6 +157,9 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) { } if (best.isNull()) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop camera lacks requested fps=%d", config.video.fps); return false; } m->camera->setCameraFormat(best); m->session->setCamera(m->camera.get()); + connect(m->camera.get(), &QCamera::activeChanged, this, [generation](bool active) { + DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop camera active=%d generation=%llu", active, (unsigned long long)generation); + }); connect(m->camera.get(), &QCamera::errorOccurred, this, [failed](QCamera::Error, const QString& error) { failed(error); }, Qt::QueuedConnection); m->camera->start(); } else if (config.source.kind == StreamVideoSource::Screen) { @@ -163,12 +171,18 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) { m->screen->setScreen(selected); } m->session->setScreenCapture(m->screen.get()); + connect(m->screen.get(), &QScreenCapture::activeChanged, this, [generation](bool active) { + DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop screen capture active=%d generation=%llu", active, (unsigned long long)generation); + }); connect(m->screen.get(), &QScreenCapture::errorOccurred, this, [failed](QScreenCapture::Error, const QString& error) { failed(error); }, Qt::QueuedConnection); m->screen->start(); } else { auto window = config.source.window.value(); if (!window.isValid()) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop window is unavailable"); return false; } m->window = std::make_unique(); m->window->setWindow(window); m->session->setWindowCapture(m->window.get()); + connect(m->window.get(), &QWindowCapture::activeChanged, this, [generation](bool active) { + DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop window capture active=%d generation=%llu", active, (unsigned long long)generation); + }); connect(m->window.get(), &QWindowCapture::errorOccurred, this, [failed](QWindowCapture::Error, const QString& error) { failed(error); }, Qt::QueuedConnection); m->window->start(); } @@ -460,6 +474,13 @@ void StreamPublishEngine::tick() { waitingSelection = m->wanted.source.id == "portal" && m->framePts < 0; } if (!confirmed && !waitingSelection && monotonicUs() - started > 8000000) sourceFailed(m->generation, text("Timed out waiting for the first video frame")); + { std::lock_guard guard(m->lock); + if (monotonicUs() - m->captureReport >= 1000000) { + m->captureReport = monotonicUs(); + DEBUG_DEBUG(DEBUG_CATEGORY_VIDEO, "desktop capture state source_generation=%llu video_generation=%llu confirmed=%llu raw_frames=%llu raw_PTS=%lld", + (unsigned long long)m->sourceGeneration, (unsigned long long)m->generation, (unsigned long long)m->confirmedGeneration, + (unsigned long long)m->capturedFrames, (long long)m->framePts); + } } if (m->wanted.audio && m_capture.needsRecovery()) { m_capture.close(); m_capture.retryLater(); } if (m->wanted.audio && m_capture.retryReady()) openMicrophone(); } diff --git a/tools/chatgui/tests/test_stream_publish.cpp b/tools/chatgui/tests/test_stream_publish.cpp index cceab5b3..b9bc119b 100644 --- a/tools/chatgui/tests/test_stream_publish.cpp +++ b/tools/chatgui/tests/test_stream_publish.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -97,6 +98,7 @@ private slots: debug_config_init(); debug_set_level(DEBUG_LEVEL_WARN); debug_set_category_level(DEBUG_CATEGORY_STREAM, DEBUG_LEVEL_DEBUG); debug_set_category_level(DEBUG_CATEGORY_SYS, DEBUG_LEVEL_ERROR); debug_set_category_level(DEBUG_CATEGORY_VIDEO, DEBUG_LEVEL_DEBUG); + if (qEnvironmentVariableIsSet("UTUN_TEST_SCREEN_CAPTURE")) QLoggingCategory::setFilterRules("qt.multimedia.*=true"); allocations = u_get_allocated_count(); ua = uasync_create(); QVERIFY(ua); inst = utun_instance_create_from_str(ua, "[global]\nauto_sockets=no\n[chatserver]\ngroup_autoconnect=0\nstorage_autoload=0\nmember_files_enabled=0\n"); diff --git a/tools/chatgui/transport/stream_bridge.cpp b/tools/chatgui/transport/stream_bridge.cpp index 2b426ddf..22f0f00d 100644 --- a/tools/chatgui/transport/stream_bridge.cpp +++ b/tools/chatgui/transport/stream_bridge.cpp @@ -159,7 +159,7 @@ void process(void*) { bridge.wantRead = false; stream_media_packet packet{}; int result = stream_media_read(bridge.handle, bridge.position, bridge.videoAhead, bridge.audioAhead, &packet); - DEBUG_DEBUG(DEBUG_CATEGORY_STREAM, "desktop media read id=%llu PTS=%lld ahead=%lld/%lld result=%d kind=%d prepared=%u records=%zu", + DEBUG_TRACE(DEBUG_CATEGORY_STREAM, "desktop media read id=%llu PTS=%lld ahead=%lld/%lld result=%d kind=%d prepared=%u records=%zu", (unsigned long long)bridge.viewer.id, (long long)bridge.position, (long long)bridge.videoAhead, (long long)bridge.audioAhead, result, result == STREAM_OK ? (int)packet.kind : -1, prepared, bridge.output.size()); if (stream_media_get_status(bridge.handle, &bridge.status) < 0) result = STREAM_ERR_FORMAT;