Observe capture activation raw frames and encoder generations

tools/chatgui/src/stream_publish_engine.cpp

@@ -45,6 +45,8 @@ struct StreamPublishEngine::Impl {
     uint64_t id = 0, group = 0, generation = 0, sourceGeneration = 0, confirmedGeneration = 0, frameGeneration = 0, audioSequence = 0;
     int64_t origin = 0, started = 0, framePts = -1, backendOrigin = -1, backendAnchor = -1;
     int channels = 2;
+    uint64_t capturedFrames = 0;
+    int64_t captureReport = 0;
     bool probeReady = false, probeDelivered = false, hasGood = false, rollingBack = false;
     StreamPublishConfig wanted, good;
     QList<int> codecs;
@@ -116,9 +118,12 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) {
     m->session = std::make_unique<QMediaCaptureSession>(); m->sink = std::make_unique<QVideoSink>();
     m->session->setVideoSink(m->sink.get());
     connect(m->sink.get(), &QVideoSink::videoFrameChanged, this, [this, generation](const QVideoFrame& frame) {
-        if (!m->running || !frame.isValid()) return;
+        if (!frame.isValid()) { DEBUG_TRACE(DEBUG_CATEGORY_VIDEO, "desktop capture sink invalid frame generation=%llu", (unsigned long long)generation); return; }
+        if (!m->running) return;
         std::lock_guard<std::mutex> guard(m->lock);
         if (generation != m->sourceGeneration) return;
+        if (!m->capturedFrames++) DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop capture first raw frame kind=%d size=%dx%d pixel=%d PTS=%lld",
+            m->wanted.source.kind, frame.width(), frame.height(), frame.pixelFormat(), (long long)frame.startTime());
         int64_t now = monotonicUs(), capture = frame.startTime();
         if (capture >= 0) {
             if (m->backendOrigin < 0) {
@@ -152,6 +157,9 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) {
         }
         if (best.isNull()) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop camera lacks requested fps=%d", config.video.fps); return false; }
         m->camera->setCameraFormat(best); m->session->setCamera(m->camera.get());
+        connect(m->camera.get(), &QCamera::activeChanged, this, [generation](bool active) {
+            DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop camera active=%d generation=%llu", active, (unsigned long long)generation);
+        });
         connect(m->camera.get(), &QCamera::errorOccurred, this, [failed](QCamera::Error, const QString& error) { failed(error); }, Qt::QueuedConnection);
         m->camera->start();
     } else if (config.source.kind == StreamVideoSource::Screen) {
@@ -163,12 +171,18 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) {
             m->screen->setScreen(selected);
         }
         m->session->setScreenCapture(m->screen.get());
+        connect(m->screen.get(), &QScreenCapture::activeChanged, this, [generation](bool active) {
+            DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop screen capture active=%d generation=%llu", active, (unsigned long long)generation);
+        });
         connect(m->screen.get(), &QScreenCapture::errorOccurred, this, [failed](QScreenCapture::Error, const QString& error) { failed(error); }, Qt::QueuedConnection);
         m->screen->start();
     } else {
         auto window = config.source.window.value<QCapturableWindow>();
         if (!window.isValid()) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop window is unavailable"); return false; }
         m->window = std::make_unique<QWindowCapture>(); m->window->setWindow(window); m->session->setWindowCapture(m->window.get());
+        connect(m->window.get(), &QWindowCapture::activeChanged, this, [generation](bool active) {
+            DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop window capture active=%d generation=%llu", active, (unsigned long long)generation);
+        });
         connect(m->window.get(), &QWindowCapture::errorOccurred, this, [failed](QWindowCapture::Error, const QString& error) { failed(error); }, Qt::QueuedConnection);
         m->window->start();
     }
@@ -460,6 +474,13 @@ void StreamPublishEngine::tick() {
           waitingSelection = m->wanted.source.id == "portal" && m->framePts < 0; }
         if (!confirmed && !waitingSelection && monotonicUs() - started > 8000000)
             sourceFailed(m->generation, text("Timed out waiting for the first video frame"));
+        { std::lock_guard<std::mutex> guard(m->lock);
+          if (monotonicUs() - m->captureReport >= 1000000) {
+              m->captureReport = monotonicUs();
+              DEBUG_DEBUG(DEBUG_CATEGORY_VIDEO, "desktop capture state source_generation=%llu video_generation=%llu confirmed=%llu raw_frames=%llu raw_PTS=%lld",
+                  (unsigned long long)m->sourceGeneration, (unsigned long long)m->generation, (unsigned long long)m->confirmedGeneration,
+                  (unsigned long long)m->capturedFrames, (long long)m->framePts);
+          } }
         if (m->wanted.audio && m_capture.needsRecovery()) { m_capture.close(); m_capture.retryLater(); }
         if (m->wanted.audio && m_capture.retryReady()) openMicrophone();
     }

tools/chatgui/tests/test_stream_publish.cpp

@@ -4,6 +4,7 @@
 #include <QTimer>
 #include <QLabel>
 #include <QScreen>
+#include <QLoggingCategory>
 #include <atomic>
 #include <thread>
 #include <cmath>
@@ -97,6 +98,7 @@ private slots:
         debug_config_init(); debug_set_level(DEBUG_LEVEL_WARN); debug_set_category_level(DEBUG_CATEGORY_STREAM, DEBUG_LEVEL_DEBUG);
         debug_set_category_level(DEBUG_CATEGORY_SYS, DEBUG_LEVEL_ERROR);
         debug_set_category_level(DEBUG_CATEGORY_VIDEO, DEBUG_LEVEL_DEBUG);
+        if (qEnvironmentVariableIsSet("UTUN_TEST_SCREEN_CAPTURE")) QLoggingCategory::setFilterRules("qt.multimedia.*=true");
         allocations = u_get_allocated_count(); ua = uasync_create(); QVERIFY(ua);
         inst = utun_instance_create_from_str(ua,
             "[global]\nauto_sockets=no\n[chatserver]\ngroup_autoconnect=0\nstorage_autoload=0\nmember_files_enabled=0\n");

tools/chatgui/transport/stream_bridge.cpp

@@ -159,7 +159,7 @@ void process(void*) {
     bridge.wantRead = false;
     stream_media_packet packet{};
     int result = stream_media_read(bridge.handle, bridge.position, bridge.videoAhead, bridge.audioAhead, &packet);
-    DEBUG_DEBUG(DEBUG_CATEGORY_STREAM, "desktop media read id=%llu PTS=%lld ahead=%lld/%lld result=%d kind=%d prepared=%u records=%zu",
+    DEBUG_TRACE(DEBUG_CATEGORY_STREAM, "desktop media read id=%llu PTS=%lld ahead=%lld/%lld result=%d kind=%d prepared=%u records=%zu",
         (unsigned long long)bridge.viewer.id, (long long)bridge.position, (long long)bridge.videoAhead,
         (long long)bridge.audioAhead, result, result == STREAM_OK ? (int)packet.kind : -1, prepared, bridge.output.size());
     if (stream_media_get_status(bridge.handle, &bridge.status) < 0) result = STREAM_ERR_FORMAT;