Keep raw preview and source permissions independent of video encoding

tools/chatgui/src/stream_publish_engine.cpp

@@ -40,7 +40,7 @@ struct StreamPublishEngine::Impl {
     std::atomic<bool> running{false}, muted{false};
     std::atomic<unsigned> audioDropped{0};
     AudioEventQueue<Audio, 32> audio;
-    uint64_t id = 0, group = 0, generation = 0, frameGeneration = 0, audioSequence = 0;
+    uint64_t id = 0, group = 0, generation = 0, sourceGeneration = 0, confirmedGeneration = 0, frameGeneration = 0, audioSequence = 0;
     int64_t origin = 0, started = 0, framePts = -1, backendOrigin = -1, backendAnchor = -1;
     int channels = 2;
     bool probeReady = false, probeDelivered = false, hasGood = false, rollingBack = false;
@@ -55,6 +55,7 @@ struct StreamPublishEngine::Impl {
     std::unique_ptr<QScreenCapture> screen;
     std::unique_ptr<QWindowCapture> window;
     QVideoFrame frame;
+    QVideoFrame previewFrame;
 #endif
 };
 
@@ -105,13 +106,13 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) {
     closeSource();
 #ifdef HAVE_STREAM_CAPTURE
     uint64_t generation;
-    { std::lock_guard<std::mutex> guard(m->lock); generation = m->generation; m->backendOrigin = m->backendAnchor = -1; }
+    { std::lock_guard<std::mutex> guard(m->lock); generation = ++m->sourceGeneration; m->backendOrigin = m->backendAnchor = -1; }
     m->session = std::make_unique<QMediaCaptureSession>(); m->sink = std::make_unique<QVideoSink>();
     m->session->setVideoSink(m->sink.get());
     connect(m->sink.get(), &QVideoSink::videoFrameChanged, this, [this, generation](const QVideoFrame& frame) {
         if (!m->running || !frame.isValid()) return;
         std::lock_guard<std::mutex> guard(m->lock);
-        if (generation != m->generation) return;
+        if (generation != m->sourceGeneration) return;
         int64_t now = monotonicUs(), capture = frame.startTime();
         if (capture >= 0) {
             if (m->backendOrigin < 0) {
@@ -124,9 +125,12 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) {
             m->backendAnchor = now;
             DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "desktop capture has no source PTS; using monotonic callback estimate");
         }
-        m->frame = frame; m->framePts = std::max<int64_t>(1, now - m->origin); m->frameGeneration = generation;
+        m->frame = m->previewFrame = frame; m->framePts = std::max<int64_t>(1, now - m->origin); m->frameGeneration = m->generation;
         m->wake.notify_all();
-    });
+    }, Qt::DirectConnection);
+    auto failed = [this, generation](const QString& error) {
+        if (generation == m->sourceGeneration) sourceFailed(m->generation, error);
+    };
     if (config.source.kind == StreamVideoSource::Camera) {
         QCameraDevice selected;
         for (const auto& camera : QMediaDevices::videoInputs()) if (camera.id() == config.source.id) selected = camera;
@@ -141,7 +145,7 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) {
         }
         if (best.isNull()) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop camera lacks requested fps=%d", config.video.fps); return false; }
         m->camera->setCameraFormat(best); m->session->setCamera(m->camera.get());
-        connect(m->camera.get(), &QCamera::errorOccurred, this, [this, generation](QCamera::Error, const QString& error) { sourceFailed(generation, error); });
+        connect(m->camera.get(), &QCamera::errorOccurred, this, [failed](QCamera::Error, const QString& error) { failed(error); });
         m->camera->start();
     } else if (config.source.kind == StreamVideoSource::Screen) {
         m->screen = std::make_unique<QScreenCapture>();
@@ -152,13 +156,13 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) {
             m->screen->setScreen(selected);
         }
         m->session->setScreenCapture(m->screen.get());
-        connect(m->screen.get(), &QScreenCapture::errorOccurred, this, [this, generation](QScreenCapture::Error, const QString& error) { sourceFailed(generation, error); });
+        connect(m->screen.get(), &QScreenCapture::errorOccurred, this, [failed](QScreenCapture::Error, const QString& error) { failed(error); });
         m->screen->start();
     } else {
         auto window = config.source.window.value<QCapturableWindow>();
         if (!window.isValid()) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop window is unavailable"); return false; }
         m->window = std::make_unique<QWindowCapture>(); m->window->setWindow(window); m->session->setWindowCapture(m->window.get());
-        connect(m->window.get(), &QWindowCapture::errorOccurred, this, [this, generation](QWindowCapture::Error, const QString& error) { sourceFailed(generation, error); });
+        connect(m->window.get(), &QWindowCapture::errorOccurred, this, [failed](QWindowCapture::Error, const QString& error) { failed(error); });
         m->window->start();
     }
     DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop capture started generation=%llu kind=%d source=%s target=%dx%d/%dfps",
@@ -175,7 +179,7 @@ void StreamPublishEngine::closeSource() {
     if (m->screen) { disconnect(m->screen.get(), nullptr, this, nullptr); m->screen->stop(); }
     if (m->window) { disconnect(m->window.get(), nullptr, this, nullptr); m->window->stop(); }
     m->session.reset(); m->camera.reset(); m->screen.reset(); m->window.reset(); m->sink.reset();
-    std::lock_guard<std::mutex> guard(m->lock); m->frame = {}; m->framePts = -1;
+    std::lock_guard<std::mutex> guard(m->lock); m->frame = {}; m->previewFrame = {}; m->framePts = -1;
 #endif
 }
 
@@ -187,7 +191,7 @@ bool StreamPublishEngine::start(uint64_t group, const StreamPublishConfig& confi
         emit errorOccurred(text("The microphone is used by a call, radio or recording")); return false;
     }
     { std::lock_guard<std::mutex> guard(m->lock);
-      m->wanted = config; m->good = {}; m->hasGood = m->rollingBack = false; m->generation++;
+      m->wanted = config; m->good = {}; m->hasGood = m->rollingBack = false; m->generation++; m->confirmedGeneration = 0;
       m->origin = m->started = monotonicUs(); m->preview = {}; m->stats = {}; }
     m->id = QRandomGenerator::global()->generate64(); if (!m->id) m->id = 1;
     m->group = group; m->audio.reset(); m->audioSequence = 0; m->muted = false;
@@ -213,8 +217,15 @@ bool StreamPublishEngine::reconfigure(const StreamPublishConfig& config) {
     if (config.audio != previous.audio || config.microphone != previous.microphone) {
         emit errorOccurred(text("Change the microphone before starting the stream")); return false;
     }
-    { std::lock_guard<std::mutex> guard(m->lock); m->wanted = config; m->generation++; m->started = monotonicUs(); }
-    if (!openSource(config)) { sourceFailed(m->generation, text("Could not open the selected video source")); return false; }
+    bool sourceChange = config.source.kind != previous.source.kind || config.source.id != previous.source.id
+        || config.source.window != previous.source.window || (config.source.kind == StreamVideoSource::Camera
+        && (config.video.width != previous.video.width || config.video.height != previous.video.height || config.video.fps != previous.video.fps));
+    { std::lock_guard<std::mutex> guard(m->lock); m->wanted = config; m->generation++; m->started = monotonicUs();
+#ifdef HAVE_STREAM_CAPTURE
+      if (!sourceChange) { m->frame = m->previewFrame; m->frameGeneration = m->generation; }
+#endif
+    }
+    if (sourceChange && !openSource(config)) { sourceFailed(m->generation, text("Could not open the selected video source")); return false; }
     m->wake.notify_all(); emit stateChanged(text("Applying video settings…")); return true;
 }
 
@@ -311,7 +322,7 @@ void StreamPublishEngine::audioLoop() {
                 if (!std::equal(wanted, wanted + 3, settings)) {
                     audio_compressor_config_t config{48000, 2, 20, 200, 0, (float)wanted[1], (float)wanted[2], wanted[2] * 5.0f, 0.25f};
                     if (audio_compressor_configure(compressor, &config) < 0) {
-                        DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop stream compressor configuration failed"); break;
+                        DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop stream compressor configuration failed"); result = -1; break;
                     }
                     audio_compressor_set_enabled(compressor, wanted[0]); std::copy(wanted, wanted + 3, settings);
                     DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop stream compressor enabled=%d max_gain=%d rise=%d stereo=1 delay=0",
@@ -321,7 +332,7 @@ void StreamPublishEngine::audioLoop() {
                 uint8_t packet[STREAM_MEDIA_MAX_OPUS];
                 result = audio_compressor_process_frame(compressor, pcm, pcm, 1920);
                 if (!result) result = opus_codec_encode(encoder, pcm, 960, packet, sizeof(packet));
-                if (result <= 0) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop stream audio encoding failed result=%d", result); break; }
+                if (result <= 0) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop stream audio encoding failed result=%d", result); result = -1; break; }
                 if (pts > lastSent) {
                     gui_stream_publish_feed(m->id, false, pts, false, QByteArray(reinterpret_cast<const char*>(packet), result)); lastSent = pts;
                 } else DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop stream audio stale PTS=%lld last=%lld", (long long)pts, (long long)lastSent);
@@ -365,6 +376,12 @@ void StreamPublishEngine::videoLoop() {
             if (capture <= lastPts || monotonicUs() - m->origin - capture > 250000) { skipped++; continue; }
             lastImage = frame.toImage();
             if (lastImage.isNull()) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop captured frame conversion failed"); continue; }
+#if QT_VERSION >= QT_VERSION_CHECK(6, 7, 0)
+            if (frame.rotation() != QtVideo::Rotation::None) lastImage = lastImage.transformed(QTransform().rotate(static_cast<int>(frame.rotation())));
+            if (frame.mirrored()) lastImage = lastImage.mirrored(true, false);
+#else
+            if (frame.rotationAngle()) lastImage = lastImage.transformed(QTransform().rotate(frame.rotationAngle()));
+#endif
         } else if (!screen || lastImage.isNull()) {
             std::unique_lock<std::mutex> lock(m->lock); m->wake.wait_for(lock, std::chrono::milliseconds(5)); continue;
         }
@@ -384,7 +401,10 @@ void StreamPublishEngine::videoLoop() {
                 if (encoded.key) {
                     bool first;
                     { std::lock_guard<std::mutex> guard(m->lock); first = !m->hasGood || m->good.video != config.video || m->rollingBack;
-                      if (m->generation == generation) { m->good = config; m->hasGood = true; m->rollingBack = false; } }
+                      first |= m->confirmedGeneration != generation;
+                      if (m->generation == generation) {
+                          m->good = config; m->hasGood = true; m->rollingBack = false; m->confirmedGeneration = generation;
+                      } }
                     if (first) QMetaObject::invokeMethod(this, [this, generation] {
                         if (generation != m->generation) return;
                         emit configurationChanged(); emit stateChanged(text("Live"));
@@ -416,15 +436,20 @@ void StreamPublishEngine::tick() {
     QImage preview;
     { std::lock_guard<std::mutex> guard(m->lock); preview = m->preview; }
     if (!preview.isNull()) emit previewChanged(preview);
+#ifdef HAVE_STREAM_CAPTURE
+    QVideoFrame rawPreview;
+    { std::lock_guard<std::mutex> guard(m->lock); rawPreview = m->previewFrame; }
+    if (rawPreview.isValid()) emit previewVideoChanged(QVariant::fromValue(rawPreview));
+#endif
     if (status.state == STREAM_PUBLISHER_ENDED || status.state == STREAM_PUBLISHER_FAILED) {
         stop(); m->id = m->group = 0;
         if (status.state == STREAM_PUBLISHER_FAILED) emit errorOccurred(text("Stream publication failed"));
         emit stateChanged(text("Stopped")); emit configurationChanged(); return;
     }
     if (m->running) {
-        bool hasGood; int64_t started;
-        { std::lock_guard<std::mutex> guard(m->lock); hasGood = m->hasGood; started = m->started; }
-        if (!hasGood && monotonicUs() - started > 8000000) sourceFailed(m->generation, text("Timed out waiting for the first video frame"));
+        bool confirmed; int64_t started;
+        { std::lock_guard<std::mutex> guard(m->lock); confirmed = m->confirmedGeneration == m->generation; started = m->started; }
+        if (!confirmed && monotonicUs() - started > 8000000) sourceFailed(m->generation, text("Timed out waiting for the first video frame"));
         if (m->wanted.audio && m_capture.needsRecovery()) { m_capture.close(); m_capture.retryLater(); }
         if (m->wanted.audio && m_capture.retryReady()) openMicrophone();
     }