Render raw preview at 30 fps and preserve fallback audio sample timing

tools/chatgui/src/stream_publish_engine.cpp

@@ -44,13 +44,14 @@ struct StreamPublishEngine::Impl {
     AudioEventQueue<Audio, 32> audio;
     uint64_t id = 0, group = 0, generation = 0, sourceGeneration = 0, confirmedGeneration = 0, frameGeneration = 0, audioSequence = 0;
     int64_t origin = 0, started = 0, framePts = -1, backendOrigin = -1, backendAnchor = -1;
+    int64_t audioAnchor = -1;
+    uint64_t audioFrames = 0;
     int channels = 2;
     uint64_t capturedFrames = 0;
     int64_t captureReport = 0;
     bool probeReady = false, probeDelivered = false, hasGood = false, rollingBack = false;
     StreamPublishConfig wanted, good;
     QList<int> codecs;
-    QImage preview;
     stream_publisher_status stats{};
 #ifdef HAVE_STREAM_CAPTURE
     std::unique_ptr<QMediaCaptureSession> session;
@@ -60,11 +61,25 @@ struct StreamPublishEngine::Impl {
     std::unique_ptr<QWindowCapture> window;
     QVideoFrame frame;
     QVideoFrame previewFrame;
+    QVideoFrame presentedPreview;
 #endif
 };
 
 StreamPublishEngine::StreamPublishEngine(QObject* parent) : QObject(parent), m(std::make_unique<Impl>()) {
-    m_timer.setInterval(100); connect(&m_timer, &QTimer::timeout, this, &StreamPublishEngine::tick); m_timer.start();
+    m_timer.setTimerType(Qt::PreciseTimer); m_timer.setInterval(100);
+    connect(&m_timer, &QTimer::timeout, this, &StreamPublishEngine::tick); m_timer.start();
+#ifdef HAVE_STREAM_CAPTURE
+    m_previewTimer.setTimerType(Qt::PreciseTimer); m_previewTimer.setInterval(33);
+    connect(&m_previewTimer, &QTimer::timeout, this, [this] {
+        if (!m->running) return;
+        QVideoFrame frame;
+        { std::lock_guard<std::mutex> guard(m->lock);
+          if (m->previewFrame == m->presentedPreview) return;
+          frame = m->presentedPreview = m->previewFrame; }
+        if (frame.isValid()) emit previewVideoChanged(QVariant::fromValue(frame));
+    });
+    m_previewTimer.start();
+#endif
     try {
         m->probeThread = std::thread([this] {
             auto codecs = supported() ? StreamVideoEncoder::available() : QList<int>();
@@ -78,7 +93,7 @@ StreamPublishEngine::StreamPublishEngine(QObject* parent) : QObject(parent), m(s
     connect(sound, &SoundManager::contextRestored, this, [this] { if (m->running && m->wanted.audio) openMicrophone(); });
 }
 StreamPublishEngine::~StreamPublishEngine() {
-    stop(); m_timer.stop();
+    stop(); m_timer.stop(); m_previewTimer.stop();
     if (m->probeThread.joinable()) m->probeThread.join();
 }
 bool StreamPublishEngine::supported() {
@@ -89,6 +104,7 @@ bool StreamPublishEngine::supported() {
 #endif
 }
 bool StreamPublishEngine::active() const { return m->id != 0; }
+bool StreamPublishEngine::running() const { return m->running.load(); }
 uint64_t StreamPublishEngine::group() const { return m->group; }
 StreamPublishConfig StreamPublishEngine::config() const { std::lock_guard<std::mutex> guard(m->lock); return m->wanted; }
 QList<int> StreamPublishEngine::codecs() const { std::lock_guard<std::mutex> guard(m->lock); return m->codecs; }
@@ -115,7 +131,8 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) {
     closeSource();
 #ifdef HAVE_STREAM_CAPTURE
     uint64_t generation;
-    { std::lock_guard<std::mutex> guard(m->lock); generation = ++m->sourceGeneration; m->backendOrigin = m->backendAnchor = -1; }
+    { std::lock_guard<std::mutex> guard(m->lock); generation = ++m->sourceGeneration;
+      m->backendOrigin = m->backendAnchor = -1; m->capturedFrames = 0; }
     m->session = std::make_unique<QMediaCaptureSession>(); m->sink = std::make_unique<QVideoSink>();
     m->session->setVideoSink(m->sink.get());
     connect(m->sink.get(), &QVideoSink::videoFrameChanged, this, [this, generation](const QVideoFrame& frame) {
@@ -196,12 +213,13 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) {
 }
 void StreamPublishEngine::closeSource() {
 #ifdef HAVE_STREAM_CAPTURE
+    emit previewVideoChanged(QVariant());
     if (m->sink) disconnect(m->sink.get(), nullptr, this, nullptr);
     if (m->camera) { disconnect(m->camera.get(), nullptr, this, nullptr); m->camera->stop(); }
     if (m->screen) { disconnect(m->screen.get(), nullptr, this, nullptr); m->screen->stop(); }
     if (m->window) { disconnect(m->window.get(), nullptr, this, nullptr); m->window->stop(); }
     m->session.reset(); m->camera.reset(); m->screen.reset(); m->window.reset(); m->sink.reset();
-    std::lock_guard<std::mutex> guard(m->lock); m->frame = {}; m->previewFrame = {}; m->framePts = -1;
+    std::lock_guard<std::mutex> guard(m->lock); m->frame = {}; m->previewFrame = {}; m->presentedPreview = {}; m->framePts = -1;
 #endif
 }
 
@@ -214,7 +232,7 @@ bool StreamPublishEngine::start(uint64_t group, const StreamPublishConfig& confi
     }
     { std::lock_guard<std::mutex> guard(m->lock);
       m->wanted = config; m->good = {}; m->hasGood = m->rollingBack = false; m->generation++; m->confirmedGeneration = 0;
-      m->origin = m->started = monotonicUs(); m->preview = {}; m->stats = {}; }
+      m->origin = m->started = monotonicUs(); m->stats = {}; }
     m->id = QRandomGenerator::global()->generate64(); if (!m->id) m->id = 1;
     m->group = group; m->audio.reset(); m->audioSequence = 0; m->muted = false;
     stream_media_format format{(uint16_t)config.video.width, (uint16_t)config.video.height, (uint16_t)config.video.rotation,
@@ -276,6 +294,7 @@ void StreamPublishEngine::stop() {
 
 bool StreamPublishEngine::openMicrophone() {
     m_capture.close();
+    m->audioAnchor = -1; m->audioFrames = 0;
     auto config = this->config(); auto* context = SoundManager::instance()->context();
     if (!context) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop stream audio context unavailable"); return false; }
     ma_device_id device; bool selected = SoundManager::deviceId(config.microphone, &device);
@@ -287,8 +306,10 @@ bool StreamPublishEngine::openMicrophone() {
         if (!m_capture.init(context, capture, "stream-capture", m->id, callback)) continue;
         m->channels = channels;
         if (!m_capture.start()) continue;
-        if (channels == 1) DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop stream input is mono; stereo transport duplicates the input");
-        emit audioChannelsChanged(channels); return true;
+        int native = m_capture.device()->capture.internalChannels;
+        if (native == 1) DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop stream input is mono; stereo transport duplicates the input");
+        DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop microphone opened requested=%d backend_channels=%d rate=%u", channels, native, m_capture.device()->sampleRate);
+        emit audioChannelsChanged(native == 1 ? 1 : channels); return true;
     }
     DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop stream cannot open microphone: %s", qPrintable(config.microphone)); return false;
 }
@@ -296,15 +317,20 @@ bool StreamPublishEngine::openMicrophone() {
 /* Hardware callback копирует максимум 20ms за элемент, без codec/DSP/allocations. */
 void StreamPublishEngine::captureAudio(const void* input, unsigned frames) {
     const auto* pcm = static_cast<const int16_t*>(input);
-    if (!m->running || !pcm) return;
+    if (!m->running) return;
     int64_t first = m_capture.callbackTimeUs() - m->origin;
+    if (m_capture.timingMeasured() || m->audioAnchor < 0 || m_capture.captureMissing()) {
+        m->audioAnchor = first; m->audioFrames = 0;
+    }
+    first = m->audioAnchor + (int64_t)m->audioFrames * 1000000 / 48000; m->audioFrames += frames;
+    if (!pcm) m->audioDropped++;
     for (unsigned offset = 0; offset < frames;) {
         Impl::Audio record; record.frames = std::min(960u, frames - offset);
         record.pts = std::max<int64_t>(0, first + (int64_t)offset * 1000000 / 48000); record.created = monotonicUs();
         record.sequence = ++m->audioSequence;
         for (unsigned i = 0; i < record.frames; i++) {
-            record.pcm[2 * i] = pcm[(offset + i) * m->channels];
-            record.pcm[2 * i + 1] = pcm[(offset + i) * m->channels + (m->channels == 2 ? 1 : 0)];
+            record.pcm[2 * i] = pcm ? pcm[(offset + i) * m->channels] : 0;
+            record.pcm[2 * i + 1] = pcm ? pcm[(offset + i) * m->channels + (m->channels == 2 ? 1 : 0)] : 0;
         }
         if (!m->audio.push(record)) m->audioDropped++;
         offset += record.frames;
@@ -428,14 +454,17 @@ void StreamPublishEngine::videoLoop() {
                       if (m->generation == generation) {
                           m->good = config; m->hasGood = true; m->rollingBack = false; m->confirmedGeneration = generation;
                       } }
-                    if (first) QMetaObject::invokeMethod(this, [this, generation] {
+                    if (first) {
+                        DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop first KEY committed generation=%llu revision=%u PTS=%llu bytes=%lld",
+                            (unsigned long long)generation, revision, (unsigned long long)encoded.pts, (long long)encoded.data.size());
+                        QMetaObject::invokeMethod(this, [this, generation] {
                         if (generation != m->generation) return;
                         emit configurationChanged(); emit stateChanged(text("Live"));
-                    });
+                        });
+                    }
                 }
             } else skipped++;
         }
-        { std::lock_guard<std::mutex> guard(m->lock); if (m->generation == generation) m->preview = lastImage; }
         if (now - report >= 1000000) {
             DEBUG_DEBUG(DEBUG_CATEGORY_VIDEO, "desktop capture generation=%llu encoded=%u skipped=%u frame_age=%lldus encoder=%s",
                 (unsigned long long)generation, count, skipped, (long long)(screen ? 0 : now - m->origin - capture), qPrintable(encoder->name()));
@@ -456,14 +485,6 @@ void StreamPublishEngine::tick() {
     }
     { std::lock_guard<std::mutex> guard(m->lock); m->stats = status; }
     emit telemetryChanged(status);
-    QImage preview;
-    { std::lock_guard<std::mutex> guard(m->lock); preview = m->preview; }
-    if (!preview.isNull()) emit previewChanged(preview);
-#ifdef HAVE_STREAM_CAPTURE
-    QVideoFrame rawPreview;
-    { std::lock_guard<std::mutex> guard(m->lock); rawPreview = m->previewFrame; }
-    if (rawPreview.isValid()) emit previewVideoChanged(QVariant::fromValue(rawPreview));
-#endif
     if (status.state == STREAM_PUBLISHER_ENDED || status.state == STREAM_PUBLISHER_FAILED) {
         stop(); m->id = m->group = 0;
         if (status.state == STREAM_PUBLISHER_FAILED) emit errorOccurred(text("Stream publication failed"));