From ba17f9908d32f59136d47dba1bdda69c295ed42c Mon Sep 17 00:00:00 2001 From: evgeny Date: Wed, 14 Oct 2026 12:06:45 +0200 Subject: [PATCH] Render raw preview at 30 fps and preserve fallback audio sample timing --- tools/chatgui/src/stream_publish_engine.cpp | 65 ++++++++++++++------- tools/chatgui/src/stream_publish_engine.h | 3 +- tools/chatgui/src/stream_publish_window.cpp | 8 ++- 3 files changed, 51 insertions(+), 25 deletions(-) diff --git a/tools/chatgui/src/stream_publish_engine.cpp b/tools/chatgui/src/stream_publish_engine.cpp index 8e343334..d1f14e3c 100644 --- a/tools/chatgui/src/stream_publish_engine.cpp +++ b/tools/chatgui/src/stream_publish_engine.cpp @@ -44,13 +44,14 @@ struct StreamPublishEngine::Impl { AudioEventQueue audio; uint64_t id = 0, group = 0, generation = 0, sourceGeneration = 0, confirmedGeneration = 0, frameGeneration = 0, audioSequence = 0; int64_t origin = 0, started = 0, framePts = -1, backendOrigin = -1, backendAnchor = -1; + int64_t audioAnchor = -1; + uint64_t audioFrames = 0; int channels = 2; uint64_t capturedFrames = 0; int64_t captureReport = 0; bool probeReady = false, probeDelivered = false, hasGood = false, rollingBack = false; StreamPublishConfig wanted, good; QList codecs; - QImage preview; stream_publisher_status stats{}; #ifdef HAVE_STREAM_CAPTURE std::unique_ptr session; @@ -60,11 +61,25 @@ struct StreamPublishEngine::Impl { std::unique_ptr window; QVideoFrame frame; QVideoFrame previewFrame; + QVideoFrame presentedPreview; #endif }; StreamPublishEngine::StreamPublishEngine(QObject* parent) : QObject(parent), m(std::make_unique()) { - m_timer.setInterval(100); connect(&m_timer, &QTimer::timeout, this, &StreamPublishEngine::tick); m_timer.start(); + m_timer.setTimerType(Qt::PreciseTimer); m_timer.setInterval(100); + connect(&m_timer, &QTimer::timeout, this, &StreamPublishEngine::tick); m_timer.start(); +#ifdef HAVE_STREAM_CAPTURE + m_previewTimer.setTimerType(Qt::PreciseTimer); m_previewTimer.setInterval(33); + connect(&m_previewTimer, &QTimer::timeout, this, [this] { + if (!m->running) return; + QVideoFrame frame; + { std::lock_guard guard(m->lock); + if (m->previewFrame == m->presentedPreview) return; + frame = m->presentedPreview = m->previewFrame; } + if (frame.isValid()) emit previewVideoChanged(QVariant::fromValue(frame)); + }); + m_previewTimer.start(); +#endif try { m->probeThread = std::thread([this] { auto codecs = supported() ? StreamVideoEncoder::available() : QList(); @@ -78,7 +93,7 @@ StreamPublishEngine::StreamPublishEngine(QObject* parent) : QObject(parent), m(s connect(sound, &SoundManager::contextRestored, this, [this] { if (m->running && m->wanted.audio) openMicrophone(); }); } StreamPublishEngine::~StreamPublishEngine() { - stop(); m_timer.stop(); + stop(); m_timer.stop(); m_previewTimer.stop(); if (m->probeThread.joinable()) m->probeThread.join(); } bool StreamPublishEngine::supported() { @@ -89,6 +104,7 @@ bool StreamPublishEngine::supported() { #endif } bool StreamPublishEngine::active() const { return m->id != 0; } +bool StreamPublishEngine::running() const { return m->running.load(); } uint64_t StreamPublishEngine::group() const { return m->group; } StreamPublishConfig StreamPublishEngine::config() const { std::lock_guard guard(m->lock); return m->wanted; } QList StreamPublishEngine::codecs() const { std::lock_guard guard(m->lock); return m->codecs; } @@ -115,7 +131,8 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) { closeSource(); #ifdef HAVE_STREAM_CAPTURE uint64_t generation; - { std::lock_guard guard(m->lock); generation = ++m->sourceGeneration; m->backendOrigin = m->backendAnchor = -1; } + { std::lock_guard guard(m->lock); generation = ++m->sourceGeneration; + m->backendOrigin = m->backendAnchor = -1; m->capturedFrames = 0; } m->session = std::make_unique(); m->sink = std::make_unique(); m->session->setVideoSink(m->sink.get()); connect(m->sink.get(), &QVideoSink::videoFrameChanged, this, [this, generation](const QVideoFrame& frame) { @@ -196,12 +213,13 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) { } void StreamPublishEngine::closeSource() { #ifdef HAVE_STREAM_CAPTURE + emit previewVideoChanged(QVariant()); if (m->sink) disconnect(m->sink.get(), nullptr, this, nullptr); if (m->camera) { disconnect(m->camera.get(), nullptr, this, nullptr); m->camera->stop(); } if (m->screen) { disconnect(m->screen.get(), nullptr, this, nullptr); m->screen->stop(); } if (m->window) { disconnect(m->window.get(), nullptr, this, nullptr); m->window->stop(); } m->session.reset(); m->camera.reset(); m->screen.reset(); m->window.reset(); m->sink.reset(); - std::lock_guard guard(m->lock); m->frame = {}; m->previewFrame = {}; m->framePts = -1; + std::lock_guard guard(m->lock); m->frame = {}; m->previewFrame = {}; m->presentedPreview = {}; m->framePts = -1; #endif } @@ -214,7 +232,7 @@ bool StreamPublishEngine::start(uint64_t group, const StreamPublishConfig& confi } { std::lock_guard guard(m->lock); m->wanted = config; m->good = {}; m->hasGood = m->rollingBack = false; m->generation++; m->confirmedGeneration = 0; - m->origin = m->started = monotonicUs(); m->preview = {}; m->stats = {}; } + m->origin = m->started = monotonicUs(); m->stats = {}; } m->id = QRandomGenerator::global()->generate64(); if (!m->id) m->id = 1; m->group = group; m->audio.reset(); m->audioSequence = 0; m->muted = false; stream_media_format format{(uint16_t)config.video.width, (uint16_t)config.video.height, (uint16_t)config.video.rotation, @@ -276,6 +294,7 @@ void StreamPublishEngine::stop() { bool StreamPublishEngine::openMicrophone() { m_capture.close(); + m->audioAnchor = -1; m->audioFrames = 0; auto config = this->config(); auto* context = SoundManager::instance()->context(); if (!context) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop stream audio context unavailable"); return false; } ma_device_id device; bool selected = SoundManager::deviceId(config.microphone, &device); @@ -287,8 +306,10 @@ bool StreamPublishEngine::openMicrophone() { if (!m_capture.init(context, capture, "stream-capture", m->id, callback)) continue; m->channels = channels; if (!m_capture.start()) continue; - if (channels == 1) DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop stream input is mono; stereo transport duplicates the input"); - emit audioChannelsChanged(channels); return true; + int native = m_capture.device()->capture.internalChannels; + if (native == 1) DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop stream input is mono; stereo transport duplicates the input"); + DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop microphone opened requested=%d backend_channels=%d rate=%u", channels, native, m_capture.device()->sampleRate); + emit audioChannelsChanged(native == 1 ? 1 : channels); return true; } DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop stream cannot open microphone: %s", qPrintable(config.microphone)); return false; } @@ -296,15 +317,20 @@ bool StreamPublishEngine::openMicrophone() { /* Hardware callback копирует максимум 20ms за элемент, без codec/DSP/allocations. */ void StreamPublishEngine::captureAudio(const void* input, unsigned frames) { const auto* pcm = static_cast(input); - if (!m->running || !pcm) return; + if (!m->running) return; int64_t first = m_capture.callbackTimeUs() - m->origin; + if (m_capture.timingMeasured() || m->audioAnchor < 0 || m_capture.captureMissing()) { + m->audioAnchor = first; m->audioFrames = 0; + } + first = m->audioAnchor + (int64_t)m->audioFrames * 1000000 / 48000; m->audioFrames += frames; + if (!pcm) m->audioDropped++; for (unsigned offset = 0; offset < frames;) { Impl::Audio record; record.frames = std::min(960u, frames - offset); record.pts = std::max(0, first + (int64_t)offset * 1000000 / 48000); record.created = monotonicUs(); record.sequence = ++m->audioSequence; for (unsigned i = 0; i < record.frames; i++) { - record.pcm[2 * i] = pcm[(offset + i) * m->channels]; - record.pcm[2 * i + 1] = pcm[(offset + i) * m->channels + (m->channels == 2 ? 1 : 0)]; + record.pcm[2 * i] = pcm ? pcm[(offset + i) * m->channels] : 0; + record.pcm[2 * i + 1] = pcm ? pcm[(offset + i) * m->channels + (m->channels == 2 ? 1 : 0)] : 0; } if (!m->audio.push(record)) m->audioDropped++; offset += record.frames; @@ -428,14 +454,17 @@ void StreamPublishEngine::videoLoop() { if (m->generation == generation) { m->good = config; m->hasGood = true; m->rollingBack = false; m->confirmedGeneration = generation; } } - if (first) QMetaObject::invokeMethod(this, [this, generation] { + if (first) { + DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop first KEY committed generation=%llu revision=%u PTS=%llu bytes=%lld", + (unsigned long long)generation, revision, (unsigned long long)encoded.pts, (long long)encoded.data.size()); + QMetaObject::invokeMethod(this, [this, generation] { if (generation != m->generation) return; emit configurationChanged(); emit stateChanged(text("Live")); - }); + }); + } } } else skipped++; } - { std::lock_guard guard(m->lock); if (m->generation == generation) m->preview = lastImage; } if (now - report >= 1000000) { DEBUG_DEBUG(DEBUG_CATEGORY_VIDEO, "desktop capture generation=%llu encoded=%u skipped=%u frame_age=%lldus encoder=%s", (unsigned long long)generation, count, skipped, (long long)(screen ? 0 : now - m->origin - capture), qPrintable(encoder->name())); @@ -456,14 +485,6 @@ void StreamPublishEngine::tick() { } { std::lock_guard guard(m->lock); m->stats = status; } emit telemetryChanged(status); - QImage preview; - { std::lock_guard guard(m->lock); preview = m->preview; } - if (!preview.isNull()) emit previewChanged(preview); -#ifdef HAVE_STREAM_CAPTURE - QVideoFrame rawPreview; - { std::lock_guard guard(m->lock); rawPreview = m->previewFrame; } - if (rawPreview.isValid()) emit previewVideoChanged(QVariant::fromValue(rawPreview)); -#endif if (status.state == STREAM_PUBLISHER_ENDED || status.state == STREAM_PUBLISHER_FAILED) { stop(); m->id = m->group = 0; if (status.state == STREAM_PUBLISHER_FAILED) emit errorOccurred(text("Stream publication failed")); diff --git a/tools/chatgui/src/stream_publish_engine.h b/tools/chatgui/src/stream_publish_engine.h index 29376b82..3065bcc6 100644 --- a/tools/chatgui/src/stream_publish_engine.h +++ b/tools/chatgui/src/stream_publish_engine.h @@ -35,6 +35,7 @@ public: bool reconfigure(const StreamPublishConfig& config); void stop(); bool active() const; + bool running() const; uint64_t group() const; StreamPublishConfig config() const; QList codecs() const; @@ -45,7 +46,6 @@ signals: void codecsChanged(); void stateChanged(const QString& state); void configurationChanged(); - void previewChanged(const QImage& frame); void previewVideoChanged(const QVariant& frame); void telemetryChanged(const stream_publisher_status& status); void errorOccurred(const QString& error); @@ -56,6 +56,7 @@ private: AudioDevice m_capture; AudioCaptureLease m_microphone{AudioCaptureLease::Mode::Exclusive}; QTimer m_timer; + QTimer m_previewTimer; bool openSource(const StreamPublishConfig& config); void closeSource(); bool openMicrophone(); diff --git a/tools/chatgui/src/stream_publish_window.cpp b/tools/chatgui/src/stream_publish_window.cpp index f6130a66..b1bcd9dc 100644 --- a/tools/chatgui/src/stream_publish_window.cpp +++ b/tools/chatgui/src/stream_publish_window.cpp @@ -30,7 +30,8 @@ StreamPublishWindow::StreamPublishWindow(StreamPublishEngine* engine, uint64_t g auto* preview = new QVideoWidget(this); m_preview = preview; preview->setMinimumHeight(160); preview->setAspectRatioMode(Qt::KeepAspectRatio); connect(engine, &StreamPublishEngine::previewVideoChanged, this, [this, preview](const QVariant& frame) { - if (isVisible()) preview->videoSink()->setVideoFrame(frame.value()); + auto video = frame.value(); + if (!video.isValid() || isVisible()) preview->videoSink()->setVideoFrame(video); }); #else m_preview = new QLabel(text("Video capture requires Qt Multimedia and FFmpeg"), this); @@ -38,12 +39,14 @@ StreamPublishWindow::StreamPublishWindow(StreamPublishEngine* engine, uint64_t g layout->addWidget(m_preview, 1); auto* scroll = new QScrollArea(this); scroll->setWidgetResizable(true); auto* settings = new QWidget(scroll); auto* form = new QFormLayout(settings); + form->setRowWrapPolicy(QFormLayout::WrapLongRows); scroll->setWidget(settings); scroll->setMinimumHeight(220); layout->addWidget(scroll); m_source = new QComboBox(settings); m_source->setSizeAdjustPolicy(QComboBox::AdjustToMinimumContentsLengthWithIcon); m_source->setMinimumContentsLength(16); m_source->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Fixed); auto* sourceRow = new QWidget(settings); auto* sourceLayout = new QHBoxLayout(sourceRow); sourceLayout->setContentsMargins(0, 0, 0, 0); sourceLayout->addWidget(m_source, 1); auto* refresh = new QPushButton(text("Refresh"), settings); sourceLayout->addWidget(refresh); connect(refresh, &QPushButton::clicked, this, &StreamPublishWindow::refreshSources); form->addRow(text("Video source"), sourceRow); + connect(m_source, QOverload::of(&QComboBox::currentIndexChanged), this, [this] { m_source->setToolTip(m_source->currentText()); }); m_mode = new QComboBox(settings); for (auto mode : {QSize(640, 360), QSize(1280, 720), QSize(1920, 1080)}) for (int fps : {15, 30, 60}) m_mode->addItem(QStringLiteral("%1 × %2 · %3 fps").arg(mode.width()).arg(mode.height()).arg(fps), @@ -129,7 +132,8 @@ void StreamPublishWindow::apply() { if (!m_updating && m_engine->active()) m_eng void StreamPublishWindow::synchronize() { m_updating = true; bool active = m_engine->active(); m_start->setText(active ? text("Stop stream") : text("Start stream")); - m_start->setEnabled(active || (StreamPublishEngine::supported() && m_source->count() && m_codec->count())); + m_start->setEnabled((active && m_engine->running()) || (!active && StreamPublishEngine::supported() && m_source->count() && m_codec->count())); + for (auto* box : {m_source, m_mode, m_codec, m_bitrate}) box->setEnabled(!active || m_engine->running()); m_audio->setEnabled(!active); m_microphone->setEnabled(!active); m_mute->setEnabled(active && m_engine->config().audio); if (active) { auto config = m_engine->config();