#include "stream_publish_engine.h" #include "sound_manager.h" #include "audio_event_queue.h" #include "../transport/stream_bridge.h" #include "debug_config.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include extern "C" { #include "opus_codec.h" #include "audio_compressor.h" } #ifdef HAVE_STREAM_CAPTURE #include #include #include #include #include #include #include #endif #ifdef UTUN_STREAM_X11_CURSOR #include "stream_screen_cursor.h" #endif #ifdef UTUN_STREAM_PORTAL #include "stream_portal_capture.h" #endif namespace { int64_t monotonicUs() { return std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()).count(); } QString text(const char* value) { return QCoreApplication::translate("ChatGui", value); } #ifdef HAVE_STREAM_CAPTURE /* Имя QScreen не уникально (например, несколько X11 screens); номер различает одинаковые дескрипторы. */ QByteArray screenId(QScreen* screen) { int duplicate = 0; for (auto* other : QGuiApplication::screens()) { if (other == screen) break; if (other->name() == screen->name() && other->manufacturer() == screen->manufacturer() && other->model() == screen->model() && other->serialNumber() == screen->serialNumber()) duplicate++; } return QJsonDocument(QJsonArray{screen->name(), screen->manufacturer(), screen->model(), screen->serialNumber(), duplicate}) .toJson(QJsonDocument::Compact); } #endif } struct StreamPublishEngine::Impl { struct Audio { int64_t pts = 0, created = 0; uint64_t sequence = 0; unsigned frames = 0; int16_t pcm[1920]{}; }; std::mutex lock; std::condition_variable wake; std::thread videoThread, audioThread, joinThread; std::thread imageThread, imageJoinThread; std::atomic workersJoined{false}, videoExited{false}, audioExited{false}, imageJoined{false}; bool stopping = false, finSent = false, destroying = false; int64_t stopStarted = 0, stopReport = 0; std::atomic running{false}, muted{false}; std::atomic audioDropped{0}, audioMissing{0}; std::atomic lastAudioAt{0}; AudioEventQueue audio; std::condition_variable imageWake; QVideoFrame imageInput; uint64_t imageGeneration = 0; int64_t imagePts = -1; bool imagePending = false, imageStop = false; uint64_t id = 0, group = 0, generation = 0, sourceGeneration = 0, confirmedGeneration = 0, frameGeneration = 0, audioSequence = 0; int64_t origin = 0, started = 0, framePts = -1, backendOrigin = -1, backendAnchor = -1; int64_t audioAnchor = -1; uint64_t audioFrames = 0; int channels = 2; uint64_t capturedFrames = 0; uint64_t convertedCaptureFrames = 0; int64_t captureReport = 0; std::atomic videoStage{"stopped"}; std::atomic videoStageAt{0}; std::atomic imageStage{"starting"}; std::atomic imageStageAt{monotonicUs()}; std::atomic imageFrames{0}; std::atomic convertedFrames{0}, encodedFrames{0}, deliveredFrames{0}, repeatedFrames{0}; void videoProgress(const char* stage) { videoStageAt.store(monotonicUs()); videoStage.store(stage); } void imageProgress(const char* stage) { imageStageAt.store(monotonicUs()); imageStage.store(stage); } bool hasGood = false, rollingBack = false, videoReady = false; uint64_t softwareRequest = 0; int softwareCodec = 0; int64_t microphoneStarted = 0; StreamPublishConfig wanted, good; QList codecs; stream_publisher_status stats{}; #ifdef HAVE_STREAM_CAPTURE std::unique_ptr session; std::unique_ptr sink; std::unique_ptr camera; std::unique_ptr screen; #ifdef UTUN_STREAM_PORTAL std::unique_ptr portal; #endif std::unique_ptr screenGrabTimer; #ifdef UTUN_STREAM_X11_CURSOR std::unique_ptr cursor; #endif std::unique_ptr window; QImage frame; QImage latestImage; QVideoFrame previewFrame; QVideoFrame presentedPreview; int64_t lastImageCaptureAt = 0; #endif }; StreamPublishEngine::StreamPublishEngine(QObject* parent) : QObject(parent), m(std::make_unique()) { m_timer.setTimerType(Qt::PreciseTimer); m_timer.setInterval(100); connect(&m_timer, &QTimer::timeout, this, &StreamPublishEngine::tick); m_timer.start(); #ifdef HAVE_STREAM_CAPTURE auto* devices = new QMediaDevices(this); connect(devices, &QMediaDevices::videoInputsChanged, this, &StreamPublishEngine::sourcesChanged); auto watchScreen = [this](QScreen* screen) { connect(screen, &QScreen::geometryChanged, this, &StreamPublishEngine::sourcesChanged); connect(screen, &QScreen::physicalDotsPerInchChanged, this, &StreamPublishEngine::sourcesChanged); }; for (auto* screen : QGuiApplication::screens()) watchScreen(screen); connect(qGuiApp, &QGuiApplication::screenAdded, this, [this, watchScreen](QScreen* screen) { watchScreen(screen); emit sourcesChanged(); }); connect(qGuiApp, &QGuiApplication::screenRemoved, this, &StreamPublishEngine::sourcesChanged); m_previewTimer.setTimerType(Qt::PreciseTimer); m_previewTimer.setInterval(33); connect(&m_previewTimer, &QTimer::timeout, this, [this] { if (!m->running) return; QVideoFrame frame; { std::lock_guard guard(m->lock); if (m->previewFrame == m->presentedPreview) return; frame = m->presentedPreview = m->previewFrame; } if (frame.isValid()) emit previewVideoChanged(QVariant::fromValue(frame)); }); m_previewTimer.start(); #endif m->codecs = supported() ? LiveVideoEncoder::available() : QList(); auto* sound = SoundManager::instance(); connect(sound, &SoundManager::contextAboutToReset, this, [this] { if (m->running && m->wanted.audio) publicationFailed(m->id, text("The audio device was disconnected. Select a microphone and start again.")); m_capture.close(); // Контекст нельзя освободить до завершения фонового закрытия устройства. }); #ifdef HAVE_STREAM_CAPTURE m->imageThread = std::thread([this] { DEBUG_INFO(DEBUG_CATEGORY_DEBUG, "desktop image worker started tid=%llu", (unsigned long long)(quintptr)QThread::currentThreadId()); imageLoop(); m->imageProgress("release-thread-resources"); DEBUG_INFO(DEBUG_CATEGORY_DEBUG, "desktop image loop returned; releasing thread resources frames=%llu tid=%llu", (unsigned long long)m->imageFrames.load(), (unsigned long long)(quintptr)QThread::currentThreadId()); }); #else m->imageJoined = true; #endif } StreamPublishEngine::~StreamPublishEngine() { const int64_t destroyAt = monotonicUs(); DEBUG_INFO(DEBUG_CATEGORY_DEBUG, "desktop publisher destruction begin id=%llu running=%d stopping=%d image_stage=%s image_frames=%llu", (unsigned long long)m->id, m->running.load(), m->stopping, m->imageStage.load(), (unsigned long long)m->imageFrames.load()); m->destroying = true; blockSignals(true); stop(); m_timer.stop(); m_previewTimer.stop(); closeSource(); #ifdef HAVE_STREAM_CAPTURE { std::lock_guard guard(m->lock); m->imageStop = true; m->imagePending = false; m->imageInput = {}; } m->imageWake.notify_all(); m->imageJoinThread = std::thread([this] { DEBUG_INFO(DEBUG_CATEGORY_DEBUG, "desktop image join begin stage=%s stage_age_ms=%lld", m->imageStage.load(), (long long)((monotonicUs() - m->imageStageAt.load()) / 1000)); m->imageThread.join(); DEBUG_INFO(DEBUG_CATEGORY_DEBUG, "desktop image join complete; native thread resources released"); m->imageJoined = true; }); #endif if (!finishStop() || !m->imageJoined) { // Qt освобождает thread-local графику через GUI. Даже при выходе приложения нельзя блокировать его join(). DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop publisher destruction waits for workers stopping=%d image_joined=%d image_stage=%s", m->stopping, m->imageJoined.load(), m->imageStage.load()); QEventLoop loop; QTimer timer; int64_t reportAt = destroyAt; bool firstPoll = true; connect(&timer, &QTimer::timeout, &loop, [this, &loop, destroyAt, &reportAt, &firstPoll] { if (firstPoll) { DEBUG_INFO(DEBUG_CATEGORY_DEBUG, "desktop publisher destruction GUI timer active"); firstPoll = false; } if (finishStop() && m->imageJoined) { loop.quit(); return; } const int64_t now = monotonicUs(); if (now - reportAt < 5000000) return; reportAt = now; DEBUG_WARN(DEBUG_CATEGORY_DEBUG, "desktop publisher destruction waiting elapsed_ms=%lld stopping=%d workers_joined=%d " "video_stage=%s image_joined=%d image_stage=%s image_stage_age_ms=%lld image_frames=%llu microphone_closing=%d", (long long)((now - destroyAt) / 1000), m->stopping, m->workersJoined.load(), m->videoStage.load(), m->imageJoined.load(), m->imageStage.load(), (long long)((now - m->imageStageAt.load()) / 1000), (unsigned long long)m->imageFrames.load(), m_capture.closing()); }); timer.start(10); const int result = loop.exec(QEventLoop::ExcludeUserInputEvents); DEBUG_INFO(DEBUG_CATEGORY_DEBUG, "desktop publisher destruction GUI loop exited result=%d stopping=%d image_joined=%d", result, m->stopping, m->imageJoined.load()); if (m->stopping || !m->imageJoined) DEBUG_ERROR(DEBUG_CATEGORY_DEBUG, "desktop publisher destruction GUI loop exited before worker cleanup completed"); } if (m->imageJoinThread.joinable()) m->imageJoinThread.join(); DEBUG_INFO(DEBUG_CATEGORY_DEBUG, "desktop publisher destruction complete elapsed_ms=%lld", (long long)((monotonicUs() - destroyAt) / 1000)); } bool StreamPublishEngine::supported() { #ifdef HAVE_STREAM_CAPTURE return true; #else return false; #endif } bool StreamPublishEngine::active() const { return m->id != 0; } bool StreamPublishEngine::running() const { return m->running.load(); } uint64_t StreamPublishEngine::group() const { return m->group; } StreamPublishConfig StreamPublishEngine::config() const { std::lock_guard guard(m->lock); return m->wanted; } QVariant StreamPublishEngine::previewFrame() const { #ifdef HAVE_STREAM_CAPTURE std::lock_guard guard(m->lock); return QVariant::fromValue(m->previewFrame); #else return {}; #endif } QList StreamPublishEngine::codecs() const { std::lock_guard guard(m->lock); return m->codecs; } bool StreamPublishEngine::cursorSupported(const StreamVideoSource& source) { #ifdef UTUN_STREAM_PORTAL if (source.kind == StreamVideoSource::Screen && source.id == "portal") return true; #endif #ifdef UTUN_STREAM_X11_CURSOR return source.kind == StreamVideoSource::Screen && source.screen && source.id != "portal" && QGuiApplication::platformName() == "xcb"; #else Q_UNUSED(source); return false; #endif } bool StreamPublishEngine::muted() const { return m->muted.load(); } void StreamPublishEngine::setMuted(bool muted) { if (!m->running || !config().audio) { DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop stream mute without active audio"); return; } bool previous = m->muted.exchange(muted); if (!gui_stream_publish_audio(m->id, !muted)) { m->muted = previous; publicationFailed(m->id, text("Could not change stream audio")); return; } DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop stream audio id=%llu muted=%d", (unsigned long long)m->id, muted); emit configurationChanged(); } QList StreamPublishEngine::sources() { QList sources; #ifdef HAVE_STREAM_CAPTURE for (const auto& camera : QMediaDevices::videoInputs()) { StreamVideoSource source{StreamVideoSource::Camera, camera.description(), camera.id(), {}}; for (const auto& format : camera.videoFormats()) { stream_video_add_camera_modes(source.modes, format.resolution(), format.minFrameRate(), format.maxFrameRate()); DEBUG_DEBUG(DEBUG_CATEGORY_VIDEO, "desktop camera capability device=%s size=%dx%d pixel=%d fps=%.3f..%.3f", camera.id().constData(), format.resolution().width(), format.resolution().height(), format.pixelFormat(), format.minFrameRate(), format.maxFrameRate()); } if (source.modes.isEmpty()) DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "desktop camera has no usable modes device=%s", camera.id().constData()); sources.append(source); } if (QGuiApplication::platformName().contains("wayland") || qEnvironmentVariable("XDG_SESSION_TYPE") == "wayland" || qEnvironmentVariableIsSet("WAYLAND_DISPLAY")) { sources.push_back({StreamVideoSource::Screen, text("Screen (system selection)"), "portal", {}}); } else { int number = 0; for (auto* screen : QGuiApplication::screens()) { QSize size = screen->size() * screen->devicePixelRatio(); sources.push_back({StreamVideoSource::Screen, QStringLiteral("%1: %2").arg(++number).arg(screen->name()), screenId(screen), {}, size, {}, screen}); DEBUG_DEBUG(DEBUG_CATEGORY_VIDEO, "desktop screen capability source=%s logical=%dx%d scale=%.3f pixels=%dx%d", qPrintable(screen->name()), screen->size().width(), screen->size().height(), screen->devicePixelRatio(), size.width(), size.height()); } for (const auto& window : QWindowCapture::capturableWindows()) sources.push_back({StreamVideoSource::Window, window.description(), {}, QVariant::fromValue(window)}); } #endif return sources; } /* Source создаётся в GUI; callbacks старого поколения не меняют новый capture. */ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) { closeSource(); #ifdef HAVE_STREAM_CAPTURE uint64_t generation; { std::lock_guard guard(m->lock); generation = ++m->sourceGeneration; m->backendOrigin = m->backendAnchor = -1; m->capturedFrames = m->convertedCaptureFrames = 0; m->lastImageCaptureAt = 0; } m->session = std::make_unique(); m->sink = std::make_unique(this); m->session->setVideoSink(m->sink.get()); connect(m->sink.get(), &QVideoSink::videoFrameChanged, this, [this, generation](const QVideoFrame& frame) { if (!frame.isValid()) { DEBUG_TRACE(DEBUG_CATEGORY_VIDEO, "desktop capture sink invalid frame generation=%llu", (unsigned long long)generation); return; } if (!m->running) return; int64_t callbackAt = monotonicUs(), capture = frame.startTime(), framePts = 0; bool convert = false; { std::lock_guard guard(m->lock); if (generation != m->sourceGeneration) return; if (!m->capturedFrames++) DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop capture first raw frame kind=%d size=%dx%d pixel=%d PTS=%lld", m->wanted.source.kind, frame.width(), frame.height(), frame.pixelFormat(), (long long)capture); if (capture >= 0) { if (m->backendOrigin < 0) { m->backendOrigin = capture; m->backendAnchor = callbackAt; DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop capture timing generation=%llu source_PTS=%lld anchor=%lld calibration=arrival-estimate", (unsigned long long)generation, (long long)capture, (long long)callbackAt); } callbackAt = m->backendAnchor + capture - m->backendOrigin; } else if (m->backendAnchor < 0) { m->backendAnchor = callbackAt; DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "desktop capture has no source PTS; using monotonic callback estimate"); } int fps = std::clamp(m->wanted.video.fps, 1, 60); convert = !m->lastImageCaptureAt || callbackAt - m->lastImageCaptureAt >= 1000000 / fps; m->previewFrame = frame; if (!convert) { m->wake.notify_all(); return; } m->lastImageCaptureAt = callbackAt; if (m->framePts < 0 && m->wanted.source.id == "portal") m->started = monotonicUs(); framePts = std::max(1, callbackAt - m->origin); } { std::lock_guard guard(m->lock); if (!m->running || generation != m->sourceGeneration) return; m->imageInput = frame; m->imageGeneration = generation; m->imagePts = framePts; m->imagePending = true; } m->imageWake.notify_one(); }, Qt::DirectConnection); auto failed = [this, generation](const QString& error) { if (generation == m->sourceGeneration) sourceFailed(m->generation, error); }; if (config.source.kind == StreamVideoSource::Camera) { QCameraDevice selected; for (const auto& camera : QMediaDevices::videoInputs()) if (camera.id() == config.source.id) selected = camera; if (selected.isNull()) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop camera disappeared"); return false; } m->camera = std::make_unique(selected); QCameraFormat best; for (const auto& format : selected.videoFormats()) { if (!stream_video_camera_supports(format.resolution(), format.minFrameRate(), format.maxFrameRate(), {QSize(config.video.width, config.video.height), config.video.fps})) continue; if (best.isNull() || format.maxFrameRate() < best.maxFrameRate()) best = format; } if (best.isNull()) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop camera mode unavailable device=%s size=%dx%d fps=%d", config.source.id.constData(), config.video.width, config.video.height, config.video.fps); return false; } m->camera->setCameraFormat(best); m->session->setCamera(m->camera.get()); DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop camera selected device=%s size=%dx%d pixel=%d capture_fps=%.3f..%.3f output_fps=%d", config.source.id.constData(), best.resolution().width(), best.resolution().height(), best.pixelFormat(), best.minFrameRate(), best.maxFrameRate(), config.video.fps); connect(m->camera.get(), &QCamera::activeChanged, this, [this, generation](bool active) { DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop camera active=%d generation=%llu", active, (unsigned long long)generation); if (!active && generation == m->sourceGeneration) sourceFailed(m->generation, text("Video capture stopped")); }, Qt::QueuedConnection); connect(m->camera.get(), &QCamera::errorOccurred, this, [failed](QCamera::Error, const QString& error) { failed(error); }, Qt::QueuedConnection); m->camera->start(); } else if (config.source.kind == StreamVideoSource::Screen) { QScreen* selected = nullptr; if (config.source.id != "portal") { for (auto* screen : QGuiApplication::screens()) if (screen == config.source.screen && screenId(screen) == config.source.id) { selected = screen; break; } if (!selected) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop screen disappeared"); return false; } } #ifdef UTUN_STREAM_PORTAL if (config.source.id == "portal") { m->portal = std::make_unique(); connect(m->portal.get(), &StreamPortalCapture::frameReady, m->sink.get(), &QVideoSink::setVideoFrame); connect(m->portal.get(), &StreamPortalCapture::errorOccurred, this, failed, Qt::QueuedConnection); m->portal->start(config.showCursor); } else #endif // Qt X11 screen capture ищет root по имени и путает мониторы; QScreen захватывает именно выбранный экран. if (selected && QGuiApplication::platformName() == "xcb") { #ifdef UTUN_STREAM_X11_CURSOR m->cursor = std::make_unique(selected); #endif m->screenGrabTimer = std::make_unique(); m->screenGrabTimer->setTimerType(Qt::PreciseTimer); connect(m->screenGrabTimer.get(), &QTimer::timeout, this, [this, generation, screen = QPointer(selected), failed] { if (generation != m->sourceGeneration || !m->running) return; int64_t captured = monotonicUs(); QImage image = screen ? screen->grabWindow(0).toImage() : QImage(); if (image.isNull()) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop X11 screen grab failed present=%d", !screen.isNull()); m->screenGrabTimer->stop(); QMetaObject::invokeMethod(this, [failed] { failed(text("Video capture stopped")); }); return; } #ifdef UTUN_STREAM_X11_CURSOR if (this->config().showCursor && !m->cursor->draw(image)) { m->screenGrabTimer->stop(); QMetaObject::invokeMethod(this, [failed] { failed(text("Could not capture the mouse cursor")); }); return; } #endif #if QT_VERSION >= QT_VERSION_CHECK(6, 8, 0) QVideoFrame frame(image); #else image = image.convertToFormat(QImage::Format_RGBA8888); QVideoFrame frame(QVideoFrameFormat(image.size(), QVideoFrameFormat::Format_RGBA8888)); if (frame.map(QVideoFrame::WriteOnly)) { for (int y = 0; y < image.height(); y++) std::memcpy(frame.bits(0) + y * frame.bytesPerLine(0), image.constScanLine(y), image.width() * 4); frame.unmap(); } else frame = {}; #endif if (!frame.isValid()) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop X11 video frame allocation failed size=%dx%d", image.width(), image.height()); m->screenGrabTimer->stop(); QMetaObject::invokeMethod(this, [failed] { failed(text("Video frame conversion failed")); }); return; } frame.setStartTime(captured); m->sink->setVideoFrame(frame); }); m->screenGrabTimer->start(std::max(1, 1000 / std::clamp(config.video.fps, 1, 60))); DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop X11 screen grab started source=%s interval_ms=%d cursor=%d supported=%d", qPrintable(config.source.name), m->screenGrabTimer->interval(), config.showCursor, cursorSupported(config.source)); } else { m->screen = std::make_unique(); m->screen->setScreen(selected); m->session->setScreenCapture(m->screen.get()); connect(m->screen.get(), &QScreenCapture::activeChanged, this, [this, generation](bool active) { DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop screen capture active=%d generation=%llu", active, (unsigned long long)generation); if (!active && generation == m->sourceGeneration) sourceFailed(m->generation, text("Video capture stopped")); }, Qt::QueuedConnection); connect(m->screen.get(), &QScreenCapture::errorOccurred, this, [failed](QScreenCapture::Error, const QString& error) { failed(error); }, Qt::QueuedConnection); m->screen->start(); } } else { auto window = config.source.window.value(); if (!window.isValid()) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop window is unavailable"); return false; } m->window = std::make_unique(); m->window->setWindow(window); m->session->setWindowCapture(m->window.get()); connect(m->window.get(), &QWindowCapture::activeChanged, this, [this, generation](bool active) { DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop window capture active=%d generation=%llu", active, (unsigned long long)generation); if (!active && generation == m->sourceGeneration) sourceFailed(m->generation, text("Video capture stopped")); }, Qt::QueuedConnection); connect(m->window.get(), &QWindowCapture::errorOccurred, this, [failed](QWindowCapture::Error, const QString& error) { failed(error); }, Qt::QueuedConnection); m->window->start(); } DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop capture started generation=%llu kind=%d source=%s target=%dx%d/%dfps", (unsigned long long)generation, config.source.kind, qPrintable(config.source.name), config.video.width, config.video.height, config.video.fps); return true; #else Q_UNUSED(config); return false; #endif } void StreamPublishEngine::closeSource() { #ifdef HAVE_STREAM_CAPTURE { std::lock_guard guard(m->lock); ++m->sourceGeneration; m->imagePending = false; m->imageInput = {}; m->frame = {}; m->latestImage = {}; m->previewFrame = {}; m->presentedPreview = {}; m->framePts = -1; } m->screenGrabTimer.reset(); #ifdef UTUN_STREAM_PORTAL m->portal.reset(); #endif #ifdef UTUN_STREAM_X11_CURSOR m->cursor.reset(); #endif emit previewVideoChanged(QVariant()); if (m->sink) disconnect(m->sink.get(), nullptr, this, nullptr); if (m->camera) { disconnect(m->camera.get(), nullptr, this, nullptr); m->camera->stop(); } if (m->screen) { disconnect(m->screen.get(), nullptr, this, nullptr); m->screen->stop(); } if (m->window) { disconnect(m->window.get(), nullptr, this, nullptr); m->window->stop(); } m->session.reset(); m->camera.reset(); m->screen.reset(); m->window.reset(); m->sink.reset(); #endif } /* Долгоживущий worker выполняет Qt Multimedia преобразование; video worker его TLS не создаёт. */ void StreamPublishEngine::imageLoop() { #ifdef HAVE_STREAM_CAPTURE for (;;) { QVideoFrame frame; uint64_t generation; int64_t pts; m->imageProgress("wait-frame"); { std::unique_lock lock(m->lock); m->imageWake.wait(lock, [this] { return m->imageStop || m->imagePending; }); if (m->imageStop) return; frame = std::move(m->imageInput); generation = m->imageGeneration; pts = m->imagePts; m->imagePending = false; } int64_t convertAt = monotonicUs(); m->imageProgress("convert-frame"); QImage image = frame.toImage(); // Отметка ставится перед деструкторами image/frame, включая return/continue внутри итерации. auto releaseFrame = qScopeGuard([this] { m->imageProgress("release-frame"); }); m->imageFrames++; m->imageProgress("transform-image"); if (image.isNull()) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop captured frame conversion failed generation=%llu format=%d size=%dx%d", (unsigned long long)generation, frame.pixelFormat(), frame.width(), frame.height()); QMetaObject::invokeMethod(this, [this, generation] { sourceFailed(generation, text("Video frame conversion failed")); }); continue; } #if QT_VERSION >= QT_VERSION_CHECK(6, 7, 0) if (frame.rotation() != QtVideo::Rotation::None) image = image.transformed(QTransform().rotate(static_cast(frame.rotation()))); if (frame.mirrored()) image = image.transformed(QTransform().scale(-1, 1)); #else if (frame.rotationAngle()) image = image.transformed(QTransform().rotate(frame.rotationAngle())); #endif m->imageProgress("publish-image"); { std::lock_guard guard(m->lock); if (m->imageStop) return; if (!m->running || generation != m->sourceGeneration) continue; if (m->framePts < 0 && m->wanted.source.id == "portal") m->started = monotonicUs(); m->latestImage = image; m->frame = std::move(image); m->framePts = pts; m->frameGeneration = m->generation; if (!m->convertedCaptureFrames++) DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop first image conversion complete generation=%llu size=%dx%d callback_tid=%llu gui_thread=%d convert_us=%lld", (unsigned long long)generation, m->latestImage.width(), m->latestImage.height(), (unsigned long long)(quintptr)QThread::currentThreadId(), QThread::currentThread() == qApp->thread(), (long long)(monotonicUs() - convertAt)); } m->wake.notify_all(); } #endif } bool StreamPublishEngine::start(uint64_t group, const StreamPublishConfig& config) { if (m->destroying) { DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop publication start during destruction"); return false; } if (active()) { DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop publication already active"); emit errorOccurred(text("A stream is already running")); return false; } if (!supported() || !codecs().contains(config.video.codec)) { DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "desktop capture/encoder unavailable codec=%d", config.video.codec); emit errorOccurred(text("Video capture or encoder is unavailable")); return false; } if (config.audio && !m_microphone.acquire()) { DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop publication blocked: microphone owned by another operation"); emit errorOccurred(text("The microphone is used by another operation")); return false; } { std::lock_guard guard(m->lock); m->wanted = config; m->good = {}; m->hasGood = m->rollingBack = false; m->generation++; m->confirmedGeneration = 0; m->videoReady = false; m->softwareRequest = 0; m->softwareCodec = 0; m->lastAudioAt = 0; m->origin = m->started = monotonicUs(); m->stats = {}; } m->convertedFrames = m->encodedFrames = m->deliveredFrames = m->repeatedFrames = 0; m->videoProgress("starting"); m->captureReport = 0; m->id = QRandomGenerator::global()->generate64(); if (!m->id) m->id = 1; m->group = group; m->audio.reset(); m->audioSequence = 0; m->muted = false; stream_media_format format{(uint16_t)config.video.width, (uint16_t)config.video.height, (uint16_t)config.video.rotation, (uint8_t)config.video.codec, (uint8_t)(config.audio ? STREAM_MEDIA_OPUS_STEREO : 0), 1}; if (!gui_stream_publish_open(m->id, group, format)) { m->id = m->group = 0; m_microphone.release(); emit errorOccurred(text("Could not start the stream")); return false; } m->running = true; m->workersJoined = m->videoExited = m->audioExited = false; try { // Отдельный join ждёт и native TLS Qt; отметки выхода из функции/thread_local для этого недостаточно. m->joinThread = std::thread([this] { { std::unique_lock lock(m->lock); m->wake.wait(lock, [this] { return !m->running; }); } if (m->videoThread.joinable()) m->videoThread.join(); m->videoExited = true; DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop stop video thread exited"); if (m->audioThread.joinable()) m->audioThread.join(); m->audioExited = true; DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop stop audio thread exited"); m->workersJoined = true; }); m->videoThread = std::thread([this, config] { videoLoop(config); m->videoProgress("release-thread-resources"); DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop video loop returned; releasing thread resources"); }); if (config.audio) m->audioThread = std::thread(&StreamPublishEngine::audioLoop, this); } catch (const std::system_error& error) { publicationFailed(m->id, QString::fromUtf8(error.what())); return false; } if (!openSource(config)) { sourceFailed(m->generation, text("Could not open the selected capture device")); return false; } if (config.audio && !openMicrophone()) { publicationFailed(m->id, text("Could not start the microphone. Select another microphone or disable stream audio.")); return false; } m->microphoneStarted = monotonicUs(); emit stateChanged(config.source.id == "portal" ? text("Select a screen in the system dialog") : text("Starting stream…")); emit configurationChanged(); return true; } bool StreamPublishEngine::reconfigure(const StreamPublishConfig& config) { if (!m->running) return false; StreamPublishConfig previous = this->config(); if (previous.source.id == "portal" && config.source.sameSource(previous.source) && config.showCursor != previous.showCursor) { DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "Wayland capture cursor mode cannot change during a portal session"); emit errorOccurred(text("Change cursor visibility before starting screen sharing")); return false; } if (config.audio != previous.audio || config.microphone != previous.microphone) { DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop microphone selection changed during live publication"); emit errorOccurred(text("Change the microphone before starting the stream")); return false; } if (config.showCursor != previous.showCursor) DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop cursor selected enabled=%d supported=%d", config.showCursor, cursorSupported(config.source)); bool sourceChange = !config.source.sameSource(previous.source) || (config.source.kind == StreamVideoSource::Camera && (config.video.width != previous.video.width || config.video.height != previous.video.height || config.video.fps != previous.video.fps)); { std::lock_guard guard(m->lock); m->wanted = config; m->generation++; m->started = monotonicUs(); m->softwareRequest = 0; #ifdef HAVE_STREAM_CAPTURE if (!sourceChange) { m->frame = m->latestImage; m->frameGeneration = m->generation; } #endif } if (sourceChange && !openSource(config)) { sourceFailed(m->generation, text("Could not open the selected video source")); return false; } #ifdef HAVE_STREAM_CAPTURE if (m->screenGrabTimer) m->screenGrabTimer->setInterval(std::max(1, 1000 / std::clamp(config.video.fps, 1, 60))); #endif m->wake.notify_all(); emit stateChanged(text("Applying video settings…")); return true; } void StreamPublishEngine::sourceFailed(uint64_t generation, const QString& error) { StreamPublishConfig restore; bool rollback; { std::lock_guard guard(m->lock); if (generation != m->generation || !m->running) return; rollback = m->hasGood && !m->rollingBack; restore = m->good; m->rollingBack = true; } DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop capture failed generation=%llu rollback=%d reason=%s", (unsigned long long)generation, rollback, qPrintable(error)); emit errorOccurred(rollback ? text("Video settings were restored: %1").arg(error) : error); if (rollback) reconfigure(restore); else stop(); } /* Ошибку общего audio worker нельзя исправить заменой только video encoder. */ void StreamPublishEngine::publicationFailed(uint64_t id, const QString& error) { if (id != m->id || !m->running) return; DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop publication failed id=%llu reason=%s", (unsigned long long)id, qPrintable(error)); emit errorOccurred(error); stop(); } /* Сначала callbacks; ожидание workers оставляет GUI доступным для освобождения Qt thread-local ресурсов. */ void StreamPublishEngine::stop() { if (!m->running.exchange(false)) return; int64_t started = monotonicUs(); m->stopping = true; m->finSent = false; m->stopStarted = m->stopReport = started; DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop stop begin id=%llu video_stage=%s stage_age_ms=%lld converted=%llu encoded=%llu delivered=%llu", (unsigned long long)m->id, m->videoStage.load(), (long long)((started - m->videoStageAt.load()) / 1000), (unsigned long long)m->convertedFrames.load(), (unsigned long long)m->encodedFrames.load(), (unsigned long long)m->deliveredFrames.load()); { std::lock_guard guard(m->lock); m->softwareRequest = 0; } m->wake.notify_all(); DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop stop closing video source"); closeSource(); DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop stop video source closed elapsed_ms=%lld", (long long)((monotonicUs() - started) / 1000)); m_capture.closeAsync(); DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop stop waiting asynchronously for workers stage=%s", m->videoStage.load()); emit stateChanged(text("Finishing stream…")); } /* FIN не обгоняет принятые access units; owner жив до обоих join, включая native TLS destructors. */ bool StreamPublishEngine::finishStop() { if (!m->stopping) return true; if (monotonicUs() - m->stopReport >= 5000000) { m->stopReport = monotonicUs(); DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "desktop stop still waiting elapsed_ms=%lld video_stage=%s video=%d audio=%d microphone=%d", (long long)((m->stopReport - m->stopStarted) / 1000), m->videoStage.load(), !m->videoExited.load(), !m->audioExited.load(), m_capture.closing()); } if (m->joinThread.joinable()) { if (!m->workersJoined) return false; m->joinThread.join(); } if (!m->finSent) { gui_stream_publish_stop(m->id); m->finSent = true; DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop workers stopped id=%llu elapsed_ms=%lld microphone_closing=%d; draining publisher", (unsigned long long)m->id, (long long)((monotonicUs() - m->stopStarted) / 1000), m_capture.closing()); } if (m_capture.closing()) return false; m_microphone.release(); m->stopping = false; return true; } uint64_t StreamPublishEngine::softwareEncodingRequest() const { std::lock_guard guard(m->lock); return m->softwareRequest; } /* Разрешение относится к кодеку текущей трансляции; устаревший ответ не запускает новый режим. */ void StreamPublishEngine::answerSoftwareEncoding(uint64_t generation, bool allow) { { std::lock_guard guard(m->lock); if (!m->running || !generation || m->softwareRequest != generation || m->generation != generation) { DEBUG_DEBUG(DEBUG_CATEGORY_VIDEO, "desktop CPU response ignored generation=%llu", (unsigned long long)generation); return; } if (allow) { m->softwareCodec = m->wanted.video.codec; m->softwareRequest = 0; m->started = monotonicUs(); } } DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop CPU encoding decision generation=%llu allow=%d", (unsigned long long)generation, allow); if (!allow) sourceFailed(generation, text("Hardware encoding is unavailable. CPU encoding was declined.")); else { m->wake.notify_all(); emit stateChanged(text("Starting stream…")); } } bool StreamPublishEngine::openMicrophone() { return m_capture.open(config().microphone, AudioCapture::Role::Stream, 2, m->id, [this](int channels, int native) { m->audioAnchor = -1; m->audioFrames = 0; m->channels = channels; if (native == 1) DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop stream input is mono; stereo transport duplicates the input"); DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop microphone opened requested=%d backend_channels=%d rate=48000", channels, native); emit audioChannelsChanged(native == 1 ? 1 : channels); return true; }, [] {}, [this](const AudioCapture::Frame& frame) { captureAudio(frame); }); } /* Hardware callback копирует максимум 20ms за элемент, без codec/DSP/allocations. */ void StreamPublishEngine::captureAudio(const AudioCapture::Frame& frame) { const auto* pcm = frame.pcm; unsigned frames = frame.frames; if (!m->running) return; if (!pcm || !frames || frame.missing) { m->audioMissing++; return; } m->lastAudioAt.store(monotonicUs(), std::memory_order_release); m->wake.notify_all(); { std::lock_guard guard(m->lock); if (!m->videoReady) return; } int64_t first = frame.timeUs - m->origin; if (frame.measured || m->audioAnchor < 0 || frame.missing) { m->audioAnchor = first; m->audioFrames = 0; } first = m->audioAnchor + (int64_t)m->audioFrames * 1000000 / 48000; m->audioFrames += frames; for (unsigned offset = 0; offset < frames;) { Impl::Audio record; record.frames = std::min(960u, frames - offset); record.pts = std::max(0, first + (int64_t)offset * 1000000 / 48000); record.created = monotonicUs(); record.sequence = ++m->audioSequence; for (unsigned i = 0; i < record.frames; i++) { record.pcm[2 * i] = pcm ? pcm[(offset + i) * m->channels] : 0; record.pcm[2 * i + 1] = pcm ? pcm[(offset + i) * m->channels + (m->channels == 2 ? 1 : 0)] : 0; } if (!m->audio.push(record)) m->audioDropped++; offset += record.frames; } m->wake.notify_all(); } void StreamPublishEngine::audioLoop() { const uint64_t id = m->id; { std::unique_lock lock(m->lock); m->wake.wait(lock, [this] { return !m->running || m->videoReady; }); } if (!m->running) return; auto* encoder = opus_codec_encoder_create(48000, 2); auto* compressor = audio_compressor_create(); int result = encoder && compressor ? opus_codec_encoder_application_set(encoder, OPUS_CODEC_APP_AUDIO) : -1; if (result >= 0) result = opus_codec_encoder_bitrate_set(encoder, 128000); if (result < 0) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop stream audio encoder initialization failed"); QMetaObject::invokeMethod(this, [this, id] { publicationFailed(id, text("Audio encoder initialization failed")); }); } else { int16_t pcm[1920]{}; unsigned used = 0; uint64_t sequence = 0; int64_t pts = 0, expected = -1, lastSent = -1; int settings[3] = {-1, -1, -1}; while (m->running) { Impl::Audio record; if (!m->audio.pop(record)) { std::unique_lock lock(m->lock); m->wake.wait_for(lock, std::chrono::milliseconds(5)); continue; } if (monotonicUs() - record.created > 250000) { m->audioDropped++; used = 0; continue; } if ((sequence && record.sequence != sequence + 1) || (expected >= 0 && std::abs(record.pts - expected) > 5000)) { DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop stream PCM discontinuity sequence=%llu->%llu PTS=%lld expected=%lld", (unsigned long long)sequence, (unsigned long long)record.sequence, (long long)record.pts, (long long)expected); used = 0; audio_compressor_reset(compressor); } sequence = record.sequence; expected = record.pts + (int64_t)record.frames * 1000000 / 48000; unsigned offset = 0; while (offset < record.frames) { if (!used) pts = record.pts + (int64_t)offset * 1000000 / 48000; unsigned take = std::min(960 - used, record.frames - offset); memcpy(pcm + 2 * used, record.pcm + 2 * offset, take * 2 * sizeof(int16_t)); used += take; offset += take; if (used < 960) continue; stream_publisher_status stats{}; gui_stream_publish_status(m->id, stats); int wanted[] = {stats.compressor_enabled, stats.compressor_max_gain_db ? stats.compressor_max_gain_db : 15, stats.compressor_rise_rate ? stats.compressor_rise_rate : 2}; if (!std::equal(wanted, wanted + 3, settings)) { audio_compressor_config_t config{48000, 2, 20, 200, 0, (float)wanted[1], (float)wanted[2], wanted[2] * 5.0f, 0.25f}; if (audio_compressor_configure(compressor, &config) < 0) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop stream compressor configuration failed"); result = -1; break; } audio_compressor_set_enabled(compressor, wanted[0]); std::copy(wanted, wanted + 3, settings); DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop stream compressor enabled=%d max_gain=%d rise=%d stereo=1 delay=0", wanted[0], wanted[1], wanted[2]); } if (m->muted) memset(pcm, 0, sizeof(pcm)); uint8_t packet[STREAM_MEDIA_MAX_OPUS]; result = audio_compressor_process_frame(compressor, pcm, pcm, 1920); if (!result) result = opus_codec_encode(encoder, pcm, 960, packet, sizeof(packet)); if (result <= 0) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop stream audio encoding failed result=%d", result); result = -1; break; } if (pts > lastSent) { gui_stream_publish_feed(m->id, false, pts, false, QByteArray(reinterpret_cast(packet), result)); lastSent = pts; } else DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop stream audio stale PTS=%lld last=%lld", (long long)pts, (long long)lastSent); used = 0; } if (result < 0) { QMetaObject::invokeMethod(this, [this, id] { publicationFailed(id, text("Audio encoding failed")); }); break; } } } opus_codec_encoder_destroy(encoder); audio_compressor_destroy(compressor); } void StreamPublishEngine::videoLoop(StreamPublishConfig initial) { #ifdef HAVE_STREAM_CAPTURE std::unique_ptr encoder; StreamPublishConfig encoding; QImage lastImage; uint64_t current = 0; uint32_t revision = 0; int64_t lastPts = -1, lastKey = -1000000, next = 0, report = 0; unsigned count = 0, skipped = 0; while (m->running) { m->videoProgress("waiting-frame"); StreamPublishConfig config; uint64_t generation; QImage frame; int64_t capture; { std::unique_lock lock(m->lock); config = m->wanted; generation = m->generation; if (config.audio && !m->lastAudioAt.load(std::memory_order_acquire)) { m->wake.wait_for(lock, std::chrono::milliseconds(5)); continue; } int64_t now = monotonicUs(); if (next > now) { m->wake.wait_for(lock, std::chrono::microseconds(next - now)); continue; } if (m->frameGeneration == generation) { frame = std::move(m->frame); capture = m->framePts; } else capture = -1; } // Размер кадра точнее QScreen при HiDPI и единственный доступный после выбора Wayland portal. if (config.source.kind == StreamVideoSource::Screen && !frame.isNull() && config.source.size != frame.size()) { QSize size = stream_video_screen_size(frame.size(), config.nativeSize ? 4096 : std::min(config.video.width, config.video.height)); if (size.isEmpty()) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop screen invalid captured size=%dx%d", frame.width(), frame.height()); QMetaObject::invokeMethod(this, [this, generation] { sourceFailed(generation, text("Invalid stream video dimensions")); }); continue; } config.source.size = frame.size(); config.video.width = size.width(); config.video.height = size.height(); if (config.automaticBitrate) config.video.bitrate = std::clamp(size.width() * size.height() * config.video.fps / 11, 250000, 16000000); DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop screen size resolved source=%s raw=%dx%d target=%dx%d native=%d generation=%llu", qPrintable(config.source.name), frame.width(), frame.height(), size.width(), size.height(), config.nativeSize, (unsigned long long)generation); QMetaObject::invokeMethod(this, [this, generation, config] { if (generation == m->generation && m->running && reconfigure(config)) emit sourcesChanged(); }); // GUI изменяет конфигурацию и возвращает последний preview frame уже с новым поколением. std::unique_lock lock(m->lock); m->wake.wait_for(lock, std::chrono::milliseconds(100), [this, generation] { return !m->running || m->generation != generation; }); continue; } if (config.source.kind == StreamVideoSource::Screen && config.source.size.isEmpty() && frame.isNull()) { std::unique_lock lock(m->lock); m->wake.wait_for(lock, std::chrono::milliseconds(5)); continue; } if (generation != current) { auto comparable = config.video; comparable.bitrate = encoding.video.bitrate; bool onlyBitrate = encoder && comparable == encoding.video && config.source.sameSource(encoding.source); if (onlyBitrate && encoder->setBitrate(config.video.bitrate)) { current = generation; encoding = config; { std::lock_guard guard(m->lock); if (m->generation == generation) { m->good = config; m->hasGood = true; m->confirmedGeneration = generation; } } QMetaObject::invokeMethod(this, [this, generation] { if (m->running && generation == m->generation) { emit configurationChanged(); emit stateChanged(text("Live")); } }); } else { auto candidate = std::make_unique(); m->videoProgress("open-hardware-encoder"); bool opened = candidate->open(config.video, LiveVideoEncoder::Backend::HardwareOnly); if (!opened && LiveVideoEncoder::available(LiveVideoEncoder::Backend::SoftwareOnly).contains(config.video.codec)) { QString details = candidate->error(); std::unique_lock lock(m->lock); if (!m->running || generation != m->generation) continue; if (m->softwareCodec != config.video.codec) { m->softwareRequest = generation; m->videoProgress("waiting-cpu-consent"); DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "desktop CPU confirmation required generation=%llu codec=%d reason=%s", (unsigned long long)generation, config.video.codec, qPrintable(details)); QMetaObject::invokeMethod(this, [this, generation, codec = config.video.codec, details] { if (softwareEncodingRequest() != generation || !m->running) return; emit stateChanged(text("Waiting for permission to encode on the CPU…")); emit softwareEncodingRequested(generation, LiveVideoEncoder::codecName(codec), details); }); m->wake.wait(lock, [this, generation] { return !m->running || m->generation != generation || m->softwareRequest != generation; }); } if (!m->running || generation != m->generation) continue; lock.unlock(); m->videoProgress("open-software-encoder"); opened = candidate->open(config.video, LiveVideoEncoder::Backend::SoftwareOnly); } if (!opened) { QString error = text("Could not start video encoding for %1. Choose another codec or video mode.") .arg(LiveVideoEncoder::codecName(config.video.codec)); QMetaObject::invokeMethod(this, [this, generation, error] { sourceFailed(generation, error); }); std::unique_lock lock(m->lock); m->wake.wait_for(lock, std::chrono::milliseconds(100), [this, generation] { return !m->running || m->generation != generation; }); continue; } // Начальный формат publisher уже зарегистрирован; уточнение размера требует новой revision даже до первого KEY. revision = revision ? revision + 1 : config.video == initial.video ? 1 : 2; m->videoProgress("replace-encoder"); encoder = std::move(candidate); current = generation; encoding = config; lastImage = {}; lastKey = -1000000; { std::lock_guard guard(m->lock); if (!m->running || generation != m->generation) continue; m->videoReady = true; } m->wake.notify_all(); DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop video prepared generation=%llu revision=%u encoder=%s", (unsigned long long)generation, revision, qPrintable(encoder->name())); } } bool screen = config.source.kind != StreamVideoSource::Camera; if (!frame.isNull()) { if (!screen && (capture <= lastPts || monotonicUs() - m->origin - capture > 250000)) { skipped++; continue; } lastImage = std::move(frame); m->convertedFrames++; } else if (!screen || lastImage.isNull()) { std::unique_lock lock(m->lock); m->wake.wait_for(lock, std::chrono::milliseconds(5)); continue; } else m->repeatedFrames++; m->videoProgress("publisher-status"); int64_t now = monotonicUs(), pts = screen ? now - m->origin : capture; next = now + 1000000 / config.video.fps; stream_publisher_status status; bool needsKey = gui_stream_publish_status(m->id, status) && status.need_key; bool key = pts - lastKey >= 1000000 || needsKey; QList output; m->videoProgress("encode-frame"); if (!encoder->encode(lastImage, std::max(lastPts + 1, pts), key, output)) { QString error = encoder->error(); QMetaObject::invokeMethod(this, [this, generation, error] { sourceFailed(generation, error); }); continue; } m->encodedFrames += output.size(); m->videoProgress("publish-frame"); stream_media_format format{(uint16_t)config.video.width, (uint16_t)config.video.height, (uint16_t)config.video.rotation, (uint8_t)config.video.codec, (uint8_t)(config.audio ? STREAM_MEDIA_OPUS_STEREO : 0), revision}; for (const auto& encoded : output) { if (gui_stream_publish_feed(m->id, true, encoded.pts, encoded.key, encoded.data, &format) == 1) { lastPts = encoded.pts; if (encoded.key) lastKey = encoded.pts; count++; m->deliveredFrames++; if (encoded.key) { bool first; { std::lock_guard guard(m->lock); first = !m->hasGood || m->good.video != config.video || m->rollingBack; first |= m->confirmedGeneration != generation; if (m->generation == generation) { m->good = config; m->hasGood = true; m->rollingBack = false; m->confirmedGeneration = generation; } } if (first) { DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop first KEY committed generation=%llu revision=%u PTS=%llu bytes=%lld", (unsigned long long)generation, revision, (unsigned long long)encoded.pts, (long long)encoded.data.size()); QMetaObject::invokeMethod(this, [this, generation] { if (!m->running || generation != m->generation) return; emit configurationChanged(); emit stateChanged(text("Live")); }); } } } else skipped++; } if (now - report >= 1000000) { DEBUG_DEBUG(DEBUG_CATEGORY_VIDEO, "desktop capture generation=%llu encoded=%u skipped=%u frame_age=%lldus encoder=%s", (unsigned long long)generation, count, skipped, (long long)(screen ? 0 : now - m->origin - capture), qPrintable(encoder->name())); report = now; count = skipped = 0; } m->videoProgress("release-frame"); } m->videoProgress("close-encoder"); encoder.reset(); m->videoProgress("stopped"); DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop video worker stopped"); #else Q_UNUSED(initial); #endif } void StreamPublishEngine::tick() { if (!active()) return; if (!finishStop()) return; if (!m->running) m_microphone.release(); stream_publisher_status status; if (!gui_stream_publish_status(m->id, status)) { stop(); if (!finishStop()) return; m_microphone.release(); m->id = m->group = 0; emit stateChanged(text("Chat service is stopped")); return; } { std::lock_guard guard(m->lock); m->stats = status; } emit telemetryChanged(status); if (status.state == STREAM_PUBLISHER_ENDED || status.state == STREAM_PUBLISHER_FAILED) { stop(); if (!finishStop()) return; m_microphone.release(); m->id = m->group = 0; if (status.state == STREAM_PUBLISHER_FAILED) emit errorOccurred(text("Stream publication failed")); emit stateChanged(text("Stopped")); emit configurationChanged(); return; } if (m->running) { int64_t audioAt = m->lastAudioAt.load(std::memory_order_acquire); if (m->wanted.audio && monotonicUs() - (audioAt ? audioAt : m->microphoneStarted) >= 1000000) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop microphone has no PCM id=%llu received=%d idle_us=%lld", (unsigned long long)m->id, audioAt != 0, (long long)(monotonicUs() - (audioAt ? audioAt : m->microphoneStarted))); publicationFailed(m->id, text("The microphone is not delivering audio. Select another microphone or disable stream audio.")); return; } bool confirmed, waitingSelection; int64_t started; { std::lock_guard guard(m->lock); confirmed = m->confirmedGeneration == m->generation; started = m->started; waitingSelection = m->wanted.source.id == "portal" && m->framePts < 0; } if (!confirmed && !waitingSelection && !softwareEncodingRequest() && monotonicUs() - started > 8000000) sourceFailed(m->generation, text("Timed out waiting for the first video frame")); { std::lock_guard guard(m->lock); if (monotonicUs() - m->captureReport >= 5000000) { m->captureReport = monotonicUs(); DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop capture progress source_generation=%llu video_generation=%llu confirmed=%llu raw_frames=%llu raw_PTS=%lld " "converted=%llu encoded=%llu delivered=%llu repeated=%llu stage=%s stage_age_ms=%lld", (unsigned long long)m->sourceGeneration, (unsigned long long)m->generation, (unsigned long long)m->confirmedGeneration, (unsigned long long)m->capturedFrames, (long long)m->framePts, (unsigned long long)m->convertedFrames.load(), (unsigned long long)m->encodedFrames.load(), (unsigned long long)m->deliveredFrames.load(), (unsigned long long)m->repeatedFrames.load(), m->videoStage.load(), (long long)((monotonicUs() - m->videoStageAt.load()) / 1000)); } } } unsigned dropped = m->audioDropped.exchange(0); if (dropped) DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop audio capture discarded blocks=%u", dropped); unsigned missing = m->audioMissing.exchange(0); if (missing) DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop microphone callbacks lack PCM count=%u; waiting for input", missing); }