diff --git a/tools/chatgui/CMakeLists.txt b/tools/chatgui/CMakeLists.txt index 02146ae4..51435d22 100644 --- a/tools/chatgui/CMakeLists.txt +++ b/tools/chatgui/CMakeLists.txt @@ -129,6 +129,23 @@ endif() # uTun core library (C, built from ../src/ and ../lib/) add_subdirectory(libutun) +# Общий sender остаётся в libutun; Qt захватывает видео, FFmpeg/Opus работают в publisher workers. +add_library(chatgui_stream_publish STATIC src/stream_video_encoder.cpp src/stream_publish_engine.cpp src/stream_publish_window.cpp) +target_link_libraries(chatgui_stream_publish PUBLIC ${QT_LIBS} utun PRIVATE ${FFMPEG_TARGET} ${FFMPEG_LIBRARIES}) +target_include_directories(chatgui_stream_publish PRIVATE ${FFMPEG_INCLUDE_DIRS} ${CMAKE_SOURCE_DIR}/../../lib/libopus/include) +if(FFMPEG_FOUND) + target_compile_definitions(chatgui_stream_publish PRIVATE HAVE_FFMPEG) +endif() +if(QT_PACKAGE STREQUAL "Qt6" AND FFMPEG_FOUND) + find_package(Qt6 6.6 COMPONENTS Multimedia MultimediaWidgets QUIET) + if(TARGET Qt6::Multimedia AND TARGET Qt6::MultimediaWidgets) + target_compile_definitions(chatgui_stream_publish PUBLIC HAVE_STREAM_CAPTURE) + target_link_libraries(chatgui_stream_publish PUBLIC Qt6::Multimedia Qt6::MultimediaWidgets) + else() + message(STATUS "Qt Multimedia 6.6+ missing: stream publication unavailable") + endif() +endif() + add_library(chatgui_notifications STATIC src/message_notification.cpp src/user_activity_monitor.cpp) target_link_libraries(chatgui_notifications PUBLIC ${QT_PACKAGE}::Widgets PRIVATE utun) if(WIN32) @@ -287,7 +304,7 @@ if(TARGET X11::Xss) elseif(UNIX AND NOT APPLE) message(STATUS "X11 screen saver library not found: X11 activity uses ChatGUI input") endif() -target_link_libraries(vibechat PRIVATE chatgui_notifications) +target_link_libraries(vibechat PRIVATE chatgui_notifications chatgui_stream_publish) if(WIN32) target_link_libraries(vibechat PRIVATE ${QT_LIBS} rlottie::rlottie ZLIB::ZLIB OpenSSL::Crypto ZXing::ZXing utun utun_voice pthread user32 ${FFMPEG_TARGET} ${FFMPEG_LIBRARIES}) diff --git a/tools/chatgui/src/stream_publish_engine.cpp b/tools/chatgui/src/stream_publish_engine.cpp index 405a4fbd..bc6353d3 100644 --- a/tools/chatgui/src/stream_publish_engine.cpp +++ b/tools/chatgui/src/stream_publish_engine.cpp @@ -40,7 +40,7 @@ struct StreamPublishEngine::Impl { std::atomic running{false}, muted{false}; std::atomic audioDropped{0}; AudioEventQueue audio; - uint64_t id = 0, group = 0, generation = 0, frameGeneration = 0, audioSequence = 0; + uint64_t id = 0, group = 0, generation = 0, sourceGeneration = 0, confirmedGeneration = 0, frameGeneration = 0, audioSequence = 0; int64_t origin = 0, started = 0, framePts = -1, backendOrigin = -1, backendAnchor = -1; int channels = 2; bool probeReady = false, probeDelivered = false, hasGood = false, rollingBack = false; @@ -55,6 +55,7 @@ struct StreamPublishEngine::Impl { std::unique_ptr screen; std::unique_ptr window; QVideoFrame frame; + QVideoFrame previewFrame; #endif }; @@ -105,13 +106,13 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) { closeSource(); #ifdef HAVE_STREAM_CAPTURE uint64_t generation; - { std::lock_guard guard(m->lock); generation = m->generation; m->backendOrigin = m->backendAnchor = -1; } + { std::lock_guard guard(m->lock); generation = ++m->sourceGeneration; m->backendOrigin = m->backendAnchor = -1; } m->session = std::make_unique(); m->sink = std::make_unique(); m->session->setVideoSink(m->sink.get()); connect(m->sink.get(), &QVideoSink::videoFrameChanged, this, [this, generation](const QVideoFrame& frame) { if (!m->running || !frame.isValid()) return; std::lock_guard guard(m->lock); - if (generation != m->generation) return; + if (generation != m->sourceGeneration) return; int64_t now = monotonicUs(), capture = frame.startTime(); if (capture >= 0) { if (m->backendOrigin < 0) { @@ -124,9 +125,12 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) { m->backendAnchor = now; DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "desktop capture has no source PTS; using monotonic callback estimate"); } - m->frame = frame; m->framePts = std::max(1, now - m->origin); m->frameGeneration = generation; + m->frame = m->previewFrame = frame; m->framePts = std::max(1, now - m->origin); m->frameGeneration = m->generation; m->wake.notify_all(); - }); + }, Qt::DirectConnection); + auto failed = [this, generation](const QString& error) { + if (generation == m->sourceGeneration) sourceFailed(m->generation, error); + }; if (config.source.kind == StreamVideoSource::Camera) { QCameraDevice selected; for (const auto& camera : QMediaDevices::videoInputs()) if (camera.id() == config.source.id) selected = camera; @@ -141,7 +145,7 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) { } if (best.isNull()) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop camera lacks requested fps=%d", config.video.fps); return false; } m->camera->setCameraFormat(best); m->session->setCamera(m->camera.get()); - connect(m->camera.get(), &QCamera::errorOccurred, this, [this, generation](QCamera::Error, const QString& error) { sourceFailed(generation, error); }); + connect(m->camera.get(), &QCamera::errorOccurred, this, [failed](QCamera::Error, const QString& error) { failed(error); }); m->camera->start(); } else if (config.source.kind == StreamVideoSource::Screen) { m->screen = std::make_unique(); @@ -152,13 +156,13 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) { m->screen->setScreen(selected); } m->session->setScreenCapture(m->screen.get()); - connect(m->screen.get(), &QScreenCapture::errorOccurred, this, [this, generation](QScreenCapture::Error, const QString& error) { sourceFailed(generation, error); }); + connect(m->screen.get(), &QScreenCapture::errorOccurred, this, [failed](QScreenCapture::Error, const QString& error) { failed(error); }); m->screen->start(); } else { auto window = config.source.window.value(); if (!window.isValid()) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop window is unavailable"); return false; } m->window = std::make_unique(); m->window->setWindow(window); m->session->setWindowCapture(m->window.get()); - connect(m->window.get(), &QWindowCapture::errorOccurred, this, [this, generation](QWindowCapture::Error, const QString& error) { sourceFailed(generation, error); }); + connect(m->window.get(), &QWindowCapture::errorOccurred, this, [failed](QWindowCapture::Error, const QString& error) { failed(error); }); m->window->start(); } DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop capture started generation=%llu kind=%d source=%s target=%dx%d/%dfps", @@ -175,7 +179,7 @@ void StreamPublishEngine::closeSource() { if (m->screen) { disconnect(m->screen.get(), nullptr, this, nullptr); m->screen->stop(); } if (m->window) { disconnect(m->window.get(), nullptr, this, nullptr); m->window->stop(); } m->session.reset(); m->camera.reset(); m->screen.reset(); m->window.reset(); m->sink.reset(); - std::lock_guard guard(m->lock); m->frame = {}; m->framePts = -1; + std::lock_guard guard(m->lock); m->frame = {}; m->previewFrame = {}; m->framePts = -1; #endif } @@ -187,7 +191,7 @@ bool StreamPublishEngine::start(uint64_t group, const StreamPublishConfig& confi emit errorOccurred(text("The microphone is used by a call, radio or recording")); return false; } { std::lock_guard guard(m->lock); - m->wanted = config; m->good = {}; m->hasGood = m->rollingBack = false; m->generation++; + m->wanted = config; m->good = {}; m->hasGood = m->rollingBack = false; m->generation++; m->confirmedGeneration = 0; m->origin = m->started = monotonicUs(); m->preview = {}; m->stats = {}; } m->id = QRandomGenerator::global()->generate64(); if (!m->id) m->id = 1; m->group = group; m->audio.reset(); m->audioSequence = 0; m->muted = false; @@ -213,8 +217,15 @@ bool StreamPublishEngine::reconfigure(const StreamPublishConfig& config) { if (config.audio != previous.audio || config.microphone != previous.microphone) { emit errorOccurred(text("Change the microphone before starting the stream")); return false; } - { std::lock_guard guard(m->lock); m->wanted = config; m->generation++; m->started = monotonicUs(); } - if (!openSource(config)) { sourceFailed(m->generation, text("Could not open the selected video source")); return false; } + bool sourceChange = config.source.kind != previous.source.kind || config.source.id != previous.source.id + || config.source.window != previous.source.window || (config.source.kind == StreamVideoSource::Camera + && (config.video.width != previous.video.width || config.video.height != previous.video.height || config.video.fps != previous.video.fps)); + { std::lock_guard guard(m->lock); m->wanted = config; m->generation++; m->started = monotonicUs(); +#ifdef HAVE_STREAM_CAPTURE + if (!sourceChange) { m->frame = m->previewFrame; m->frameGeneration = m->generation; } +#endif + } + if (sourceChange && !openSource(config)) { sourceFailed(m->generation, text("Could not open the selected video source")); return false; } m->wake.notify_all(); emit stateChanged(text("Applying video settings…")); return true; } @@ -311,7 +322,7 @@ void StreamPublishEngine::audioLoop() { if (!std::equal(wanted, wanted + 3, settings)) { audio_compressor_config_t config{48000, 2, 20, 200, 0, (float)wanted[1], (float)wanted[2], wanted[2] * 5.0f, 0.25f}; if (audio_compressor_configure(compressor, &config) < 0) { - DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop stream compressor configuration failed"); break; + DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop stream compressor configuration failed"); result = -1; break; } audio_compressor_set_enabled(compressor, wanted[0]); std::copy(wanted, wanted + 3, settings); DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop stream compressor enabled=%d max_gain=%d rise=%d stereo=1 delay=0", @@ -321,7 +332,7 @@ void StreamPublishEngine::audioLoop() { uint8_t packet[STREAM_MEDIA_MAX_OPUS]; result = audio_compressor_process_frame(compressor, pcm, pcm, 1920); if (!result) result = opus_codec_encode(encoder, pcm, 960, packet, sizeof(packet)); - if (result <= 0) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop stream audio encoding failed result=%d", result); break; } + if (result <= 0) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop stream audio encoding failed result=%d", result); result = -1; break; } if (pts > lastSent) { gui_stream_publish_feed(m->id, false, pts, false, QByteArray(reinterpret_cast(packet), result)); lastSent = pts; } else DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop stream audio stale PTS=%lld last=%lld", (long long)pts, (long long)lastSent); @@ -365,6 +376,12 @@ void StreamPublishEngine::videoLoop() { if (capture <= lastPts || monotonicUs() - m->origin - capture > 250000) { skipped++; continue; } lastImage = frame.toImage(); if (lastImage.isNull()) { DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop captured frame conversion failed"); continue; } +#if QT_VERSION >= QT_VERSION_CHECK(6, 7, 0) + if (frame.rotation() != QtVideo::Rotation::None) lastImage = lastImage.transformed(QTransform().rotate(static_cast(frame.rotation()))); + if (frame.mirrored()) lastImage = lastImage.mirrored(true, false); +#else + if (frame.rotationAngle()) lastImage = lastImage.transformed(QTransform().rotate(frame.rotationAngle())); +#endif } else if (!screen || lastImage.isNull()) { std::unique_lock lock(m->lock); m->wake.wait_for(lock, std::chrono::milliseconds(5)); continue; } @@ -384,7 +401,10 @@ void StreamPublishEngine::videoLoop() { if (encoded.key) { bool first; { std::lock_guard guard(m->lock); first = !m->hasGood || m->good.video != config.video || m->rollingBack; - if (m->generation == generation) { m->good = config; m->hasGood = true; m->rollingBack = false; } } + first |= m->confirmedGeneration != generation; + if (m->generation == generation) { + m->good = config; m->hasGood = true; m->rollingBack = false; m->confirmedGeneration = generation; + } } if (first) QMetaObject::invokeMethod(this, [this, generation] { if (generation != m->generation) return; emit configurationChanged(); emit stateChanged(text("Live")); @@ -416,15 +436,20 @@ void StreamPublishEngine::tick() { QImage preview; { std::lock_guard guard(m->lock); preview = m->preview; } if (!preview.isNull()) emit previewChanged(preview); +#ifdef HAVE_STREAM_CAPTURE + QVideoFrame rawPreview; + { std::lock_guard guard(m->lock); rawPreview = m->previewFrame; } + if (rawPreview.isValid()) emit previewVideoChanged(QVariant::fromValue(rawPreview)); +#endif if (status.state == STREAM_PUBLISHER_ENDED || status.state == STREAM_PUBLISHER_FAILED) { stop(); m->id = m->group = 0; if (status.state == STREAM_PUBLISHER_FAILED) emit errorOccurred(text("Stream publication failed")); emit stateChanged(text("Stopped")); emit configurationChanged(); return; } if (m->running) { - bool hasGood; int64_t started; - { std::lock_guard guard(m->lock); hasGood = m->hasGood; started = m->started; } - if (!hasGood && monotonicUs() - started > 8000000) sourceFailed(m->generation, text("Timed out waiting for the first video frame")); + bool confirmed; int64_t started; + { std::lock_guard guard(m->lock); confirmed = m->confirmedGeneration == m->generation; started = m->started; } + if (!confirmed && monotonicUs() - started > 8000000) sourceFailed(m->generation, text("Timed out waiting for the first video frame")); if (m->wanted.audio && m_capture.needsRecovery()) { m_capture.close(); m_capture.retryLater(); } if (m->wanted.audio && m_capture.retryReady()) openMicrophone(); } diff --git a/tools/chatgui/src/stream_publish_engine.h b/tools/chatgui/src/stream_publish_engine.h index 7cd375e4..8e5a2909 100644 --- a/tools/chatgui/src/stream_publish_engine.h +++ b/tools/chatgui/src/stream_publish_engine.h @@ -46,6 +46,7 @@ signals: void stateChanged(const QString& state); void configurationChanged(); void previewChanged(const QImage& frame); + void previewVideoChanged(const QVariant& frame); void telemetryChanged(const stream_publisher_status& status); void errorOccurred(const QString& error); void audioChannelsChanged(int channels);