From 3e4d5d235f78446e60bcc467eafb514db0939494 Mon Sep 17 00:00:00 2001 From: evgeny Date: Thu, 15 Oct 2026 20:16:59 +0200 Subject: [PATCH] Isolate video call camera capture from the GUI process --- doc/service_lifecycle.md | 27 +++ tools/chatgui/CMakeLists.txt | 19 +- tools/chatgui/src/call_camera_buffer.h | 79 +++++++ tools/chatgui/src/call_camera_capture.cpp | 186 +++++++++++++++ tools/chatgui/src/call_camera_capture.h | 26 +++ tools/chatgui/src/call_camera_protocol.h | 45 ++++ tools/chatgui/src/call_camera_source.h | 20 ++ tools/chatgui/src/call_camera_worker.cpp | 216 +++++++++++++++++ tools/chatgui/src/call_video_engine.cpp | 178 ++++++-------- tools/chatgui/src/call_video_engine.h | 5 +- tools/chatgui/src/callwindow.cpp | 9 +- tools/chatgui/src/main.cpp | 3 + .../tests/test_call_camera_capture.cpp | 217 ++++++++++++++++++ 13 files changed, 915 insertions(+), 115 deletions(-) create mode 100644 tools/chatgui/src/call_camera_buffer.h create mode 100644 tools/chatgui/src/call_camera_capture.cpp create mode 100644 tools/chatgui/src/call_camera_capture.h create mode 100644 tools/chatgui/src/call_camera_protocol.h create mode 100644 tools/chatgui/src/call_camera_source.h create mode 100644 tools/chatgui/src/call_camera_worker.cpp create mode 100644 tools/chatgui/tests/test_call_camera_capture.cpp diff --git a/doc/service_lifecycle.md b/doc/service_lifecycle.md index 480cd0f3..392aa205 100644 --- a/doc/service_lifecycle.md +++ b/doc/service_lifecycle.md @@ -155,6 +155,33 @@ Fatal ошибка audio worker передаётся в GUI с session ID и з откат видеоконфигурации применяется только к video/capture. Поздний отказ старого publisher не завершает новую сессию. Причина ошибки сохраняется в окне после окончания drain. +В desktop-видеозвонке Windows/Linux камера живёт в отдельном процессе того же +`vibechat --call-camera-worker`. Служебный вход выполняется до конфига, БД и +single-instance lock. Перечисление устройств, Qt/V4L2, преобразование и освобождение +аппаратных кадров происходят только там. GUI получает собственный `QImage`; TX не +обращается к драйверу. Смена codec не перезапускает захват и не ждёт следующего GOP. + +Пиксели одного кадра передаются через mapping (Windows — pagefile-backed объект ОС, +Linux — временный файл), метаданные/команды — через локальный сокет. +Worker записывает mapping до FRAME; следующий write разрешён +только после ACK, который GUI посылает после копирования в свою память. Общего mutex +нет. В worker ожидает только последний raw-кадр; поколения отсекают старые результаты. +Preview обновляется по готовности, TX получает condition-variable wake. PTS сохраняет +время захвата; кадры старше 250мс отбрасываются до encode и перед send. Отброшенный +encoded frame требует следующего KEY, поскольку encoder уже продвинул reference chain. + +Отключение камеры и удаление окна не ждут worker. Независимый Session сохраняет +QProcess и mapping до finished, затем удаляет себя после отсоединения владельца. +Зависший stop получает kill через 1.5с, отсутствие прогресса — через 5с; таймеры +живут в родительском процессе. При выходе приложения kill отправляется без wait; +GUI не уничтожает живой QProcess. При отказе новый процесс запускается лишь по +явной команде пользователя и после finished прежнего. Лог категории call показывает +последнюю стадию enumerate/start/convert/release/stop и возраст кадра. +`test_call_camera_capture` проверяет настоящую IPC-цепочку с подставным устройством: +блокировки start/map/stop, поздние кадры после смены поколения, задержку 720p, +повтор после отказа, отсутствие рестарта камеры при смене codec, удаление окна, +выход приложения и независимое освобождение mapping в каждом процессе. + Статус транспорта UP подтверждает восстановление физического пути. READY сессии группы означает согласованный JOIN/ACCEPT и завершённый обмен её таблицей; именно его ждёт recovery. REINIT меняет эпохи обмена, сохраняя рабочие маршруты. Эпохи diff --git a/tools/chatgui/CMakeLists.txt b/tools/chatgui/CMakeLists.txt index ef3c1db5..2247759e 100644 --- a/tools/chatgui/CMakeLists.txt +++ b/tools/chatgui/CMakeLists.txt @@ -160,8 +160,10 @@ if(FFMPEG_FOUND) set_tests_properties(test_call_video_clock PROPERTIES TIMEOUT 15) endif() endif() +add_library(chatgui_camera STATIC src/call_camera_capture.cpp src/call_camera_worker.cpp src/call_camera_source.h) +target_link_libraries(chatgui_camera PUBLIC ${QT_LIBS} utun) add_library(chatgui_stream_publish STATIC src/stream_publish_engine.cpp src/stream_publish_window.cpp src/call_video_engine.cpp) -target_link_libraries(chatgui_stream_publish PUBLIC chatgui_live_video) +target_link_libraries(chatgui_stream_publish PUBLIC chatgui_live_video chatgui_camera) target_include_directories(chatgui_stream_publish PRIVATE ${CMAKE_SOURCE_DIR}/../../lib/libopus/include) set(GUI_CAPTURE_ENABLED 0) if(QT_PACKAGE STREQUAL "Qt6" AND FFMPEG_FOUND) @@ -170,7 +172,22 @@ if(QT_PACKAGE STREQUAL "Qt6" AND FFMPEG_FOUND) set(Qt6_FOUND ${GUI_BASE_QT6_FOUND}) if(TARGET Qt6::Multimedia AND TARGET Qt6::MultimediaWidgets) target_compile_definitions(chatgui_stream_publish PUBLIC HAVE_STREAM_CAPTURE) + target_compile_definitions(chatgui_camera PUBLIC HAVE_STREAM_CAPTURE) + target_link_libraries(chatgui_camera PUBLIC Qt6::Multimedia) target_link_libraries(chatgui_stream_publish PUBLIC Qt6::Multimedia Qt6::MultimediaWidgets) + add_executable(test_call_camera_capture tests/test_call_camera_capture.cpp) + target_link_libraries(test_call_camera_capture PRIVATE chatgui_camera ${QT_TEST_LIB}) + if(UNIX AND NOT APPLE) + target_sources(test_call_camera_capture PRIVATE src/call_video_engine.cpp) + target_compile_definitions(test_call_camera_capture PRIVATE TEST_CALL_VIDEO_ENGINE) + target_link_libraries(test_call_camera_capture PRIVATE chatgui_live_video) + foreach(symbol call_audio_get_clock call_video_port_open call_video_port_close call_video_port_status + call_video_port_pull call_video_port_request_key call_video_port_decoded) + target_link_options(test_call_camera_capture PRIVATE "-Wl,--wrap=${symbol}") + endforeach() + endif() + add_test(NAME test_call_camera_capture COMMAND test_call_camera_capture) + set_tests_properties(test_call_camera_capture PROPERTIES TIMEOUT 40 ENVIRONMENT "QT_QPA_PLATFORM=offscreen") set(GUI_CAPTURE_ENABLED 1) else() message(STATUS "Qt Multimedia 6.6+ missing: stream publication unavailable") diff --git a/tools/chatgui/src/call_camera_buffer.h b/tools/chatgui/src/call_camera_buffer.h new file mode 100644 index 00000000..ccb9ac3c --- /dev/null +++ b/tools/chatgui/src/call_camera_buffer.h @@ -0,0 +1,79 @@ +#pragma once +#include "call_camera_protocol.h" +#include "debug_config.h" +#include +#ifdef Q_OS_WIN +#include +#else +#include +#include +#endif + +// Владение пикселями задаётся FRAME/ACK. Закрытие mapping никогда не ждёт другой процесс. +class CallCameraBuffer { + QString m_name, m_error; + uchar* m_data = nullptr; +#ifdef Q_OS_WIN + HANDLE m_mapping = nullptr; +#else + std::unique_ptr m_file; +#endif + bool fail(const QString& error) { m_error = error; close(); return false; } + bool map() { +#ifdef Q_OS_WIN + m_data = static_cast(MapViewOfFile(m_mapping, FILE_MAP_WRITE, 0, 0, CallCameraProtocol::FrameBytes)); + if (!m_data) return fail(QStringLiteral("MapViewOfFile error=%1").arg(GetLastError())); +#else + m_data = m_file->map(0, CallCameraProtocol::FrameBytes); + if (!m_data) return fail(m_file->errorString()); +#endif + return true; + } +public: + CallCameraBuffer() = default; + CallCameraBuffer(const CallCameraBuffer&) = delete; + CallCameraBuffer& operator=(const CallCameraBuffer&) = delete; + ~CallCameraBuffer() { close(); } + bool create() { + close(); +#ifdef Q_OS_WIN + m_name = QStringLiteral("Local\\vibechat-camera-") + QUuid::createUuid().toString(QUuid::Id128); + m_mapping = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, 0, CallCameraProtocol::FrameBytes, + reinterpret_cast(m_name.utf16())); + const DWORD error = GetLastError(); + if (!m_mapping || error == ERROR_ALREADY_EXISTS) return fail(QStringLiteral("CreateFileMapping error=%1").arg(error)); +#else + auto file = std::make_unique(); + if (!file->open() || !file->resize(CallCameraProtocol::FrameBytes)) return fail(file->errorString()); + m_name = file->fileName(); m_file = std::move(file); +#endif + return map(); + } + bool attach(const QString& name) { + close(); m_name = name; +#ifdef Q_OS_WIN + m_mapping = OpenFileMappingW(FILE_MAP_WRITE, FALSE, reinterpret_cast(name.utf16())); + if (!m_mapping) return fail(QStringLiteral("OpenFileMapping error=%1").arg(GetLastError())); +#else + m_file = std::make_unique(name); + if (!m_file->open(QIODevice::ReadWrite | QIODevice::ExistingOnly)) return fail(m_file->errorString()); + if (m_file->size() != CallCameraProtocol::FrameBytes) return fail(QStringLiteral("Invalid camera buffer size")); +#endif + return map(); + } + void close() { +#ifdef Q_OS_WIN + if (m_data && !UnmapViewOfFile(m_data)) + DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera buffer unmap error=%lu", (unsigned long)GetLastError()); + if (m_mapping && !CloseHandle(m_mapping)) + DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera buffer close error=%lu", (unsigned long)GetLastError()); + m_mapping = nullptr; +#else + m_file.reset(); +#endif + m_data = nullptr; + } + uchar* data() const { return m_data; } + QString name() const { return m_name; } + QString errorString() const { return m_error; } +}; diff --git a/tools/chatgui/src/call_camera_capture.cpp b/tools/chatgui/src/call_camera_capture.cpp new file mode 100644 index 00000000..2ba44221 --- /dev/null +++ b/tools/chatgui/src/call_camera_capture.cpp @@ -0,0 +1,186 @@ +#include "call_camera_capture.h" +#include "call_camera_protocol.h" +#include "call_camera_buffer.h" +#include "../../../src/call/call_video_playout.h" +#include "debug_config.h" +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace CallCameraProtocol; + +struct CallCameraCapture::Session : QObject { + QPointer owner; + QLocalServer server; + QProcess process; + QLocalSocket* socket = nullptr; + QTimer watchdog; + CallCameraBuffer buffer; + uchar* pixels = nullptr; + Reader reader; + quint64 generation = 0; + bool enabled = false, retiring = false, failureReported = false, restartRequested = false, exiting = false; + QByteArray device; + int width = 640, height = 480; + QString phase = QStringLiteral("launch"); + qint64 progressAt = 0, frameAt = 0, stopAt = 0; + + explicit Session(CallCameraCapture* target) : owner(target) { + server.setSocketOptions(QLocalServer::UserAccessOption); + connect(&server, &QLocalServer::newConnection, this, [this] { + auto* connection = server.nextPendingConnection(); + if (socket) { connection->abort(); connection->deleteLater(); return; } + socket = connection; socket->setParent(this); socket->setReadBufferSize(MaxPacket + 4); + server.close(); progressAt = call_video_playout_now_us(); + connect(socket, &QLocalSocket::readyRead, this, [this] { + if (!reader.read(*socket, [this](quint8 type, quint64 revision, QDataStream& in) { return receive(type, revision, in); })) + fail(QStringLiteral("Invalid camera worker response")); + }); + if (retiring) send(packet(Quit, generation)); + else { + send(packet(Buffer, 0, [this](QDataStream& out) { out << buffer.name(); })); + sendConfiguration(); + } + }); + process.setProcessChannelMode(QProcess::MergedChannels); + connect(&process, &QProcess::readyReadStandardOutput, this, [this] { + const auto log = process.readAllStandardOutput().trimmed(); + for (const auto& line : log.split('\n')) { + if (line.isEmpty()) continue; + if (line.contains("[ERROR]")) DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera worker: %s", line.constData()); + else if (line.contains("[WARN]")) DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera worker: %s", line.constData()); + else if (line.contains("[DEBUG]")) DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "camera worker: %s", line.constData()); + else DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera worker: %s", line.constData()); + } + }); + connect(&process, &QProcess::errorOccurred, this, [this](QProcess::ProcessError error) { + if (error == QProcess::FailedToStart) { + fail(QStringLiteral("Cannot start camera worker: %1").arg(process.errorString())); + if (retiring) deleteLater(); + } + }); + connect(&process, QOverload::of(&QProcess::finished), this, [this](int code, QProcess::ExitStatus status) { + DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera worker exited code=%d status=%d retiring=%d", code, int(status), retiring); + if (socket) { socket->disconnect(this); socket->deleteLater(); socket = nullptr; } + server.close(); watchdog.stop(); + if (retiring) deleteLater(); + else if (restartRequested) { restartRequested = false; launch(); } + else if (!failureReported) fail(QStringLiteral("Camera worker stopped")); + }); + watchdog.setInterval(250); + connect(&watchdog, &QTimer::timeout, this, [this] { + const qint64 now = call_video_playout_now_us(); + if (stopAt && now >= stopAt) { + DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera stop timed out phase=%s; killing worker", qPrintable(phase)); + stopAt = 0; + if (!retiring) fail(QStringLiteral("Camera did not stop")); else process.kill(); + } else if (!retiring && !failureReported && ((phase != "idle" && now - progressAt >= 5000000) + || (enabled && now - frameAt >= 5000000))) { + DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera stalled generation=%llu phase=%s operation_age_us=%lld frame_age_us=%lld", + (unsigned long long)generation, qPrintable(phase), (long long)(now - progressAt), (long long)(now - frameAt)); + fail(QStringLiteral("Camera stopped responding")); + } + }); + // При выходе приложения не вызываем деструктор живого QProcess: он синхронно ждёт child. + connect(qApp, &QCoreApplication::aboutToQuit, this, [this] { + exiting = true; enabled = false; failureReported = true; restartRequested = false; process.kill(); + buffer.close(); pixels = nullptr; + }); + QTimer::singleShot(0, this, [this] { if (!retiring && !generation && !failureReported) launch(); }); + } + void launch() { + if (exiting || process.state() != QProcess::NotRunning) return; + server.close(); + reader = {}; failureReported = false; phase = QStringLiteral("launch"); progressAt = call_video_playout_now_us(); + if (!pixels) { + if (!buffer.create()) { + fail(QStringLiteral("Cannot allocate camera frame buffer: %1").arg(buffer.errorString())); return; + } + pixels = buffer.data(); + } + const QString name = QStringLiteral("vibechat-camera-") + QUuid::createUuid().toString(QUuid::Id128); + if (!server.listen(name)) { fail(QStringLiteral("Cannot create camera IPC: %1").arg(server.errorString())); return; } + DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera worker starting; hardware isolated from GUI"); + process.start(QCoreApplication::applicationFilePath(), {QStringLiteral("--call-camera-worker"), server.fullServerName()}); + watchdog.start(); + } + void send(const QByteArray& data) { + if (!socket || socket->state() != QLocalSocket::ConnectedState) return; + if (socket->write(data) != data.size()) fail(QStringLiteral("Cannot write camera IPC")); + socket->flush(); + } + void sendConfiguration() { + send(packet(Configure, generation, [this](QDataStream& out) { out << enabled << device << qint32(width) << qint32(height); })); + } + void configure(bool active, const QByteArray& id, int w, int h) { + if (exiting) return; + if (enabled == active && device == id && width == w && height == h && !failureReported) return; + enabled = active; device = id; width = w; height = h; ++generation; + progressAt = call_video_playout_now_us(); + frameAt = progressAt; + stopAt = active ? 0 : progressAt + 1500000; + DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera request generation=%llu enabled=%d size=%dx%d device=%s", + (unsigned long long)generation, active, w, h, id.constData()); + if (failureReported && active) restartRequested = true; + if (process.state() == QProcess::NotRunning) { restartRequested = false; launch(); } + if (socket) sendConfiguration(); + } + void shutdown() { + owner = nullptr; retiring = true; enabled = false; ++generation; + if (process.state() == QProcess::NotRunning) { deleteLater(); return; } + stopAt = call_video_playout_now_us() + 1500000; watchdog.start(); send(packet(Quit, generation)); + } + void fail(const QString& error) { + if (failureReported) return; + failureReported = true; enabled = false; restartRequested = false; stopAt = 0; + DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera worker failure phase=%s: %s", qPrintable(phase), qPrintable(error)); + process.kill(); + if (owner) emit owner->failed(error); + } + bool receive(quint8 type, quint64 revision, QDataStream& in) { + if (type == Devices) { + quint32 count = 0; in >> count; + if (in.status() != QDataStream::Ok || count > 128) return false; + CallCameraDevices devices; + for (quint32 i = 0; i < count; ++i) { QString name; QByteArray id; in >> name >> id; devices.append({name, id}); } + if (owner) emit owner->devicesChanged(devices); + } else if (type == Stage) { + QString next; in >> next; + phase = next; progressAt = call_video_playout_now_us(); + if (revision == generation && next == "idle") stopAt = 0; + } else if (type == Error) { + QString error; in >> error; + if (revision == generation) fail(error); + } else if (type == Frame) { + qint64 pts = -1, correction = 0; qint32 w = 0, h = 0; in >> pts >> correction >> w >> h; + if (in.status() != QDataStream::Ok || pts < 0 || correction < 0 || w < 1 || h < 1 || w > 1280 || h > 720 || !pixels) return false; + const qint64 now = call_video_playout_now_us(); + const bool current = owner && enabled && revision == generation && !failureReported; + if (current && pts <= now && now - pts <= CallCameraMaxFrameAgeUs) { + QImage image(w, h, QImage::Format_RGB32); + if (image.isNull()) return false; + std::atomic_thread_fence(std::memory_order_acquire); + std::memcpy(image.bits(), pixels, size_t(w) * h * 4); + progressAt = frameAt = now; + send(packet(Ack, revision)); + emit owner->frameReady(image, pts, correction); + return true; + } else { + if (current) DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera dropped stale frame age_us=%lld", (long long)(now - pts)); + } + send(packet(Ack, revision)); + } else return false; + return true; + } +}; + +CallCameraCapture::CallCameraCapture(QObject* parent) : QObject(parent), m(new Session(this)) {} +CallCameraCapture::~CallCameraCapture() { m->shutdown(); } +void CallCameraCapture::configure(bool enabled, const QByteArray& device, int width, int height) { + m->configure(enabled, device, width, height); +} diff --git a/tools/chatgui/src/call_camera_capture.h b/tools/chatgui/src/call_camera_capture.h new file mode 100644 index 00000000..9636fc81 --- /dev/null +++ b/tools/chatgui/src/call_camera_capture.h @@ -0,0 +1,26 @@ +#pragma once +#include +#include +#include + +using CallCameraDevices = QList>; +constexpr qint64 CallCameraMaxFrameAgeUs = 250000; + +// GUI получает только собственную CPU-память. Процесс захвата завершается асинхронно. +class CallCameraCapture : public QObject { + Q_OBJECT +public: + explicit CallCameraCapture(QObject* parent = nullptr); + ~CallCameraCapture() override; + void configure(bool enabled, const QByteArray& device = {}, int width = 640, int height = 480); +signals: + void devicesChanged(const CallCameraDevices& devices); + void frameReady(const QImage& image, qint64 pts, qint64 correction); + void failed(const QString& error); +private: + struct Session; + Session* m; +}; + +// Вызывается до обычной инициализации приложения, конфига, БД и single-instance lock. +int call_camera_worker_main(int argc, char** argv); diff --git a/tools/chatgui/src/call_camera_protocol.h b/tools/chatgui/src/call_camera_protocol.h new file mode 100644 index 00000000..529d32b3 --- /dev/null +++ b/tools/chatgui/src/call_camera_protocol.h @@ -0,0 +1,45 @@ +#pragma once +#include +#include +#include +#include + +namespace CallCameraProtocol { +enum Type : quint8 { Configure = 1, Quit, Ack, Devices, Frame, Stage, Error, Buffer }; +constexpr quint32 FrameBytes = 1280 * 720 * 4; +constexpr quint32 MaxPacket = 65536; + +// Только команды и метаданные. Пиксели — в mapping с владением Frame -> Ack, без shared mutex. +inline QByteArray packet(Type type, quint64 generation, const std::function& write = {}) { + QByteArray result; + QDataStream out(&result, QIODevice::WriteOnly); out.setVersion(QDataStream::Qt_5_15); + out << quint32(0) << quint8(type) << generation; + if (write) write(out); + out.device()->seek(0); out << quint32(result.size() - 4); + return result; +} + +class Reader { + QByteArray pending; + quint32 length = 0; +public: + bool read(QLocalSocket& socket, const std::function& consume) { + while (socket.bytesAvailable()) { + const qsizetype wanted = length ? length : 4; + pending += socket.read(wanted - pending.size()); + if (pending.size() != wanted) return true; + QDataStream in(pending); in.setVersion(QDataStream::Qt_5_15); + if (!length) { + in >> length; pending.clear(); + if (length < 9 || length > MaxPacket) return false; + } else { + quint8 type; quint64 generation; in >> type >> generation; + bool valid = consume(type, generation, in) && in.status() == QDataStream::Ok && in.atEnd(); + pending.clear(); length = 0; + if (!valid) return false; + } + } + return true; + } +}; +} diff --git a/tools/chatgui/src/call_camera_source.h b/tools/chatgui/src/call_camera_source.h new file mode 100644 index 00000000..448c4c8e --- /dev/null +++ b/tools/chatgui/src/call_camera_source.h @@ -0,0 +1,20 @@ +#pragma once +#include "call_camera_capture.h" +#ifdef HAVE_STREAM_CAPTURE +#include +#include +#include + +// Backend принадлежит только процессу захвата; тесты подставляют неисправное устройство. +class CallCameraSource : public QObject { + Q_OBJECT +public: + virtual CallCameraDevices devices() = 0; + virtual bool start(const QByteArray& device, int width, int height) = 0; + virtual void stop() = 0; +signals: + void frame(const QVideoFrame& frame); + void error(const QString& message); +}; +int call_camera_worker_run(const QString& endpoint, std::unique_ptr source); +#endif diff --git a/tools/chatgui/src/call_camera_worker.cpp b/tools/chatgui/src/call_camera_worker.cpp new file mode 100644 index 00000000..8be52f4a --- /dev/null +++ b/tools/chatgui/src/call_camera_worker.cpp @@ -0,0 +1,216 @@ +#include "call_camera_source.h" +#include "call_camera_protocol.h" +#include "call_camera_buffer.h" +#include "call_video_capture_clock.h" +#include "call_camera_rate.h" +#include "../../../src/call/call_video_playout.h" +#include "debug_config.h" +#include +#include +#include +#include +#include +#include +#include +#ifdef HAVE_STREAM_CAPTURE +#include +#include +#include +#include + +using namespace CallCameraProtocol; + +class QtCallCameraSource : public CallCameraSource { + std::unique_ptr camera; + std::unique_ptr session; + std::unique_ptr sink; + CallCameraRate rate; +public: + ~QtCallCameraSource() override { stop(); } + CallCameraDevices devices() override { + CallCameraDevices result; + for (const auto& device : QMediaDevices::videoInputs()) result.append({device.description(), device.id()}); + return result; + } + bool start(const QByteArray& device, int width, int height) override { + QCameraDevice selected; + for (const auto& item : QMediaDevices::videoInputs()) if (item.id() == device) selected = item; + if (selected.isNull()) { emit error(QStringLiteral("Camera is unavailable")); return false; } + session = std::make_unique(); sink = std::make_unique(); + camera = std::make_unique(selected); + session->setCamera(camera.get()); session->setVideoSink(sink.get()); + QCameraFormat best; int distance = INT_MAX; + for (const auto& format : selected.videoFormats()) { + if (format.maxFrameRate() < 29.9) continue; + int score = std::abs(format.resolution().width() - width) + std::abs(format.resolution().height() - height); + if (score < distance) { best = format; distance = score; } + } + if (!best.isNull()) camera->setCameraFormat(best); + const auto format = camera->cameraFormat(); + DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera selected device=%s size=%dx%d pixel=%d fps=%.3f..%.3f", + device.constData(), format.resolution().width(), format.resolution().height(), format.pixelFormat(), + format.minFrameRate(), format.maxFrameRate()); + connect(camera.get(), &QCamera::activeChanged, this, [this, device](bool active) { + if (active) rate.activate(device); else rate.restore(); + }); + connect(camera.get(), &QCamera::errorOccurred, this, [this](QCamera::Error, const QString& message) { emit error(message); }); + connect(sink.get(), &QVideoSink::videoFrameChanged, this, &CallCameraSource::frame, Qt::DirectConnection); + camera->start(); return true; + } + void stop() override { + if (sink) disconnect(sink.get(), nullptr, this, nullptr); + if (camera) { disconnect(camera.get(), nullptr, this, nullptr); camera->stop(); } + rate.restore(); session.reset(); camera.reset(); sink.reset(); + } +}; + +class CameraWorker : public QObject { + struct Pending { + std::mutex lock; + QVideoFrame raw; + qint64 arrival = 0; + bool posted = false; + }; + QLocalSocket socket; + CallCameraBuffer buffer; + uchar* pixels = nullptr; + Reader reader; + std::unique_ptr source; + std::shared_ptr pending; + QMetaObject::Connection captureConnection, errorConnection; + CallVideoCaptureClock clock; + quint64 generation = 0, inFlight = 0; + bool sending = false; + int width = 640, height = 480; + qint64 lastPts = -1, reportAt = 0, convertMax = 0; + unsigned frames = 0; + + void send(const QByteArray& data) { + if (socket.write(data) != data.size()) { + DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera worker IPC write failed: %s", qPrintable(socket.errorString())); + QCoreApplication::exit(2); + } + socket.flush(); // Стадия должна уйти до потенциально блокирующего вызова драйвера. + } + void stage(const QString& name) { send(packet(Stage, generation, [&](QDataStream& out) { out << name; })); } + void stop() { + stage(QStringLiteral("stop")); + disconnect(captureConnection); disconnect(errorConnection); source->stop(); pending.reset(); + DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera stopped generation=%llu", (unsigned long long)generation); + stage(QStringLiteral("idle")); + } + void convert(std::shared_ptr slot, quint64 revision) { + if (slot != pending || revision != generation || sending) return; + QVideoFrame raw; qint64 arrival; + { std::lock_guard guard(slot->lock); + raw.swap(slot->raw); arrival = slot->arrival; slot->posted = false; } + if (!raw.isValid()) return; + const qint64 started = call_video_playout_now_us(), sourcePts = raw.startTime(); + if (clock.source >= 0 && sourcePts < clock.source) + DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera clock reset source=%lld previous=%lld", (long long)sourcePts, (long long)clock.source); + const qint64 pts = clock.map(sourcePts, arrival); + if (pts <= lastPts || started - pts > CallCameraMaxFrameAgeUs) { + DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera skips old frame age_us=%lld pts=%lld last=%lld", + (long long)(started - pts), (long long)pts, (long long)lastPts); return; + } + stage(QStringLiteral("convert")); + QImage image = raw.toImage(); + if (image.isNull()) { + DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera frame conversion failed"); + send(packet(Error, generation, [](QDataStream& out) { out << QStringLiteral("Camera frame conversion failed"); })); return; + } + if (image.width() > width || image.height() > height) image = image.scaled(width, height, Qt::KeepAspectRatio, Qt::FastTransformation); + image = image.convertToFormat(QImage::Format_RGB32); + const qint64 converted = call_video_playout_now_us(); + if (converted - pts <= CallCameraMaxFrameAgeUs) { + std::memcpy(pixels, image.constBits(), size_t(image.width()) * image.height() * 4); + std::atomic_thread_fence(std::memory_order_release); + auto bytes = packet(Frame, generation, [&](QDataStream& out) { + out << pts << qint64(clock.correction) << qint32(image.width()) << qint32(image.height()); + }); + sending = true; inFlight = generation; lastPts = pts; send(bytes); ++frames; + } else DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera conversion exceeded frame age budget age_us=%lld", (long long)(converted - pts)); + stage(QStringLiteral("release")); image = {}; raw = {}; stage(QStringLiteral("running")); + convertMax = std::max(convertMax, call_video_playout_now_us() - started); + if (started - reportAt >= 1000000) { + DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "camera flow frames=%u convert_max_us=%lld age_us=%lld clock_correction_us=%lld", + frames, (long long)convertMax, (long long)(converted - pts), (long long)clock.correction); + reportAt = started; convertMax = 0; frames = 0; + } + } + bool receive(quint8 type, quint64 revision, QDataStream& in) { + if (type == Buffer) { + QString name; in >> name; + if (pixels) return false; + if (!buffer.attach(name)) { + DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera frame mapping failed: %s", qPrintable(buffer.errorString())); return false; + } + pixels = buffer.data(); + } else if (type == Quit) { stop(); QCoreApplication::quit(); } + else if (type == Ack) { + if (sending && revision == inFlight) { sending = false; if (pending) convert(pending, generation); } + } else if (type == Configure) { + bool enabled = false; QByteArray device; qint32 w = 0, h = 0; in >> enabled >> device >> w >> h; + if (in.status() != QDataStream::Ok || w < 1 || h < 1 || w > 1280 || h > 720 || !pixels) return false; + stop(); generation = revision; width = w; height = h; clock = {}; lastPts = -1; + if (!enabled) { stage(QStringLiteral("idle")); return true; } + stage(QStringLiteral("start")); + pending = std::make_shared(); + errorConnection = connect(source.get(), &CallCameraSource::error, this, [this, revision](const QString& error) { + if (revision != generation) return; + DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera backend: %s", qPrintable(error)); + send(packet(Error, revision, [&](QDataStream& out) { out << error; })); + }, Qt::QueuedConnection); + captureConnection = connect(source.get(), &CallCameraSource::frame, this, + [this, slot = pending, revision](const QVideoFrame& frame) { + if (!frame.isValid()) return; + const qint64 arrival = call_video_playout_now_us(); + QVideoFrame previous; bool post; + { std::lock_guard guard(slot->lock); + previous.swap(slot->raw); slot->raw = frame; slot->arrival = arrival; + post = !slot->posted; slot->posted = true; } + if (post) QMetaObject::invokeMethod(this, [this, slot, revision] { convert(slot, revision); }, Qt::QueuedConnection); + }, Qt::DirectConnection); + if (source->start(device, width, height)) stage(QStringLiteral("running")); + } else return false; + return true; + } +public: + explicit CameraWorker(std::unique_ptr backend) : source(std::move(backend)) { + connect(&socket, &QLocalSocket::connected, this, [this] { + stage(QStringLiteral("enumerate")); + const auto devices = source->devices(); + send(packet(Devices, 0, [&](QDataStream& out) { + out << quint32(devices.size()); for (const auto& device : devices) out << device.first << device.second; + })); + stage(QStringLiteral("idle")); + }); + connect(&socket, &QLocalSocket::readyRead, this, [this] { + if (!reader.read(socket, [this](quint8 type, quint64 revision, QDataStream& in) { return receive(type, revision, in); })) { + DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera worker invalid command"); QCoreApplication::exit(2); + } + }); + connect(&socket, &QLocalSocket::disconnected, qApp, &QCoreApplication::quit); + connect(&socket, &QLocalSocket::errorOccurred, this, [this](QLocalSocket::LocalSocketError) { + DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera worker IPC: %s", qPrintable(socket.errorString())); QCoreApplication::exit(2); + }); + } + ~CameraWorker() override { disconnect(captureConnection); source->stop(); } + void connectTo(const QString& endpoint) { socket.connectToServer(endpoint); } +}; + +int call_camera_worker_run(const QString& endpoint, std::unique_ptr source) { + CameraWorker worker(std::move(source)); worker.connectTo(endpoint); return QCoreApplication::exec(); +} +#endif + +int call_camera_worker_main(int argc, char** argv) { +#ifdef HAVE_STREAM_CAPTURE + QGuiApplication app(argc, argv); debug_config_init(); debug_set_level(DEBUG_LEVEL_INFO); + debug_set_category_level(DEBUG_CATEGORY_CALL, DEBUG_LEVEL_DEBUG); + if (argc != 3) { DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera worker endpoint missing"); return 2; } + return call_camera_worker_run(QString::fromLocal8Bit(argv[2]), std::make_unique()); +#else + Q_UNUSED(argc); Q_UNUSED(argv); return 2; +#endif +} diff --git a/tools/chatgui/src/call_video_engine.cpp b/tools/chatgui/src/call_video_engine.cpp index 6c6ba3ad..bac1d7eb 100644 --- a/tools/chatgui/src/call_video_engine.cpp +++ b/tools/chatgui/src/call_video_engine.cpp @@ -2,8 +2,7 @@ #include "call_video_encoder.h" #include "call_video_decoder.h" #include "call_video_cadence.h" -#include "call_video_capture_clock.h" -#include "call_camera_rate.h" +#include "call_camera_capture.h" #include "../../../src/call/call_video_port.h" #include "../../../src/call/call_video_playout.h" #include "../../../src/call/call_audio.h" @@ -11,40 +10,36 @@ #include "mem.h" #include #include +#include #include #include #include -#ifdef HAVE_STREAM_CAPTURE -#include -#include -#include -#include -#include -#include -#endif namespace { int64_t nowUs() { return call_video_playout_now_us(); } -void idle() { std::this_thread::sleep_for(std::chrono::milliseconds(5)); } } struct CallVideoEngine::Impl { std::mutex lock; + std::condition_variable wake; + uint64_t wakeSerial = 0; std::thread tx, rx; std::atomic running{false}; uint64_t id = 0, revision = 1; bool enabled = false; int width = 640, height = 480, codec = 0; int64_t pts = -1; - CallVideoCaptureClock captureClock; + int64_t clockCorrection = 0; + QImage capturedImage; + QByteArray cameraId; + bool captureActive = false; uint64_t captured = 0; QImage shown; int64_t shownPts = -1; uint64_t publicationEpoch = 0; bool publicationPending = false; int64_t shownAt = 0; - int64_t previewPts = -1, previewReportAt = 0, previewMaxUs = 0; + int64_t previewReportAt = 0, previewMaxUs = 0; unsigned previewFrames = 0; - bool previewVisible = false; QString status; bool statusDirty = false; void report(const QString& text) { @@ -52,11 +47,7 @@ struct CallVideoEngine::Impl { DEBUG_INFO(DEBUG_CATEGORY_CALL, "desktop video: %s", qPrintable(text)); } #ifdef HAVE_STREAM_CAPTURE - std::unique_ptr camera; - std::unique_ptr capture; - std::unique_ptr sink; - CallCameraRate cameraRate; - QVideoFrame raw; + std::unique_ptr capture; #endif }; CallVideoEngine::CallVideoEngine(QObject* parent) : QObject(parent), m(std::make_unique()) { @@ -67,31 +58,31 @@ CallVideoEngine::CallVideoEngine(QObject* parent) : QObject(parent), m(std::make { std::lock_guard guard(m->lock); if (m->statusDirty) { status = m->status; m->statusDirty = false; report = true; } } if (report) emit statusChanged(status); + }); #ifdef HAVE_STREAM_CAPTURE - QVideoFrame preview; bool clear = false; - int64_t now = nowUs(); + m->capture = std::make_unique(); + connect(m->capture.get(), &CallCameraCapture::devicesChanged, this, &CallVideoEngine::camerasChanged); + connect(m->capture.get(), &CallCameraCapture::failed, this, [this](const QString& error) { { std::lock_guard guard(m->lock); - if (!m->enabled && m->previewVisible) { m->previewVisible = false; clear = true; } - if (m->enabled && m->raw.isValid() && m->pts != m->previewPts) { - preview = m->raw; m->previewPts = m->pts; - } } - if (clear) emit localImageChanged({}); - if (preview.isValid()) { - QImage local = preview.toImage(); - if (!local.isNull()) { - m->previewVisible = true; - emit localImageChanged(local.scaled(320, 240, Qt::KeepAspectRatio, Qt::SmoothTransformation)); - ++m->previewFrames; - m->previewMaxUs = std::max(m->previewMaxUs, nowUs() - now); - } else DEBUG_WARN(DEBUG_CATEGORY_CALL, "call camera preview conversion failed"); - } - if (m->previewVisible && now - m->previewReportAt >= 1000000) { - DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "video local preview call=%llu interval_us=%lld frames=%u update_max_us=%lld", - (unsigned long long)m->id, (long long)(now - m->previewReportAt), m->previewFrames, (long long)m->previewMaxUs); + m->enabled = false; ++m->revision; ++m->wakeSerial; m->capturedImage = {}; m->pts = -1; } + m->captureActive = false; m->wake.notify_one(); emit localImageChanged({}); + m->report(error); emit statusChanged(error); + }); + connect(m->capture.get(), &CallCameraCapture::frameReady, this, [this](const QImage& image, qint64 pts, qint64 correction) { + const int64_t now = nowUs(); + { std::lock_guard guard(m->lock); + if (!m->running || !m->enabled || pts <= m->pts) return; + m->capturedImage = image; m->pts = pts; m->clockCorrection = correction; ++m->captured; ++m->wakeSerial; } + m->wake.notify_one(); + emit localImageChanged(image.scaled(320, 240, Qt::KeepAspectRatio, Qt::FastTransformation)); + ++m->previewFrames; m->previewMaxUs = std::max(m->previewMaxUs, nowUs() - now); + if (now - m->previewReportAt >= 1000000) { + DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "video local preview call=%llu frames=%u update_max_us=%lld capture_age_us=%lld", + (unsigned long long)m->id, m->previewFrames, (long long)m->previewMaxUs, (long long)(now - pts)); m->previewReportAt = now; m->previewFrames = 0; m->previewMaxUs = 0; } -#endif }); +#endif } CallVideoEngine::~CallVideoEngine() { stop(); } // В GUI одновременно не больше одного уведомления; при паузе UI остаётся последний выбранный кадр. @@ -112,13 +103,6 @@ void CallVideoEngine::publishImage(const QImage& image, int64_t pts) { (unsigned long long)m->id, (long long)pts, (long long)nowUs(), (long long)(nowUs() - selected)); }, Qt::QueuedConnection); } -QList> CallVideoEngine::cameras() { - QList> result; -#ifdef HAVE_STREAM_CAPTURE - for (const auto& camera : QMediaDevices::videoInputs()) result.append({camera.description(), camera.id()}); -#endif - return result; -} void CallVideoEngine::start(quint64 id) { if (m->running && m->id == id) return; stop(); m->id = id; m->running = true; m_timer.start(); @@ -127,14 +111,11 @@ void CallVideoEngine::start(quint64 id) { } void CallVideoEngine::closeCamera() { #ifdef HAVE_STREAM_CAPTURE - if (m->sink) disconnect(m->sink.get(), nullptr, this, nullptr); - if (m->camera) { disconnect(m->camera.get(), nullptr, this, nullptr); m->camera->stop(); } - m->cameraRate.restore(); - m->capture.reset(); m->camera.reset(); m->sink.reset(); - { std::lock_guard guard(m->lock); m->raw = {}; m->pts = -1; m->captureClock = {}; } + if (m->captureActive) m->capture->configure(false, m->cameraId, m->width, m->height); #endif - m->previewPts = -1; m->previewVisible = false; m->previewReportAt = nowUs(); m->previewFrames = 0; m->previewMaxUs = 0; - emit localImageChanged({}); + m->captureActive = false; + { std::lock_guard guard(m->lock); m->capturedImage = {}; m->pts = -1; ++m->wakeSerial; } + m->wake.notify_one(); emit localImageChanged({}); } void CallVideoEngine::stop() { m->running = false; m_timer.stop(); closeCamera(); @@ -146,50 +127,20 @@ void CallVideoEngine::stop() { m->id = 0; m->enabled = false; emit imageChanged({}); } void CallVideoEngine::setVideo(bool enabled, const QByteArray& cameraId, int width, int height, int codec) { - closeCamera(); - uint64_t revision; + const bool active = enabled && m->running; + const bool captureChanged = active != m->captureActive || cameraId != m->cameraId || width != m->width || height != m->height; { std::lock_guard guard(m->lock); - m->enabled = enabled; m->width = width; m->height = height; m->codec = codec; revision = ++m->revision; } - if (!enabled || !m->running) return; + if (m->enabled == enabled && cameraId == m->cameraId && width == m->width && height == m->height && codec == m->codec + && !captureChanged) return; + m->enabled = enabled; m->width = width; m->height = height; m->codec = codec; ++m->revision; ++m->wakeSerial; + if (captureChanged) { m->capturedImage = {}; m->pts = -1; } } + m->cameraId = cameraId; m->captureActive = active; m->wake.notify_one(); + if (!captureChanged) return; + emit localImageChanged({}); #ifdef HAVE_STREAM_CAPTURE - QCameraDevice selected; - for (const auto& camera : QMediaDevices::videoInputs()) if (camera.id() == cameraId) selected = camera; - if (selected.isNull()) { m->report(tr("Camera is unavailable")); return; } - m->capture = std::make_unique(); m->sink = std::make_unique(); - m->camera = std::make_unique(selected); m->capture->setCamera(m->camera.get()); m->capture->setVideoSink(m->sink.get()); - QCameraFormat best; int distance = INT_MAX; - for (const auto& f : selected.videoFormats()) { - if (f.maxFrameRate() < 29.9) continue; - int score = std::abs(f.resolution().width() - width) + std::abs(f.resolution().height() - height); - if (score < distance) { best = f; distance = score; } - } - if (!best.isNull()) m->camera->setCameraFormat(best); - const auto selectedFormat = m->camera->cameraFormat(); - DEBUG_INFO(DEBUG_CATEGORY_CALL, "call camera selected device=%s size=%dx%d pixel=%d fps=%.3f..%.3f", - selected.id().constData(), selectedFormat.resolution().width(), selectedFormat.resolution().height(), - selectedFormat.pixelFormat(), selectedFormat.minFrameRate(), selectedFormat.maxFrameRate()); - connect(m->camera.get(), &QCamera::activeChanged, this, [this, device = selected.id()](bool active) { - if (active) m->cameraRate.activate(device); - else m->cameraRate.restore(); - }); - connect(m->sink.get(), &QVideoSink::videoFrameChanged, this, [this, revision](const QVideoFrame& frame) { - if (!frame.isValid() || !m->running) return; - std::lock_guard guard(m->lock); - if (revision != m->revision) return; - const int64_t arrival = nowUs(), source = frame.startTime(); - if (m->pts < 0) DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera clock source=%lld monotonic=%lld calibration=arrival-upper-bound", - (long long)source, (long long)arrival); - if (m->captureClock.source >= 0 && source < m->captureClock.source) - DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera clock reset source=%lld previous=%lld", (long long)source, (long long)m->captureClock.source); - const int64_t pts = m->captureClock.map(source, arrival); - if (pts > m->pts) { m->raw = frame; m->pts = pts; ++m->captured; } - }, Qt::DirectConnection); - connect(m->camera.get(), &QCamera::errorOccurred, this, [this](QCamera::Error, const QString& error) { - DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call camera failed: %s", qPrintable(error)); m->report(error); - }); - m->camera->start(); + m->capture->configure(active, cameraId, width, height); #else - Q_UNUSED(cameraId); Q_UNUSED(revision); m->report(tr("Video capture is unavailable in this build")); + if (active) m->report(tr("Video capture is unavailable in this build")); #endif } @@ -212,27 +163,27 @@ void CallVideoEngine::transmit() { uint64_t appliedRevision = 0, keySerial = 0, request = 0, frameId = 0; uint64_t userRevision = 0, rejectedGeneration = 0; unsigned excluded = 0; - int64_t lastPts = -1, lastKey = -1; + int64_t lastPts = -1; bool forceKey = true; int64_t reportAt = nowUs(), encodeMax = 0; uint64_t captureBase = 0; unsigned encodedCount = 0, keyCount = 0, sendBusy = 0, paced = 0; CallVideoCadence cadence; + uint64_t wakeSerial = 0; + auto idle = [this, &wakeSerial] { + std::unique_lock guard(m->lock); + m->wake.wait_for(guard, std::chrono::milliseconds(5), [&] { return !m->running || m->wakeSerial != wakeSerial; }); + }; while (m->running) { struct call_video_port_status state{}; if (call_video_port_status(m->id, &state) < 0) break; if (state.error) { m->report(tr("Core rejected video format")); break; } uint64_t revision, captured; bool enabled; int width, height, codec; int64_t pts, clockCorrection; -#ifdef HAVE_STREAM_CAPTURE - QVideoFrame raw; -#endif + QImage image; { std::lock_guard guard(m->lock); revision = m->revision; enabled = m->enabled; width = m->width; height = m->height; codec = m->codec; pts = m->pts; captured = m->captured; - clockCorrection = m->captureClock.correction; -#ifdef HAVE_STREAM_CAPTURE - raw = m->raw; -#endif + clockCorrection = m->clockCorrection; image = m->capturedImage; wakeSerial = m->wakeSerial; } int64_t now = nowUs(); if (now - reportAt >= 1000000) { @@ -248,7 +199,7 @@ void CallVideoEngine::transmit() { if (format.generation && state.decoded_generation == format.generation && !state.decoded_accepted && rejectedGeneration != format.generation) { rejectedGeneration = format.generation; excluded |= 1u << format.codec; - if (!codec) { appliedRevision = 0; lastKey = -1; m->report(tr("Receiver rejected video; selecting fallback")); } + if (!codec) { appliedRevision = 0; m->report(tr("Receiver rejected video; selecting fallback")); } else { m->report(tr("Receiver cannot decode selected codec")); std::lock_guard guard(m->lock); m->enabled = false; continue; @@ -257,7 +208,6 @@ void CallVideoEngine::transmit() { if (format.generation && state.have_peer && !call_video_format_supported(&caps, &state.peer, &format)) appliedRevision = 0; if (!enabled || !state.have_peer || pts <= lastPts) { idle(); continue; } if (revision != appliedRevision) { - if (lastKey >= 0 && pts - lastKey < 1000000) { idle(); continue; } call_video_format next{format.generation + 1, (uint16_t)width, (uint16_t)height, 30, 0, (uint8_t)codec, 0, 4000000, 0}; bool opened = false; uint32_t minimum = state.minimum_bps ? state.minimum_bps : 300000; @@ -286,9 +236,10 @@ void CallVideoEngine::transmit() { } #ifdef HAVE_STREAM_CAPTURE if (!cadence.accept(pts)) { lastPts = pts; ++paced; idle(); continue; } - const int64_t convertStarted = nowUs(); - QImage image = raw.toImage(); - if (image.isNull()) { DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera frame conversion failed"); lastPts = pts; continue; } + if (image.isNull() || nowUs() - pts > CallCameraMaxFrameAgeUs) { + DEBUG_WARN(DEBUG_CATEGORY_CALL, "video TX skips old capture age_us=%lld", (long long)(nowUs() - pts)); + lastPts = pts; continue; + } LiveEncodedFrame output; CallVideoRateCompletion completion; int64_t encodeStarted = nowUs(); bool encoded = encoder.encode(image, pts, forceKey, output, completion); lastPts = pts; @@ -300,18 +251,23 @@ void CallVideoEngine::transmit() { } if (!encoded) { forceKey = true; m->report(encoder.error()); continue; } ++encodedCount; if (output.key) ++keyCount; - if (output.key) lastKey = pts; call_video_frame packet{}; packet.call_id = m->id; packet.frame_id = ++frameId; packet.pts_us = pts; packet.format = format; packet.size = packet.length = output.data.size(); packet.data = reinterpret_cast(output.data.constData()); packet.flags = output.key ? CALL_VIDEO_KEY : 0; - forceKey = call_video_port_send(&packet) != 1; + { std::lock_guard guard(m->lock); + if (!m->running || !m->enabled || m->revision != revision || nowUs() - pts > CallCameraMaxFrameAgeUs) { + DEBUG_WARN(DEBUG_CATEGORY_CALL, "video TX discards obsolete encoded frame revision=%llu/%llu enabled=%d age_us=%lld", + (unsigned long long)revision, (unsigned long long)m->revision, m->enabled, (long long)(nowUs() - pts)); + forceKey = true; continue; + } + forceKey = call_video_port_send(&packet) != 1; } // Одинаковое окно исходного PTS на всех этапах: примерно один кадр в секунду, без трассы каждого кадра. if (pts % 1000000 < 34000) DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG, - "video timing stage=encode call=%llu frame=%llu pts=%lld mono_us=%lld select_us=%lld convert_us=%lld encode_us=%lld accepted=%d", + "video timing stage=encode call=%llu frame=%llu pts=%lld mono_us=%lld encode_us=%lld accepted=%d", (unsigned long long)m->id, (unsigned long long)frameId, (long long)pts, (long long)nowUs(), - (long long)convertStarted, (long long)(encodeStarted - convertStarted), (long long)(nowUs() - encodeStarted), !forceKey); + (long long)(nowUs() - encodeStarted), !forceKey); if (forceKey) ++sendBusy; #else idle(); diff --git a/tools/chatgui/src/call_video_engine.h b/tools/chatgui/src/call_video_engine.h index b2019df2..df72ffcb 100644 --- a/tools/chatgui/src/call_video_engine.h +++ b/tools/chatgui/src/call_video_engine.h @@ -3,8 +3,9 @@ #include #include #include +#include "call_camera_capture.h" -// GUI владеет камерой, TX/RX workers — codec; отображение читает фактический audio clock. +// Камера изолирована в процессе; TX/RX владеют codec и получают только CPU-изображения. class CallVideoEngine : public QObject { Q_OBJECT public: @@ -13,8 +14,8 @@ public: void start(quint64 id); void stop(); void setVideo(bool enabled, const QByteArray& camera, int width, int height, int codec); - static QList> cameras(); signals: + void camerasChanged(const CallCameraDevices& devices); void imageChanged(const QImage& image); void localImageChanged(const QImage& image); void statusChanged(const QString& status); diff --git a/tools/chatgui/src/callwindow.cpp b/tools/chatgui/src/callwindow.cpp index 9d02288e..d5c1533b 100644 --- a/tools/chatgui/src/callwindow.cpp +++ b/tools/chatgui/src/callwindow.cpp @@ -272,7 +272,14 @@ CallWindow::CallWindow(QWidget* parent) }); auto* modes = new QHBoxLayout(); m_camera = new QComboBox(this); - for (const auto& camera : CallVideoEngine::cameras()) m_camera->addItem(camera.first, camera.second); + connect(m_video, &CallVideoEngine::camerasChanged, this, [this](const CallCameraDevices& cameras) { + const auto selected = m_camera->currentData(); + m_camera->clear(); + for (const auto& camera : cameras) m_camera->addItem(camera.first, camera.second); + const int index = m_camera->findData(selected); + if (index >= 0) m_camera->setCurrentIndex(index); + if (m_videoEnabled->isChecked()) applyVideo(); + }); m_resolution = new QComboBox(this); m_resolution->addItem("640×480", QSize(640, 480)); m_resolution->addItem("1280×720", QSize(1280, 720)); m_codec = new QComboBox(this); m_codec->addItem(callText("Auto (H.265 → H.264)"), 0); m_codec->addItem("H.265", 2); m_codec->addItem("H.264", 1); modes->addWidget(m_camera, 1); modes->addWidget(m_resolution); modes->addWidget(m_codec); videoLayout->addLayout(modes); diff --git a/tools/chatgui/src/main.cpp b/tools/chatgui/src/main.cpp index f4fb3460..204377de 100644 --- a/tools/chatgui/src/main.cpp +++ b/tools/chatgui/src/main.cpp @@ -12,7 +12,9 @@ #include #include #include "gui_language.h" +#include "call_camera_capture.h" #include +#include #ifdef _WIN32 #include #include @@ -72,6 +74,7 @@ static struct ShutdownSentinel { } g_shutdown_sentinel; int main(int argc, char *argv[]) { + if (argc >= 2 && std::strcmp(argv[1], "--call-camera-worker") == 0) return call_camera_worker_main(argc, argv); #ifdef Q_OS_LINUX /* На Wayland композитор владеет позицией окна (move() игнорируется) — * для полноценного ручного move/resize с пропорцией запускаемся через XWayland. */ diff --git a/tools/chatgui/tests/test_call_camera_capture.cpp b/tools/chatgui/tests/test_call_camera_capture.cpp new file mode 100644 index 00000000..5c16cc2f --- /dev/null +++ b/tools/chatgui/tests/test_call_camera_capture.cpp @@ -0,0 +1,217 @@ +#include "../src/call_camera_source.h" +#include "../src/call_camera_buffer.h" +#include "../../../src/call/call_video_playout.h" +#include "debug_config.h" +#include +#include +#include +#include +#include +#include +#ifdef TEST_CALL_VIDEO_ENGINE +#include "../src/call_video_engine.h" +#include "../../../src/call/call_video_port.h" +#include "../../../src/call/call_audio.h" +extern "C" int __wrap_call_audio_get_clock(uint64_t, int64_t, call_clock_snapshot* out) { *out = {}; return 0; } +extern "C" int __wrap_call_video_port_open(uint64_t, const call_video_caps*) { return 0; } +extern "C" void __wrap_call_video_port_close(uint64_t) {} +extern "C" int __wrap_call_video_port_status(uint64_t, struct call_video_port_status* out) { *out = {}; return 0; } +extern "C" int __wrap_call_video_port_pull(uint64_t, call_video_frame*) { return 0; } +extern "C" void __wrap_call_video_port_request_key(uint64_t) {} +extern "C" int __wrap_call_video_port_decoded(uint64_t, uint64_t, int) { return 0; } +#endif +#if QT_VERSION >= QT_VERSION_CHECK(6, 8, 0) +#include +#endif +#ifdef Q_OS_WIN +#include +#else +#include +#endif + +static void record(const QByteArray& event) { + QFile file(qEnvironmentVariable("UTUN_CAMERA_TEST_LOG")); + if (file.open(QIODevice::WriteOnly | QIODevice::Append)) + file.write(QByteArray::number(QCoreApplication::applicationPid()) + ' ' + event + '\n'); +} +static bool alive(qint64 pid) { +#ifdef Q_OS_WIN + HANDLE handle = OpenProcess(SYNCHRONIZE, FALSE, DWORD(pid)); + if (!handle) return false; + bool running = WaitForSingleObject(handle, 0) == WAIT_TIMEOUT; CloseHandle(handle); return running; +#else + return kill(pid, 0) == 0; +#endif +} + +#if QT_VERSION >= QT_VERSION_CHECK(6, 8, 0) +class StuckBuffer : public QAbstractVideoBuffer { +public: + MapData map(QVideoFrame::MapMode) override { record("convert-blocked"); QThread::sleep(60); return {}; } + QVideoFrameFormat format() const override { return {QSize(16, 16), QVideoFrameFormat::Format_BGRX8888}; } +}; +#endif +class TestSource : public CallCameraSource { + QTimer timer; + QByteArray device; + int width = 16, height = 16; +public: + TestSource() { + record("created"); timer.setInterval(33); timer.setTimerType(Qt::PreciseTimer); + connect(&timer, &QTimer::timeout, this, [this] { +#if QT_VERSION >= QT_VERSION_CHECK(6, 8, 0) + if (device == "convert-stall") { + QVideoFrame raw(std::make_unique()); raw.setStartTime(call_video_playout_now_us()); emit frame(raw); return; + } +#endif + QVideoFrame raw(QVideoFrameFormat(QSize(width, height), QVideoFrameFormat::Format_BGRX8888)); + if (!raw.map(QVideoFrame::WriteOnly)) qFatal("test frame map failed"); + QImage image(raw.bits(0), width, height, raw.bytesPerLine(0), QImage::Format_RGB32); + image.fill(device == "blue" ? Qt::blue : Qt::red); raw.unmap(); + raw.setStartTime(call_video_playout_now_us()); emit frame(raw); + }); + } + CallCameraDevices devices() override { return {{QStringLiteral("Test camera"), "red"}}; } + bool start(const QByteArray& id, int w, int h) override { + device = id; width = w; height = h; record("start:" + id); + if (id == "start-stall") QThread::sleep(60); + timer.start(); return true; + } + void stop() override { + timer.stop(); + if (!device.isEmpty()) record("stop:" + device); + if (device == "stop-stall") QThread::sleep(60); + device.clear(); + } +}; + +class Heartbeat { + QTimer timer; + QElapsedTimer clock; + qint64 previous = 0; +public: + qint64 maximum = 0; + unsigned ticks = 0; + Heartbeat() { + clock.start(); timer.setInterval(5); timer.setTimerType(Qt::PreciseTimer); + QObject::connect(&timer, &QTimer::timeout, [&] { + const qint64 now = clock.elapsed(); maximum = std::max(maximum, now - previous); previous = now; ++ticks; + }); timer.start(); + } +}; + +class CameraTest : public QObject { + Q_OBJECT + QTemporaryDir directory; + QString path; + QByteArray log() { QFile file(path); if (!file.open(QIODevice::ReadOnly)) return {}; return file.readAll(); } + qint64 pid() { return log().split(' ').value(0).toLongLong(); } +private slots: + void init() { + path = directory.filePath(QString::number(QDateTime::currentMSecsSinceEpoch())); + qputenv("UTUN_CAMERA_TEST_LOG", path.toUtf8()); + } + void latestFrameAndGeneration() { + auto capture = std::make_unique(); + unsigned frames = 0; qint64 maxAge = 0; QColor color; bool expectBlue = false, wrongGeneration = false; + connect(capture.get(), &CallCameraCapture::frameReady, this, [&](const QImage& image, qint64 pts, qint64) { + ++frames; maxAge = std::max(maxAge, call_video_playout_now_us() - pts); color = image.pixelColor(0, 0); + if (expectBlue && color != QColor(Qt::blue)) wrongGeneration = true; + }); + QSignalSpy errors(capture.get(), &CallCameraCapture::failed); + capture->configure(true, "red", 1280, 720); + { Heartbeat heartbeat; + QTRY_VERIFY_WITH_TIMEOUT(frames >= 15, 4000); + qInfo("camera 720p latency max=%lldus GUI gap=%lldms", (long long)maxAge, (long long)heartbeat.maximum); + QVERIFY(maxAge < 100000); QVERIFY(heartbeat.maximum < 150); } + QCOMPARE(color, QColor(Qt::red)); + // Пока GUI занят, разрешён лишь один передаваемый кадр; после паузы берётся свежий. + QThread::msleep(350); const unsigned before = frames; + expectBlue = true; + capture->configure(true, "blue", 1280, 720); + QTRY_VERIFY_WITH_TIMEOUT(frames > before && color == QColor(Qt::blue), 2000); + QVERIFY(!wrongGeneration); + const unsigned stopped = frames; capture->configure(false, "blue", 1280, 720); + QTest::qWait(150); QCOMPARE(frames, stopped); QCOMPARE(errors.size(), 0); + const qint64 child = pid(); QVERIFY(child > 0); capture.reset(); QTRY_VERIFY_WITH_TIMEOUT(!alive(child), 3000); + } + void bufferLifetime() { + CallCameraBuffer creator, peer; + QVERIFY(creator.create()); const QString name = creator.name(); + QVERIFY(peer.attach(name)); creator.data()[0] = 17; + QCOMPARE(peer.data()[0], uchar(17)); + creator.close(); peer.data()[0] = 23; QCOMPARE(peer.data()[0], uchar(23)); + peer.close(); QVERIFY(!peer.attach(name)); + } + void blockedOperation_data() { + QTest::addColumn("device"); + QTest::newRow("start") << QByteArray("start-stall"); +#if QT_VERSION >= QT_VERSION_CHECK(6, 8, 0) + QTest::newRow("conversion") << QByteArray("convert-stall"); +#endif + } + void blockedOperation() { + QFETCH(QByteArray, device); + auto capture = std::make_unique(); + QSignalSpy errors(capture.get(), &CallCameraCapture::failed); Heartbeat heartbeat; + capture->configure(true, device); + QTRY_COMPARE_WITH_TIMEOUT(errors.size(), 1, 7500); + QVERIFY(heartbeat.ticks > 100); QVERIFY(heartbeat.maximum < 150); + qInfo("blocked %s: GUI gap max=%lldms ticks=%u", device.constData(), (long long)heartbeat.maximum, heartbeat.ticks); + if (device == "convert-stall") QVERIFY(log().contains("convert-blocked")); + const qint64 child = pid(); QVERIFY(child > 0); QTRY_VERIFY_WITH_TIMEOUT(!alive(child), 3000); + // Тот же владелец допускает явный повтор после отказа, без зависшего старого процесса. + unsigned frames = 0; + connect(capture.get(), &CallCameraCapture::frameReady, this, [&](const QImage&, qint64, qint64) { ++frames; }); + capture->configure(true, "blue"); QTRY_VERIFY_WITH_TIMEOUT(frames > 0, 3000); + capture.reset(); + } + void destroyWhileStopping() { + auto capture = std::make_unique(); + unsigned frames = 0; + connect(capture.get(), &CallCameraCapture::frameReady, this, [&](const QImage&, qint64, qint64) { ++frames; }); + capture->configure(true, "stop-stall"); QTRY_VERIFY_WITH_TIMEOUT(frames > 0, 3000); + const qint64 child = pid(); QVERIFY(child > 0); + Heartbeat heartbeat; QElapsedTimer elapsed; elapsed.start(); capture.reset(); + QVERIFY(elapsed.elapsed() < 50); + QTRY_VERIFY_WITH_TIMEOUT(!alive(child), 3500); + QVERIFY(log().contains("stop:stop-stall")); QVERIFY(heartbeat.maximum < 150); + } + void applicationExit() { + QProcess probe; probe.start(QCoreApplication::applicationFilePath(), {QStringLiteral("--exit-probe")}); + QTRY_COMPARE_WITH_TIMEOUT(probe.state(), QProcess::NotRunning, 3000); + QCOMPARE(probe.exitStatus(), QProcess::NormalExit); QCOMPARE(probe.exitCode(), 0); + QVERIFY(!probe.readAllStandardError().contains("Destroyed while process")); + } +#ifdef TEST_CALL_VIDEO_ENGINE + void engineKeepsCameraAcrossCodecChange() { + auto engine = std::make_unique(); + unsigned frames = 0; + connect(engine.get(), &CallVideoEngine::localImageChanged, this, [&](const QImage& image) { if (!image.isNull()) ++frames; }); + engine->start(7); engine->setVideo(true, "red", 640, 480, 1); + QTRY_VERIFY_WITH_TIMEOUT(frames >= 5, 4000); + QCOMPARE(log().count("start:red"), 1); + unsigned before = frames; engine->setVideo(true, "red", 640, 480, 2); + QTRY_VERIFY_WITH_TIMEOUT(frames >= before + 5, 2000); + QCOMPARE(log().count("start:red"), 1); QVERIFY(!log().contains("stop:red")); + engine->setVideo(true, "stop-stall", 640, 480, 2); + QTRY_VERIFY_WITH_TIMEOUT(log().contains("start:stop-stall"), 2000); + const qint64 child = pid(); Heartbeat heartbeat; QElapsedTimer elapsed; elapsed.start(); + engine.reset(); QVERIFY(elapsed.elapsed() < 150); + QTRY_VERIFY_WITH_TIMEOUT(!alive(child), 3500); QVERIFY(heartbeat.maximum < 150); + } +#endif +}; + +int main(int argc, char** argv) { + QGuiApplication app(argc, argv); debug_config_init(); debug_set_level(DEBUG_LEVEL_INFO); + if (app.arguments().value(1) == "--call-camera-worker") + return call_camera_worker_run(app.arguments().value(2), std::make_unique()); + if (app.arguments().value(1) == "--exit-probe") { + auto capture = std::make_unique(); capture->configure(true, "stop-stall"); + QTimer::singleShot(500, &app, &QCoreApplication::quit); + const int result = app.exec(); capture.reset(); return result; + } + CameraTest test; return QTest::qExec(&test, argc, argv); +} +#include "test_call_camera_capture.moc"