jeka/utun2
1
0 Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Isolate video call camera capture from the GUI process

Browse Source
This commit is contained in:
evgeny committed 2026-10-15 20:16:59 +02:00
1 parent 0b136ae2ab
commit 3e4d5d235f
13 files changed
+915 -115
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/chatgui/src/call_video_engine.cpp
+67 -111
View File
@@ -2,8 +2,7 @@
#include "call_video_encoder.h"
#include "call_video_decoder.h"
#include "call_video_cadence.h"
#include "call_video_capture_clock.h"
#include "call_camera_rate.h"
#include "call_camera_capture.h"
#include "../../../src/call/call_video_port.h"
#include "../../../src/call/call_video_playout.h"
#include "../../../src/call/call_audio.h"
@@ -11,40 +10,36 @@
#include "mem.h"
#include <thread>
#include <mutex>
#include <condition_variable>
#include <atomic>
#include <deque>
#include <chrono>
#ifdef HAVE_STREAM_CAPTURE
#include <QCamera>
#include <QCameraDevice>
#include <QMediaDevices>
#include <QMediaCaptureSession>
#include <QVideoSink>
#include <QVideoFrame>
#endif
namespace {
int64_t nowUs() { return call_video_playout_now_us(); }
void idle() { std::this_thread::sleep_for(std::chrono::milliseconds(5)); }
}
struct CallVideoEngine::Impl {
std::mutex lock;
std::condition_variable wake;
uint64_t wakeSerial = 0;
std::thread tx, rx;
std::atomic<bool> running{false};
uint64_t id = 0, revision = 1;
bool enabled = false;
int width = 640, height = 480, codec = 0;
int64_t pts = -1;
CallVideoCaptureClock captureClock;
int64_t clockCorrection = 0;
QImage capturedImage;
QByteArray cameraId;
bool captureActive = false;
uint64_t captured = 0;
QImage shown;
int64_t shownPts = -1;
uint64_t publicationEpoch = 0;
bool publicationPending = false;
int64_t shownAt = 0;
int64_t previewPts = -1, previewReportAt = 0, previewMaxUs = 0;
int64_t previewReportAt = 0, previewMaxUs = 0;
unsigned previewFrames = 0;
bool previewVisible = false;
QString status;
bool statusDirty = false;
void report(const QString& text) {
@@ -52,11 +47,7 @@ struct CallVideoEngine::Impl {
DEBUG_INFO(DEBUG_CATEGORY_CALL, "desktop video: %s", qPrintable(text));
}
#ifdef HAVE_STREAM_CAPTURE
std::unique_ptr<QCamera> camera;
std::unique_ptr<QMediaCaptureSession> capture;
std::unique_ptr<QVideoSink> sink;
CallCameraRate cameraRate;
QVideoFrame raw;
std::unique_ptr<CallCameraCapture> capture;
#endif
};
CallVideoEngine::CallVideoEngine(QObject* parent) : QObject(parent), m(std::make_unique<Impl>()) {
@@ -67,31 +58,31 @@ CallVideoEngine::CallVideoEngine(QObject* parent) : QObject(parent), m(std::make
{ std::lock_guard<std::mutex> guard(m->lock);
if (m->statusDirty) { status = m->status; m->statusDirty = false; report = true; } }
if (report) emit statusChanged(status);
});
#ifdef HAVE_STREAM_CAPTURE
QVideoFrame preview; bool clear = false;
int64_t now = nowUs();
m->capture = std::make_unique<CallCameraCapture>();
connect(m->capture.get(), &CallCameraCapture::devicesChanged, this, &CallVideoEngine::camerasChanged);
connect(m->capture.get(), &CallCameraCapture::failed, this, [this](const QString& error) {
{ std::lock_guard<std::mutex> guard(m->lock);
if (!m->enabled && m->previewVisible) { m->previewVisible = false; clear = true; }
if (m->enabled && m->raw.isValid() && m->pts != m->previewPts) {
preview = m->raw; m->previewPts = m->pts;
} }
if (clear) emit localImageChanged({});
if (preview.isValid()) {
QImage local = preview.toImage();
if (!local.isNull()) {
m->previewVisible = true;
emit localImageChanged(local.scaled(320, 240, Qt::KeepAspectRatio, Qt::SmoothTransformation));
++m->previewFrames;
m->previewMaxUs = std::max(m->previewMaxUs, nowUs() - now);
} else DEBUG_WARN(DEBUG_CATEGORY_CALL, "call camera preview conversion failed");
}
if (m->previewVisible && now - m->previewReportAt >= 1000000) {
DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "video local preview call=%llu interval_us=%lld frames=%u update_max_us=%lld",
(unsigned long long)m->id, (long long)(now - m->previewReportAt), m->previewFrames, (long long)m->previewMaxUs);
m->enabled = false; ++m->revision; ++m->wakeSerial; m->capturedImage = {}; m->pts = -1; }
m->captureActive = false; m->wake.notify_one(); emit localImageChanged({});
m->report(error); emit statusChanged(error);
});
connect(m->capture.get(), &CallCameraCapture::frameReady, this, [this](const QImage& image, qint64 pts, qint64 correction) {
const int64_t now = nowUs();
{ std::lock_guard<std::mutex> guard(m->lock);
if (!m->running || !m->enabled || pts <= m->pts) return;
m->capturedImage = image; m->pts = pts; m->clockCorrection = correction; ++m->captured; ++m->wakeSerial; }
m->wake.notify_one();
emit localImageChanged(image.scaled(320, 240, Qt::KeepAspectRatio, Qt::FastTransformation));
++m->previewFrames; m->previewMaxUs = std::max(m->previewMaxUs, nowUs() - now);
if (now - m->previewReportAt >= 1000000) {
DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "video local preview call=%llu frames=%u update_max_us=%lld capture_age_us=%lld",
(unsigned long long)m->id, m->previewFrames, (long long)m->previewMaxUs, (long long)(now - pts));
m->previewReportAt = now; m->previewFrames = 0; m->previewMaxUs = 0;
}
#endif
});
#endif
}
CallVideoEngine::~CallVideoEngine() { stop(); }
// В GUI одновременно не больше одного уведомления; при паузе UI остаётся последний выбранный кадр.
@@ -112,13 +103,6 @@ void CallVideoEngine::publishImage(const QImage& image, int64_t pts) {
(unsigned long long)m->id, (long long)pts, (long long)nowUs(), (long long)(nowUs() - selected));
}, Qt::QueuedConnection);
}
QList<QPair<QString, QByteArray>> CallVideoEngine::cameras() {
QList<QPair<QString, QByteArray>> result;
#ifdef HAVE_STREAM_CAPTURE
for (const auto& camera : QMediaDevices::videoInputs()) result.append({camera.description(), camera.id()});
#endif
return result;
}
void CallVideoEngine::start(quint64 id) {
if (m->running && m->id == id) return;
stop(); m->id = id; m->running = true; m_timer.start();
@@ -127,14 +111,11 @@ void CallVideoEngine::start(quint64 id) {
}
void CallVideoEngine::closeCamera() {
#ifdef HAVE_STREAM_CAPTURE
if (m->sink) disconnect(m->sink.get(), nullptr, this, nullptr);
if (m->camera) { disconnect(m->camera.get(), nullptr, this, nullptr); m->camera->stop(); }
m->cameraRate.restore();
m->capture.reset(); m->camera.reset(); m->sink.reset();
{ std::lock_guard<std::mutex> guard(m->lock); m->raw = {}; m->pts = -1; m->captureClock = {}; }
if (m->captureActive) m->capture->configure(false, m->cameraId, m->width, m->height);
#endif
m->previewPts = -1; m->previewVisible = false; m->previewReportAt = nowUs(); m->previewFrames = 0; m->previewMaxUs = 0;
emit localImageChanged({});
m->captureActive = false;
{ std::lock_guard<std::mutex> guard(m->lock); m->capturedImage = {}; m->pts = -1; ++m->wakeSerial; }
m->wake.notify_one(); emit localImageChanged({});
}
void CallVideoEngine::stop() {
m->running = false; m_timer.stop(); closeCamera();
@@ -146,50 +127,20 @@ void CallVideoEngine::stop() {
m->id = 0; m->enabled = false; emit imageChanged({});
}
void CallVideoEngine::setVideo(bool enabled, const QByteArray& cameraId, int width, int height, int codec) {
closeCamera();
uint64_t revision;
const bool active = enabled && m->running;
const bool captureChanged = active != m->captureActive || cameraId != m->cameraId || width != m->width || height != m->height;
{ std::lock_guard<std::mutex> guard(m->lock);
m->enabled = enabled; m->width = width; m->height = height; m->codec = codec; revision = ++m->revision; }
if (!enabled || !m->running) return;
if (m->enabled == enabled && cameraId == m->cameraId && width == m->width && height == m->height && codec == m->codec
&& !captureChanged) return;
m->enabled = enabled; m->width = width; m->height = height; m->codec = codec; ++m->revision; ++m->wakeSerial;
if (captureChanged) { m->capturedImage = {}; m->pts = -1; } }
m->cameraId = cameraId; m->captureActive = active; m->wake.notify_one();
if (!captureChanged) return;
emit localImageChanged({});
#ifdef HAVE_STREAM_CAPTURE
QCameraDevice selected;
for (const auto& camera : QMediaDevices::videoInputs()) if (camera.id() == cameraId) selected = camera;
if (selected.isNull()) { m->report(tr("Camera is unavailable")); return; }
m->capture = std::make_unique<QMediaCaptureSession>(); m->sink = std::make_unique<QVideoSink>();
m->camera = std::make_unique<QCamera>(selected); m->capture->setCamera(m->camera.get()); m->capture->setVideoSink(m->sink.get());
QCameraFormat best; int distance = INT_MAX;
for (const auto& f : selected.videoFormats()) {
if (f.maxFrameRate() < 29.9) continue;
int score = std::abs(f.resolution().width() - width) + std::abs(f.resolution().height() - height);
if (score < distance) { best = f; distance = score; }
}
if (!best.isNull()) m->camera->setCameraFormat(best);
const auto selectedFormat = m->camera->cameraFormat();
DEBUG_INFO(DEBUG_CATEGORY_CALL, "call camera selected device=%s size=%dx%d pixel=%d fps=%.3f..%.3f",
selected.id().constData(), selectedFormat.resolution().width(), selectedFormat.resolution().height(),
selectedFormat.pixelFormat(), selectedFormat.minFrameRate(), selectedFormat.maxFrameRate());
connect(m->camera.get(), &QCamera::activeChanged, this, [this, device = selected.id()](bool active) {
if (active) m->cameraRate.activate(device);
else m->cameraRate.restore();
});
connect(m->sink.get(), &QVideoSink::videoFrameChanged, this, [this, revision](const QVideoFrame& frame) {
if (!frame.isValid() || !m->running) return;
std::lock_guard<std::mutex> guard(m->lock);
if (revision != m->revision) return;
const int64_t arrival = nowUs(), source = frame.startTime();
if (m->pts < 0) DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera clock source=%lld monotonic=%lld calibration=arrival-upper-bound",
(long long)source, (long long)arrival);
if (m->captureClock.source >= 0 && source < m->captureClock.source)
DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera clock reset source=%lld previous=%lld", (long long)source, (long long)m->captureClock.source);
const int64_t pts = m->captureClock.map(source, arrival);
if (pts > m->pts) { m->raw = frame; m->pts = pts; ++m->captured; }
}, Qt::DirectConnection);
connect(m->camera.get(), &QCamera::errorOccurred, this, [this](QCamera::Error, const QString& error) {
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call camera failed: %s", qPrintable(error)); m->report(error);
});
m->camera->start();
m->capture->configure(active, cameraId, width, height);
#else
Q_UNUSED(cameraId); Q_UNUSED(revision); m->report(tr("Video capture is unavailable in this build"));
if (active) m->report(tr("Video capture is unavailable in this build"));
#endif
}
@@ -212,27 +163,27 @@ void CallVideoEngine::transmit() {
uint64_t appliedRevision = 0, keySerial = 0, request = 0, frameId = 0;
uint64_t userRevision = 0, rejectedGeneration = 0;
unsigned excluded = 0;
int64_t lastPts = -1, lastKey = -1;
int64_t lastPts = -1;
bool forceKey = true;
int64_t reportAt = nowUs(), encodeMax = 0;
uint64_t captureBase = 0;
unsigned encodedCount = 0, keyCount = 0, sendBusy = 0, paced = 0;
CallVideoCadence cadence;
uint64_t wakeSerial = 0;
auto idle = [this, &wakeSerial] {
std::unique_lock<std::mutex> guard(m->lock);
m->wake.wait_for(guard, std::chrono::milliseconds(5), [&] { return !m->running || m->wakeSerial != wakeSerial; });
};
while (m->running) {
struct call_video_port_status state{};
if (call_video_port_status(m->id, &state) < 0) break;
if (state.error) { m->report(tr("Core rejected video format")); break; }
uint64_t revision, captured; bool enabled; int width, height, codec; int64_t pts, clockCorrection;
#ifdef HAVE_STREAM_CAPTURE
QVideoFrame raw;
#endif
QImage image;
{ std::lock_guard<std::mutex> guard(m->lock);
revision = m->revision; enabled = m->enabled; width = m->width; height = m->height; codec = m->codec; pts = m->pts;
captured = m->captured;
clockCorrection = m->captureClock.correction;
#ifdef HAVE_STREAM_CAPTURE
raw = m->raw;
#endif
clockCorrection = m->clockCorrection; image = m->capturedImage; wakeSerial = m->wakeSerial;
}
int64_t now = nowUs();
if (now - reportAt >= 1000000) {
@@ -248,7 +199,7 @@ void CallVideoEngine::transmit() {
if (format.generation && state.decoded_generation == format.generation && !state.decoded_accepted
&& rejectedGeneration != format.generation) {
rejectedGeneration = format.generation; excluded |= 1u << format.codec;
if (!codec) { appliedRevision = 0; lastKey = -1; m->report(tr("Receiver rejected video; selecting fallback")); }
if (!codec) { appliedRevision = 0; m->report(tr("Receiver rejected video; selecting fallback")); }
else {
m->report(tr("Receiver cannot decode selected codec"));
std::lock_guard<std::mutex> guard(m->lock); m->enabled = false; continue;
@@ -257,7 +208,6 @@ void CallVideoEngine::transmit() {
if (format.generation && state.have_peer && !call_video_format_supported(&caps, &state.peer, &format)) appliedRevision = 0;
if (!enabled || !state.have_peer || pts <= lastPts) { idle(); continue; }
if (revision != appliedRevision) {
if (lastKey >= 0 && pts - lastKey < 1000000) { idle(); continue; }
call_video_format next{format.generation + 1, (uint16_t)width, (uint16_t)height, 30, 0, (uint8_t)codec, 0, 4000000, 0};
bool opened = false;
uint32_t minimum = state.minimum_bps ? state.minimum_bps : 300000;
@@ -286,9 +236,10 @@ void CallVideoEngine::transmit() {
}
#ifdef HAVE_STREAM_CAPTURE
if (!cadence.accept(pts)) { lastPts = pts; ++paced; idle(); continue; }
const int64_t convertStarted = nowUs();
QImage image = raw.toImage();
if (image.isNull()) { DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera frame conversion failed"); lastPts = pts; continue; }
if (image.isNull() || nowUs() - pts > CallCameraMaxFrameAgeUs) {
DEBUG_WARN(DEBUG_CATEGORY_CALL, "video TX skips old capture age_us=%lld", (long long)(nowUs() - pts));
lastPts = pts; continue;
}
LiveEncodedFrame output; CallVideoRateCompletion completion;
int64_t encodeStarted = nowUs();
bool encoded = encoder.encode(image, pts, forceKey, output, completion); lastPts = pts;
@@ -300,18 +251,23 @@ void CallVideoEngine::transmit() {
}
if (!encoded) { forceKey = true; m->report(encoder.error()); continue; }
++encodedCount; if (output.key) ++keyCount;
if (output.key) lastKey = pts;
call_video_frame packet{};
packet.call_id = m->id; packet.frame_id = ++frameId; packet.pts_us = pts; packet.format = format;
packet.size = packet.length = output.data.size(); packet.data = reinterpret_cast<const uint8_t*>(output.data.constData());
packet.flags = output.key ? CALL_VIDEO_KEY : 0;
forceKey = call_video_port_send(&packet) != 1;
{ std::lock_guard<std::mutex> guard(m->lock);
if (!m->running || !m->enabled || m->revision != revision || nowUs() - pts > CallCameraMaxFrameAgeUs) {
DEBUG_WARN(DEBUG_CATEGORY_CALL, "video TX discards obsolete encoded frame revision=%llu/%llu enabled=%d age_us=%lld",
(unsigned long long)revision, (unsigned long long)m->revision, m->enabled, (long long)(nowUs() - pts));
forceKey = true; continue;
}
forceKey = call_video_port_send(&packet) != 1; }
// Одинаковое окно исходного PTS на всех этапах: примерно один кадр в секунду, без трассы каждого кадра.
if (pts % 1000000 < 34000)
DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG,
"video timing stage=encode call=%llu frame=%llu pts=%lld mono_us=%lld select_us=%lld convert_us=%lld encode_us=%lld accepted=%d",
"video timing stage=encode call=%llu frame=%llu pts=%lld mono_us=%lld encode_us=%lld accepted=%d",
(unsigned long long)m->id, (unsigned long long)frameId, (long long)pts, (long long)nowUs(),
(long long)convertStarted, (long long)(encodeStarted - convertStarted), (long long)(nowUs() - encodeStarted), !forceKey);
(long long)(nowUs() - encodeStarted), !forceKey);
if (forceKey) ++sendBusy;
#else
idle();