jeka
/
utun2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Connect desktop camera workers to negotiated call video and audio clock

master
evgeny 5 days ago
parent
b091aa0df2
commit
cabae34791
2 changed files with 269 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 242
      tools/chatgui/src/call_video_engine.cpp
  2. 27
      tools/chatgui/src/call_video_engine.h

242
tools/chatgui/src/call_video_engine.cpp
Unescape View File

@ -0,0 +1,242 @@
#include "call_video_engine.h"
#include "call_video_encoder.h"
#include "call_video_decoder.h"
#include "../../../src/call/call_video_port.h"
#include "../../../src/call/call_audio.h"
#include "debug_config.h"
#include "mem.h"
#include <thread>
#include <mutex>
#include <atomic>
#include <deque>
#include <chrono>
#ifdef HAVE_STREAM_CAPTURE
#include <QCamera>
#include <QCameraDevice>
#include <QMediaDevices>
#include <QMediaCaptureSession>
#include <QVideoSink>
#include <QVideoFrame>
#endif
namespace {
int64_t nowUs() { return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now().time_since_epoch()).count(); }
void idle() { std::this_thread::sleep_for(std::chrono::milliseconds(5)); }
}
struct CallVideoEngine::Impl {
std::mutex lock;
std::thread tx, rx;
std::atomic<bool> running{false};
uint64_t id = 0, revision = 1;
bool enabled = false;
int width = 640, height = 480, codec = 0;
int64_t pts = -1, backend = -1, anchor = -1;
QImage shown;
uint64_t shownSerial = 0, deliveredSerial = 0;
QString status;
bool statusDirty = false;
void report(const QString& text) {
std::lock_guard<std::mutex> guard(lock); status = text; statusDirty = true;
DEBUG_INFO(DEBUG_CATEGORY_CALL, "desktop video: %s", qPrintable(text));
}
#ifdef HAVE_STREAM_CAPTURE
std::unique_ptr<QCamera> camera;
std::unique_ptr<QMediaCaptureSession> capture;
std::unique_ptr<QVideoSink> sink;
QVideoFrame raw;
#endif
};
CallVideoEngine::CallVideoEngine(QObject* parent) : QObject(parent), m(std::make_unique<Impl>()) {
m_timer.setInterval(20);
connect(&m_timer, &QTimer::timeout, this, [this] {
QImage image; QString status; bool changed = false, report = false;
{ std::lock_guard<std::mutex> guard(m->lock);
if (m->shownSerial != m->deliveredSerial) { image = m->shown; m->deliveredSerial = m->shownSerial; changed = true; }
if (m->statusDirty) { status = m->status; m->statusDirty = false; report = true; } }
if (changed) emit imageChanged(image);
if (report) emit statusChanged(status);
});
}
CallVideoEngine::~CallVideoEngine() { stop(); }
QList<QPair<QString, QByteArray>> CallVideoEngine::cameras() {
QList<QPair<QString, QByteArray>> result;
#ifdef HAVE_STREAM_CAPTURE
for (const auto& camera : QMediaDevices::videoInputs()) result.append({camera.description(), camera.id()});
#endif
return result;
}
void CallVideoEngine::start(quint64 id) {
if (m->running && m->id == id) return;
stop(); m->id = id; m->running = true; m_timer.start();
try { m->tx = std::thread(&CallVideoEngine::transmit, this); m->rx = std::thread(&CallVideoEngine::receive, this); }
catch (const std::exception& error) { DEBUG_ERROR(DEBUG_CATEGORY_CALL, "video worker start: %s", error.what()); stop(); emit statusChanged(error.what()); }
}
void CallVideoEngine::closeCamera() {
#ifdef HAVE_STREAM_CAPTURE
if (m->sink) disconnect(m->sink.get(), nullptr, this, nullptr);
if (m->camera) { disconnect(m->camera.get(), nullptr, this, nullptr); m->camera->stop(); }
m->capture.reset(); m->camera.reset(); m->sink.reset();
std::lock_guard<std::mutex> guard(m->lock); m->raw = {}; m->pts = m->backend = m->anchor = -1;
#endif
}
void CallVideoEngine::stop() {
m->running = false; m_timer.stop(); closeCamera();
if (m->tx.joinable()) m->tx.join();
if (m->rx.joinable()) m->rx.join();
if (m->id) call_video_port_close(m->id);
m->id = 0; m->enabled = false; m->shown = {}; emit imageChanged({});
}
void CallVideoEngine::setVideo(bool enabled, const QByteArray& cameraId, int width, int height, int codec) {
closeCamera();
uint64_t revision;
{ std::lock_guard<std::mutex> guard(m->lock);
m->enabled = enabled; m->width = width; m->height = height; m->codec = codec; revision = ++m->revision; }
if (!enabled || !m->running) return;
#ifdef HAVE_STREAM_CAPTURE
QCameraDevice selected;
for (const auto& camera : QMediaDevices::videoInputs()) if (camera.id() == cameraId) selected = camera;
if (selected.isNull()) { m->report(tr("Camera is unavailable")); return; }
m->capture = std::make_unique<QMediaCaptureSession>(); m->sink = std::make_unique<QVideoSink>();
m->camera = std::make_unique<QCamera>(selected); m->capture->setCamera(m->camera.get()); m->capture->setVideoSink(m->sink.get());
QCameraFormat best; int distance = INT_MAX;
for (const auto& f : selected.videoFormats()) {
if (f.maxFrameRate() < 29.9) continue;
int score = std::abs(f.resolution().width() - width) + std::abs(f.resolution().height() - height);
if (score < distance) { best = f; distance = score; }
}
if (!best.isNull()) m->camera->setCameraFormat(best);
connect(m->sink.get(), &QVideoSink::videoFrameChanged, this, [this, revision](const QVideoFrame& frame) {
if (!frame.isValid() || !m->running) return;
std::lock_guard<std::mutex> guard(m->lock);
if (revision != m->revision) return;
int64_t pts = nowUs();
if (frame.startTime() >= 0) {
if (m->backend < 0) {
m->backend = frame.startTime(); m->anchor = pts;
DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera clock source=%lld monotonic=%lld calibration=arrival-estimate",
(long long)m->backend, (long long)m->anchor);
}
pts = m->anchor + frame.startTime() - m->backend;
}
if (pts > m->pts) { m->raw = frame; m->pts = pts; }
}, Qt::DirectConnection);
connect(m->camera.get(), &QCamera::errorOccurred, this, [this](QCamera::Error, const QString& error) {
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call camera failed: %s", qPrintable(error)); m->report(error);
});
m->camera->start();
#else
Q_UNUSED(cameraId); Q_UNUSED(revision); m->report(tr("Video capture is unavailable in this build"));
#endif
}
void CallVideoEngine::transmit() {
call_video_caps caps{};
// Каждый объявленный codec проверяется реальным encoder + независимым decoder при open.
for (unsigned codec = 1; codec <= 2 && m->running; ++codec) {
call_video_format probe{1, 1280, 720, 30, 0, (uint8_t)codec, (uint8_t)(codec == 1 ? 31 : 93), 4000000, 1000000};
CallVideoEncoder encoder;
if (encoder.open(probe, 1000000)) {
call_video_limit limit{1280, 720, 30, 1280 * 720 * 30, 4000000, 1000000, probe.level};
caps.encode[codec - 1] = caps.decode[codec - 1] = limit;
} else DEBUG_WARN(DEBUG_CATEGORY_CALL, "call codec probe unavailable codec=%u: %s", codec, qPrintable(encoder.error()));
}
if (!m->running) return;
if (call_video_port_open(m->id, &caps) < 0) { m->report(tr("Video transport is unavailable")); return; }
CallVideoEncoder encoder;
call_video_format format{};
uint64_t appliedRevision = 0, keySerial = 0, request = 0, frameId = 0;
int64_t lastPts = -1, lastKey = -1;
bool forceKey = true;
while (m->running) {
call_video_port_status state{};
if (call_video_port_status(m->id, &state) < 0) break;
uint64_t revision; bool enabled; int width, height, codec; int64_t pts;
#ifdef HAVE_STREAM_CAPTURE
QVideoFrame raw;
#endif
{ std::lock_guard<std::mutex> guard(m->lock);
revision = m->revision; enabled = m->enabled; width = m->width; height = m->height; codec = m->codec; pts = m->pts;
#ifdef HAVE_STREAM_CAPTURE
raw = m->raw;
#endif
}
if (!enabled || !state.have_peer || pts <= lastPts || (lastPts >= 0 && pts - lastPts < 33333)) { idle(); continue; }
if (revision != appliedRevision) {
if (lastKey >= 0 && pts - lastKey < 1000000) { idle(); continue; }
call_video_format next{format.generation + 1, (uint16_t)width, (uint16_t)height, 30, 0, (uint8_t)codec, 0, 4000000, 1000000};
if (call_video_choose(&caps, &state.peer, &next) <= 0 || !encoder.open(next, 1000000)) {
m->report(tr("Selected video mode is unavailable to the peer"));
std::lock_guard<std::mutex> guard(m->lock); if (m->revision == revision) m->enabled = false;
continue;
}
format = next; appliedRevision = revision; forceKey = true; request = 0;
if (call_video_port_format(m->id, &format, encoder.bitrate()) < 0) break;
m->report(QString("%1 · %2×%3 · %4 kbit/s").arg(format.codec == 2 ? "H.265" : "H.264").arg(width).arg(height).arg(encoder.bitrate() / 1000));
}
if (state.configured_generation != format.generation) { idle(); continue; }
if (state.key_serial != keySerial) { keySerial = state.key_serial; forceKey = true; }
if (state.rate_generation == format.generation && state.rate_request > request) {
request = state.rate_request;
if (!encoder.requestBitrate(format.generation, request, state.rate_bps))
call_video_port_applied(m->id, format.generation, request, 0);
}
#ifdef HAVE_STREAM_CAPTURE
QImage image = raw.toImage();
if (image.isNull()) { DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera frame conversion failed"); lastPts = pts; continue; }
LiveEncodedFrame output; CallVideoRateCompletion completion;
bool encoded = encoder.encode(image, pts, forceKey, output, completion); lastPts = pts;
if (completion.request) {
call_video_port_applied(m->id, completion.generation, completion.request, completion.bitrate);
m->report(QString("%1 · %2×%3 · %4 kbit/s").arg(format.codec == 2 ? "H.265" : "H.264")
.arg(format.width).arg(format.height).arg(encoder.bitrate() / 1000));
}
if (!encoded) { forceKey = true; m->report(encoder.error()); continue; }
if (output.key) lastKey = pts;
call_video_frame packet{};
packet.call_id = m->id; packet.frame_id = ++frameId; packet.pts_us = pts; packet.format = format;
packet.size = packet.length = output.data.size(); packet.data = reinterpret_cast<const uint8_t*>(output.data.constData());
packet.flags = output.key ? CALL_VIDEO_KEY : 0;
forceKey = call_video_port_send(&packet) != 1;
#else
idle();
#endif
}
}
void CallVideoEngine::receive() {
struct Picture { int64_t pts; QImage image; };
std::deque<Picture> pictures;
CallVideoDecoder decoder;
uint64_t sourceEpoch = 0; int64_t lastShown = -1, report = 0; unsigned shown = 0;
while (m->running) {
int64_t now = nowUs(); call_clock_snapshot clock{};
call_audio_get_clock(m->id, now, &clock);
if (clock.valid) {
if (sourceEpoch && sourceEpoch != clock.source_epoch) {
pictures.clear(); decoder.close(); call_video_port_request_key(m->id); lastShown = -1;
DEBUG_INFO(DEBUG_CATEGORY_CALL, "video playback audio source epoch changed");
}
sourceEpoch = clock.source_epoch;
QImage selected;
while (!pictures.empty() && pictures.front().pts <= clock.pts_us) {
if (pictures.front().pts > lastShown) { selected = pictures.front().image; lastShown = pictures.front().pts; }
pictures.pop_front();
}
if (!selected.isNull()) { std::lock_guard<std::mutex> guard(m->lock); m->shown = selected; ++m->shownSerial; ++shown; }
}
if (pictures.size() < 16) {
call_video_frame packet{};
if (call_video_port_pull(m->id, &packet) == 1) {
QImage image;
if (decoder.decode(packet, image)) pictures.push_back({packet.pts_us, image});
else call_video_port_request_key(m->id);
u_free((void*)packet.data); continue;
}
}
if (now - report >= 1000000) {
DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "video playback shown=%u queue=%zu audio_valid=%d audio_pts=%lld video_pts=%lld",
shown, pictures.size(), clock.valid, (long long)clock.pts_us, (long long)lastShown); report = now;
}
idle();
}
}

27
tools/chatgui/src/call_video_engine.h
Unescape View File

@ -0,0 +1,27 @@
#pragma once
#include <QObject>
#include <QImage>
#include <QTimer>
#include <memory>
// GUI владеет камерой, TX/RX workers — codec; отображение читает фактический audio clock.
class CallVideoEngine : public QObject {
Q_OBJECT
public:
explicit CallVideoEngine(QObject* parent = nullptr);
~CallVideoEngine() override;
void start(quint64 id);
void stop();
void setVideo(bool enabled, const QByteArray& camera, int width, int height, int codec);
static QList<QPair<QString, QByteArray>> cameras();
signals:
void imageChanged(const QImage& image);
void statusChanged(const QString& status);
private:
struct Impl;
std::unique_ptr<Impl> m;
QTimer m_timer;
void closeCamera();
void transmit();
void receive();
};
Write Preview
Loading…
Cancel
Save