Files
utun2/tools/chatgui/src/call_video_engine.cpp
T

276 lines
14 KiB
C++
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#include "call_video_engine.h"
#include "call_video_encoder.h"
#include "call_video_decoder.h"
#include "../../../src/call/call_video_port.h"
#include "../../../src/call/call_audio.h"
#include "debug_config.h"
#include "mem.h"
#include <thread>
#include <mutex>
#include <atomic>
#include <deque>
#include <chrono>
#ifdef HAVE_STREAM_CAPTURE
#include <QCamera>
#include <QCameraDevice>
#include <QMediaDevices>
#include <QMediaCaptureSession>
#include <QVideoSink>
#include <QVideoFrame>
#endif
namespace {
int64_t nowUs() { return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now().time_since_epoch()).count(); }
void idle() { std::this_thread::sleep_for(std::chrono::milliseconds(5)); }
}
struct CallVideoEngine::Impl {
std::mutex lock;
std::thread tx, rx;
std::atomic<bool> running{false};
uint64_t id = 0, revision = 1;
bool enabled = false;
int width = 640, height = 480, codec = 0;
int64_t pts = -1, backend = -1, anchor = -1;
QImage shown;
uint64_t shownSerial = 0, deliveredSerial = 0;
QString status;
bool statusDirty = false;
void report(const QString& text) {
std::lock_guard<std::mutex> guard(lock); status = text; statusDirty = true;
DEBUG_INFO(DEBUG_CATEGORY_CALL, "desktop video: %s", qPrintable(text));
}
#ifdef HAVE_STREAM_CAPTURE
std::unique_ptr<QCamera> camera;
std::unique_ptr<QMediaCaptureSession> capture;
std::unique_ptr<QVideoSink> sink;
QVideoFrame raw;
#endif
};
CallVideoEngine::CallVideoEngine(QObject* parent) : QObject(parent), m(std::make_unique<Impl>()) {
m_timer.setInterval(20);
connect(&m_timer, &QTimer::timeout, this, [this] {
QImage image; QString status; bool changed = false, report = false;
{ std::lock_guard<std::mutex> guard(m->lock);
if (m->shownSerial != m->deliveredSerial) { image = m->shown; m->deliveredSerial = m->shownSerial; changed = true; }
if (m->statusDirty) { status = m->status; m->statusDirty = false; report = true; } }
if (changed) emit imageChanged(image);
if (report) emit statusChanged(status);
});
}
CallVideoEngine::~CallVideoEngine() { stop(); }
QList<QPair<QString, QByteArray>> CallVideoEngine::cameras() {
QList<QPair<QString, QByteArray>> result;
#ifdef HAVE_STREAM_CAPTURE
for (const auto& camera : QMediaDevices::videoInputs()) result.append({camera.description(), camera.id()});
#endif
return result;
}
void CallVideoEngine::start(quint64 id) {
if (m->running && m->id == id) return;
stop(); m->id = id; m->running = true; m_timer.start();
try { m->tx = std::thread(&CallVideoEngine::transmit, this); m->rx = std::thread(&CallVideoEngine::receive, this); }
catch (const std::exception& error) { DEBUG_ERROR(DEBUG_CATEGORY_CALL, "video worker start: %s", error.what()); stop(); emit statusChanged(error.what()); }
}
void CallVideoEngine::closeCamera() {
#ifdef HAVE_STREAM_CAPTURE
if (m->sink) disconnect(m->sink.get(), nullptr, this, nullptr);
if (m->camera) { disconnect(m->camera.get(), nullptr, this, nullptr); m->camera->stop(); }
m->capture.reset(); m->camera.reset(); m->sink.reset();
std::lock_guard<std::mutex> guard(m->lock); m->raw = {}; m->pts = m->backend = m->anchor = -1;
#endif
}
void CallVideoEngine::stop() {
m->running = false; m_timer.stop(); closeCamera();
if (m->tx.joinable()) m->tx.join();
if (m->rx.joinable()) m->rx.join();
if (m->id) call_video_port_close(m->id);
m->id = 0; m->enabled = false; m->shown = {}; emit imageChanged({});
}
void CallVideoEngine::setVideo(bool enabled, const QByteArray& cameraId, int width, int height, int codec) {
closeCamera();
uint64_t revision;
{ std::lock_guard<std::mutex> guard(m->lock);
m->enabled = enabled; m->width = width; m->height = height; m->codec = codec; revision = ++m->revision; }
if (!enabled || !m->running) return;
#ifdef HAVE_STREAM_CAPTURE
QCameraDevice selected;
for (const auto& camera : QMediaDevices::videoInputs()) if (camera.id() == cameraId) selected = camera;
if (selected.isNull()) { m->report(tr("Camera is unavailable")); return; }
m->capture = std::make_unique<QMediaCaptureSession>(); m->sink = std::make_unique<QVideoSink>();
m->camera = std::make_unique<QCamera>(selected); m->capture->setCamera(m->camera.get()); m->capture->setVideoSink(m->sink.get());
QCameraFormat best; int distance = INT_MAX;
for (const auto& f : selected.videoFormats()) {
if (f.maxFrameRate() < 29.9) continue;
int score = std::abs(f.resolution().width() - width) + std::abs(f.resolution().height() - height);
if (score < distance) { best = f; distance = score; }
}
if (!best.isNull()) m->camera->setCameraFormat(best);
connect(m->sink.get(), &QVideoSink::videoFrameChanged, this, [this, revision](const QVideoFrame& frame) {
if (!frame.isValid() || !m->running) return;
std::lock_guard<std::mutex> guard(m->lock);
if (revision != m->revision) return;
int64_t pts = nowUs();
if (frame.startTime() >= 0) {
if (m->backend < 0) {
m->backend = frame.startTime(); m->anchor = pts;
DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera clock source=%lld monotonic=%lld calibration=arrival-estimate",
(long long)m->backend, (long long)m->anchor);
}
pts = m->anchor + frame.startTime() - m->backend;
}
if (pts > m->pts) { m->raw = frame; m->pts = pts; }
}, Qt::DirectConnection);
connect(m->camera.get(), &QCamera::errorOccurred, this, [this](QCamera::Error, const QString& error) {
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call camera failed: %s", qPrintable(error)); m->report(error);
});
m->camera->start();
#else
Q_UNUSED(cameraId); Q_UNUSED(revision); m->report(tr("Video capture is unavailable in this build"));
#endif
}
void CallVideoEngine::transmit() {
call_video_caps caps = CallVideoDecoder::capabilities();
// Не связываем приём с наличием собственного encoder.
for (unsigned codec = 1; codec <= 2 && m->running; ++codec) {
call_video_format probe{1, 1280, 720, 30, 0, (uint8_t)codec, (uint8_t)(codec == 1 ? 31 : 93), 4000000,
(uint8_t)(codec == 1 ? CALL_VIDEO_CONSTRAINED_BASELINE : CALL_VIDEO_MAIN)};
CallVideoEncoder encoder;
if (encoder.open(probe, 1000000)) {
call_video_limit limit{1280, 720, 30, 1280 * 720 * 30, 4000000, probe.profile, probe.level};
caps.encode[codec - 1] = limit;
} else DEBUG_WARN(DEBUG_CATEGORY_CALL, "call codec probe unavailable codec=%u: %s", codec, qPrintable(encoder.error()));
}
if (!m->running) return;
if (call_video_port_open(m->id, &caps) < 0) { m->report(tr("Video transport is unavailable")); return; }
CallVideoEncoder encoder;
call_video_format format{};
uint64_t appliedRevision = 0, keySerial = 0, request = 0, frameId = 0;
uint64_t userRevision = 0, rejectedGeneration = 0;
unsigned excluded = 0;
int64_t lastPts = -1, lastKey = -1;
bool forceKey = true;
while (m->running) {
struct call_video_port_status state{};
if (call_video_port_status(m->id, &state) < 0) break;
uint64_t revision; bool enabled; int width, height, codec; int64_t pts;
#ifdef HAVE_STREAM_CAPTURE
QVideoFrame raw;
#endif
{ std::lock_guard<std::mutex> guard(m->lock);
revision = m->revision; enabled = m->enabled; width = m->width; height = m->height; codec = m->codec; pts = m->pts;
#ifdef HAVE_STREAM_CAPTURE
raw = m->raw;
#endif
}
if (revision != userRevision) { userRevision = revision; excluded = 0; }
if (format.generation && state.decoded_generation == format.generation && !state.decoded_accepted
&& rejectedGeneration != format.generation) {
rejectedGeneration = format.generation; excluded |= 1u << format.codec;
if (!codec) { appliedRevision = 0; lastKey = -1; m->report(tr("Receiver rejected video; selecting fallback")); }
else {
m->report(tr("Receiver cannot decode selected codec"));
std::lock_guard<std::mutex> guard(m->lock); m->enabled = false; continue;
}
}
if (format.generation && state.have_peer && !call_video_format_supported(&caps, &state.peer, &format)) appliedRevision = 0;
if (!enabled || !state.have_peer || pts <= lastPts || (lastPts >= 0 && pts - lastPts < 33333)) { idle(); continue; }
if (revision != appliedRevision) {
if (lastKey >= 0 && pts - lastKey < 1000000) { idle(); continue; }
call_video_format next{format.generation + 1, (uint16_t)width, (uint16_t)height, 30, 0, (uint8_t)codec, 0, 4000000, 0};
bool opened = false;
uint32_t minimum = state.minimum_bps ? state.minimum_bps : 300000;
uint32_t maximum = state.maximum_bps ? state.maximum_bps : 4000000;
uint32_t initialRate = std::max(minimum, std::min(maximum, encoder.bitrate() ? encoder.bitrate() : 1000000));
for (int candidate = 2; candidate >= 1; --candidate) {
if ((codec && codec != candidate) || (excluded & (1u << candidate))) continue;
next.codec = candidate; next.level = next.profile = 0;
if (call_video_choose(&caps, &state.peer, &next) > 0 && encoder.open(next, initialRate)) { opened = true; break; }
}
if (!opened) {
m->report(tr("Selected video mode is unavailable to the peer"));
std::lock_guard<std::mutex> guard(m->lock); if (m->revision == revision) m->enabled = false;
continue;
}
format = next; appliedRevision = revision; forceKey = true; request = 0;
if (call_video_port_format(m->id, &format, encoder.bitrate()) < 0) break;
m->report(QString("%1 · %2×%3 · %4 kbit/s").arg(format.codec == 2 ? "H.265" : "H.264").arg(width).arg(height).arg(encoder.bitrate() / 1000));
}
if (state.configured_generation != format.generation) { idle(); continue; }
if (state.key_serial != keySerial) { keySerial = state.key_serial; forceKey = true; }
if (state.rate_generation == format.generation && state.rate_request > request) {
request = state.rate_request;
if (!encoder.requestBitrate(format.generation, request, state.rate_bps))
call_video_port_applied(m->id, format.generation, request, 0);
}
#ifdef HAVE_STREAM_CAPTURE
QImage image = raw.toImage();
if (image.isNull()) { DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera frame conversion failed"); lastPts = pts; continue; }
LiveEncodedFrame output; CallVideoRateCompletion completion;
bool encoded = encoder.encode(image, pts, forceKey, output, completion); lastPts = pts;
if (completion.request) {
call_video_port_applied(m->id, completion.generation, completion.request, completion.bitrate);
m->report(QString("%1 · %2×%3 · %4 kbit/s").arg(format.codec == 2 ? "H.265" : "H.264")
.arg(format.width).arg(format.height).arg(encoder.bitrate() / 1000));
}
if (!encoded) { forceKey = true; m->report(encoder.error()); continue; }
if (output.key) lastKey = pts;
call_video_frame packet{};
packet.call_id = m->id; packet.frame_id = ++frameId; packet.pts_us = pts; packet.format = format;
packet.size = packet.length = output.data.size(); packet.data = reinterpret_cast<const uint8_t*>(output.data.constData());
packet.flags = output.key ? CALL_VIDEO_KEY : 0;
forceKey = call_video_port_send(&packet) != 1;
#else
idle();
#endif
}
}
void CallVideoEngine::receive() {
struct Picture { int64_t pts; QImage image; };
std::deque<Picture> pictures;
CallVideoDecoder decoder;
uint64_t sourceEpoch = 0; int64_t lastShown = -1, report = 0; unsigned shown = 0;
uint64_t reportedGeneration = 0, rejectedGeneration = 0;
while (m->running) {
int64_t now = nowUs(); call_clock_snapshot clock{};
call_audio_get_clock(m->id, now, &clock);
if (clock.valid) {
if (sourceEpoch && sourceEpoch != clock.source_epoch) {
pictures.clear(); decoder.close(); call_video_port_request_key(m->id); lastShown = -1;
DEBUG_INFO(DEBUG_CATEGORY_CALL, "video playback audio source epoch changed");
}
sourceEpoch = clock.source_epoch;
QImage selected;
while (!pictures.empty() && pictures.front().pts <= clock.pts_us) {
if (pictures.front().pts > lastShown && pictures.front().pts >= clock.pts_us - 500000) {
selected = pictures.front().image; lastShown = pictures.front().pts;
}
pictures.pop_front();
}
if (!selected.isNull()) { std::lock_guard<std::mutex> guard(m->lock); m->shown = selected; ++m->shownSerial; ++shown; }
}
if (pictures.size() < 16) {
call_video_frame packet{};
if (call_video_port_pull(m->id, &packet) == 1) {
if (packet.format.generation == rejectedGeneration) { u_free((void*)packet.data); continue; }
QImage image;
if (decoder.decode(packet, image)) {
pictures.push_back({packet.pts_us, image});
if (reportedGeneration != packet.format.generation) {
reportedGeneration = packet.format.generation; call_video_port_decoded(m->id, reportedGeneration, 1);
}
} else {
rejectedGeneration = packet.format.generation; call_video_port_decoded(m->id, rejectedGeneration, 0);
}
u_free((void*)packet.data); continue;
}
}
if (now - report >= 1000000) {
DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "video playback shown=%u queue=%zu audio_valid=%d audio_pts=%lld video_pts=%lld",
shown, pictures.size(), clock.valid, (long long)clock.pts_us, (long long)lastShown); report = now;
}
idle();
}
}