You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
303 lines
16 KiB
303 lines
16 KiB
#include "call_video_engine.h" |
|
#include "call_video_encoder.h" |
|
#include "call_video_decoder.h" |
|
#include "../../../src/call/call_video_port.h" |
|
#include "../../../src/call/call_audio.h" |
|
#include "debug_config.h" |
|
#include "mem.h" |
|
#include <thread> |
|
#include <mutex> |
|
#include <atomic> |
|
#include <deque> |
|
#include <chrono> |
|
#ifdef HAVE_STREAM_CAPTURE |
|
#include <QCamera> |
|
#include <QCameraDevice> |
|
#include <QMediaDevices> |
|
#include <QMediaCaptureSession> |
|
#include <QVideoSink> |
|
#include <QVideoFrame> |
|
#endif |
|
|
|
namespace { |
|
int64_t nowUs() { return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now().time_since_epoch()).count(); } |
|
void idle() { std::this_thread::sleep_for(std::chrono::milliseconds(5)); } |
|
} |
|
struct CallVideoEngine::Impl { |
|
std::mutex lock; |
|
std::thread tx, rx; |
|
std::atomic<bool> running{false}; |
|
uint64_t id = 0, revision = 1; |
|
bool enabled = false; |
|
int width = 640, height = 480, codec = 0; |
|
int64_t pts = -1, backend = -1, anchor = -1; |
|
QImage shown; |
|
uint64_t shownSerial = 0, deliveredSerial = 0; |
|
int64_t previewPts = -1, previewAt = 0; |
|
bool previewVisible = false; |
|
QString status; |
|
bool statusDirty = false; |
|
void report(const QString& text) { |
|
std::lock_guard<std::mutex> guard(lock); status = text; statusDirty = true; |
|
DEBUG_INFO(DEBUG_CATEGORY_CALL, "desktop video: %s", qPrintable(text)); |
|
} |
|
#ifdef HAVE_STREAM_CAPTURE |
|
std::unique_ptr<QCamera> camera; |
|
std::unique_ptr<QMediaCaptureSession> capture; |
|
std::unique_ptr<QVideoSink> sink; |
|
QVideoFrame raw; |
|
#endif |
|
}; |
|
CallVideoEngine::CallVideoEngine(QObject* parent) : QObject(parent), m(std::make_unique<Impl>()) { |
|
m_timer.setInterval(20); |
|
connect(&m_timer, &QTimer::timeout, this, [this] { |
|
QImage image; QString status; bool changed = false, report = false; |
|
{ std::lock_guard<std::mutex> guard(m->lock); |
|
if (m->shownSerial != m->deliveredSerial) { image = m->shown; m->deliveredSerial = m->shownSerial; changed = true; } |
|
if (m->statusDirty) { status = m->status; m->statusDirty = false; report = true; } } |
|
if (changed) emit imageChanged(image); |
|
if (report) emit statusChanged(status); |
|
#ifdef HAVE_STREAM_CAPTURE |
|
QVideoFrame preview; bool clear = false; |
|
int64_t now = nowUs(); |
|
{ std::lock_guard<std::mutex> guard(m->lock); |
|
if (!m->enabled && m->previewVisible) { m->previewVisible = false; clear = true; } |
|
if (m->enabled && m->raw.isValid() && m->pts != m->previewPts && now - m->previewAt >= 100000) { |
|
preview = m->raw; m->previewPts = m->pts; m->previewAt = now; |
|
} } |
|
if (clear) emit localImageChanged({}); |
|
if (preview.isValid()) { |
|
QImage local = preview.toImage(); |
|
if (!local.isNull()) { |
|
m->previewVisible = true; |
|
emit localImageChanged(local.scaled(320, 240, Qt::KeepAspectRatio, Qt::SmoothTransformation)); |
|
} else DEBUG_WARN(DEBUG_CATEGORY_CALL, "call camera preview conversion failed"); |
|
} |
|
#endif |
|
}); |
|
} |
|
CallVideoEngine::~CallVideoEngine() { stop(); } |
|
QList<QPair<QString, QByteArray>> CallVideoEngine::cameras() { |
|
QList<QPair<QString, QByteArray>> result; |
|
#ifdef HAVE_STREAM_CAPTURE |
|
for (const auto& camera : QMediaDevices::videoInputs()) result.append({camera.description(), camera.id()}); |
|
#endif |
|
return result; |
|
} |
|
void CallVideoEngine::start(quint64 id) { |
|
if (m->running && m->id == id) return; |
|
stop(); m->id = id; m->running = true; m_timer.start(); |
|
try { m->tx = std::thread(&CallVideoEngine::transmit, this); m->rx = std::thread(&CallVideoEngine::receive, this); } |
|
catch (const std::exception& error) { DEBUG_ERROR(DEBUG_CATEGORY_CALL, "video worker start: %s", error.what()); stop(); emit statusChanged(error.what()); } |
|
} |
|
void CallVideoEngine::closeCamera() { |
|
#ifdef HAVE_STREAM_CAPTURE |
|
if (m->sink) disconnect(m->sink.get(), nullptr, this, nullptr); |
|
if (m->camera) { disconnect(m->camera.get(), nullptr, this, nullptr); m->camera->stop(); } |
|
m->capture.reset(); m->camera.reset(); m->sink.reset(); |
|
{ std::lock_guard<std::mutex> guard(m->lock); m->raw = {}; m->pts = m->backend = m->anchor = -1; } |
|
#endif |
|
m->previewPts = -1; m->previewVisible = false; emit localImageChanged({}); |
|
} |
|
void CallVideoEngine::stop() { |
|
m->running = false; m_timer.stop(); closeCamera(); |
|
if (m->tx.joinable()) m->tx.join(); |
|
if (m->rx.joinable()) m->rx.join(); |
|
if (m->id) call_video_port_close(m->id); |
|
m->id = 0; m->enabled = false; m->shown = {}; emit imageChanged({}); |
|
} |
|
void CallVideoEngine::setVideo(bool enabled, const QByteArray& cameraId, int width, int height, int codec) { |
|
closeCamera(); |
|
uint64_t revision; |
|
{ std::lock_guard<std::mutex> guard(m->lock); |
|
m->enabled = enabled; m->width = width; m->height = height; m->codec = codec; revision = ++m->revision; } |
|
if (!enabled || !m->running) return; |
|
#ifdef HAVE_STREAM_CAPTURE |
|
QCameraDevice selected; |
|
for (const auto& camera : QMediaDevices::videoInputs()) if (camera.id() == cameraId) selected = camera; |
|
if (selected.isNull()) { m->report(tr("Camera is unavailable")); return; } |
|
m->capture = std::make_unique<QMediaCaptureSession>(); m->sink = std::make_unique<QVideoSink>(); |
|
m->camera = std::make_unique<QCamera>(selected); m->capture->setCamera(m->camera.get()); m->capture->setVideoSink(m->sink.get()); |
|
QCameraFormat best; int distance = INT_MAX; |
|
for (const auto& f : selected.videoFormats()) { |
|
if (f.maxFrameRate() < 29.9) continue; |
|
int score = std::abs(f.resolution().width() - width) + std::abs(f.resolution().height() - height); |
|
if (score < distance) { best = f; distance = score; } |
|
} |
|
if (!best.isNull()) m->camera->setCameraFormat(best); |
|
connect(m->sink.get(), &QVideoSink::videoFrameChanged, this, [this, revision](const QVideoFrame& frame) { |
|
if (!frame.isValid() || !m->running) return; |
|
std::lock_guard<std::mutex> guard(m->lock); |
|
if (revision != m->revision) return; |
|
int64_t pts = nowUs(); |
|
if (frame.startTime() >= 0) { |
|
if (m->backend < 0) { |
|
m->backend = frame.startTime(); m->anchor = pts; |
|
DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera clock source=%lld monotonic=%lld calibration=arrival-estimate", |
|
(long long)m->backend, (long long)m->anchor); |
|
} |
|
pts = m->anchor + frame.startTime() - m->backend; |
|
} |
|
if (pts > m->pts) { m->raw = frame; m->pts = pts; } |
|
}, Qt::DirectConnection); |
|
connect(m->camera.get(), &QCamera::errorOccurred, this, [this](QCamera::Error, const QString& error) { |
|
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call camera failed: %s", qPrintable(error)); m->report(error); |
|
}); |
|
m->camera->start(); |
|
#else |
|
Q_UNUSED(cameraId); Q_UNUSED(revision); m->report(tr("Video capture is unavailable in this build")); |
|
#endif |
|
} |
|
|
|
void CallVideoEngine::transmit() { |
|
call_video_caps caps = CallVideoDecoder::capabilities(); |
|
// Не связываем приём с наличием собственного encoder. |
|
for (unsigned codec = 1; codec <= 2 && m->running; ++codec) { |
|
call_video_format probe{1, 1280, 720, 30, 0, (uint8_t)codec, (uint8_t)(codec == 1 ? 31 : 93), 4000000, |
|
(uint8_t)(codec == 1 ? CALL_VIDEO_CONSTRAINED_BASELINE : CALL_VIDEO_MAIN)}; |
|
CallVideoEncoder encoder; |
|
if (encoder.open(probe, 1000000)) { |
|
call_video_limit limit{1280, 720, 30, 1280 * 720 * 30, 4000000, probe.profile, probe.level}; |
|
caps.encode[codec - 1] = limit; |
|
} else DEBUG_WARN(DEBUG_CATEGORY_CALL, "call codec probe unavailable codec=%u: %s", codec, qPrintable(encoder.error())); |
|
} |
|
if (!m->running) return; |
|
if (call_video_port_open(m->id, &caps) < 0) { m->report(tr("Video transport is unavailable")); return; } |
|
CallVideoEncoder encoder; |
|
call_video_format format{}; |
|
uint64_t appliedRevision = 0, keySerial = 0, request = 0, frameId = 0; |
|
uint64_t userRevision = 0, rejectedGeneration = 0; |
|
unsigned excluded = 0; |
|
int64_t lastPts = -1, lastKey = -1; |
|
bool forceKey = true; |
|
while (m->running) { |
|
struct call_video_port_status state{}; |
|
if (call_video_port_status(m->id, &state) < 0) break; |
|
if (state.error) { m->report(tr("Core rejected video format")); break; } |
|
uint64_t revision; bool enabled; int width, height, codec; int64_t pts; |
|
#ifdef HAVE_STREAM_CAPTURE |
|
QVideoFrame raw; |
|
#endif |
|
{ std::lock_guard<std::mutex> guard(m->lock); |
|
revision = m->revision; enabled = m->enabled; width = m->width; height = m->height; codec = m->codec; pts = m->pts; |
|
#ifdef HAVE_STREAM_CAPTURE |
|
raw = m->raw; |
|
#endif |
|
} |
|
if (revision != userRevision) { userRevision = revision; excluded = 0; } |
|
if (format.generation && state.decoded_generation == format.generation && !state.decoded_accepted |
|
&& rejectedGeneration != format.generation) { |
|
rejectedGeneration = format.generation; excluded |= 1u << format.codec; |
|
if (!codec) { appliedRevision = 0; lastKey = -1; m->report(tr("Receiver rejected video; selecting fallback")); } |
|
else { |
|
m->report(tr("Receiver cannot decode selected codec")); |
|
std::lock_guard<std::mutex> guard(m->lock); m->enabled = false; continue; |
|
} |
|
} |
|
if (format.generation && state.have_peer && !call_video_format_supported(&caps, &state.peer, &format)) appliedRevision = 0; |
|
if (!enabled || !state.have_peer || pts <= lastPts || (lastPts >= 0 && pts - lastPts < 33333)) { idle(); continue; } |
|
if (revision != appliedRevision) { |
|
if (lastKey >= 0 && pts - lastKey < 1000000) { idle(); continue; } |
|
call_video_format next{format.generation + 1, (uint16_t)width, (uint16_t)height, 30, 0, (uint8_t)codec, 0, 4000000, 0}; |
|
bool opened = false; |
|
uint32_t minimum = state.minimum_bps ? state.minimum_bps : 300000; |
|
uint32_t maximum = state.maximum_bps ? state.maximum_bps : 4000000; |
|
uint32_t initialRate = std::max(minimum, std::min(maximum, encoder.bitrate() ? encoder.bitrate() : 1000000)); |
|
for (int candidate = 2; candidate >= 1; --candidate) { |
|
if ((codec && codec != candidate) || (excluded & (1u << candidate))) continue; |
|
next.codec = candidate; next.level = next.profile = 0; |
|
if (call_video_choose(&caps, &state.peer, &next) > 0 && encoder.open(next, initialRate)) { opened = true; break; } |
|
} |
|
if (!opened) { |
|
m->report(tr("Selected video mode is unavailable to the peer")); |
|
std::lock_guard<std::mutex> guard(m->lock); if (m->revision == revision) m->enabled = false; |
|
continue; |
|
} |
|
format = next; appliedRevision = revision; forceKey = true; request = 0; |
|
if (call_video_port_format(m->id, &format, encoder.bitrate()) < 0) break; |
|
m->report(QString("%1 · %2×%3 · %4 kbit/s").arg(format.codec == 2 ? "H.265" : "H.264").arg(width).arg(height).arg(encoder.bitrate() / 1000)); |
|
} |
|
if (state.configured_generation != format.generation) { idle(); continue; } |
|
if (state.key_serial != keySerial) { keySerial = state.key_serial; forceKey = true; } |
|
if (state.rate_generation == format.generation && state.rate_request > request) { |
|
request = state.rate_request; |
|
if (!encoder.requestBitrate(format.generation, request, state.rate_bps)) |
|
call_video_port_applied(m->id, format.generation, request, 0); |
|
} |
|
#ifdef HAVE_STREAM_CAPTURE |
|
QImage image = raw.toImage(); |
|
if (image.isNull()) { DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera frame conversion failed"); lastPts = pts; continue; } |
|
LiveEncodedFrame output; CallVideoRateCompletion completion; |
|
bool encoded = encoder.encode(image, pts, forceKey, output, completion); lastPts = pts; |
|
if (completion.request) { |
|
call_video_port_applied(m->id, completion.generation, completion.request, completion.bitrate); |
|
m->report(QString("%1 · %2×%3 · %4 kbit/s").arg(format.codec == 2 ? "H.265" : "H.264") |
|
.arg(format.width).arg(format.height).arg(encoder.bitrate() / 1000)); |
|
} |
|
if (!encoded) { forceKey = true; m->report(encoder.error()); continue; } |
|
if (output.key) lastKey = pts; |
|
call_video_frame packet{}; |
|
packet.call_id = m->id; packet.frame_id = ++frameId; packet.pts_us = pts; packet.format = format; |
|
packet.size = packet.length = output.data.size(); packet.data = reinterpret_cast<const uint8_t*>(output.data.constData()); |
|
packet.flags = output.key ? CALL_VIDEO_KEY : 0; |
|
forceKey = call_video_port_send(&packet) != 1; |
|
#else |
|
idle(); |
|
#endif |
|
} |
|
} |
|
|
|
void CallVideoEngine::receive() { |
|
struct Picture { int64_t pts; QImage image; }; |
|
std::deque<Picture> pictures; |
|
CallVideoDecoder decoder; |
|
uint64_t sourceEpoch = 0; int64_t lastShown = -1, report = 0, lastPictureAt = 0; unsigned shown = 0; |
|
uint64_t reportedGeneration = 0, rejectedGeneration = 0; |
|
while (m->running) { |
|
int64_t now = nowUs(); call_clock_snapshot clock{}; |
|
call_audio_get_clock(m->id, now, &clock); |
|
if (clock.valid) { |
|
if (sourceEpoch && sourceEpoch != clock.source_epoch) { |
|
pictures.clear(); decoder.close(); call_video_port_request_key(m->id); lastShown = -1; |
|
DEBUG_INFO(DEBUG_CATEGORY_CALL, "video playback audio source epoch changed"); |
|
} |
|
sourceEpoch = clock.source_epoch; |
|
QImage selected; |
|
while (!pictures.empty() && pictures.front().pts <= clock.pts_us) { |
|
if (pictures.front().pts > lastShown && pictures.front().pts >= clock.pts_us - 500000) { |
|
selected = pictures.front().image; lastShown = pictures.front().pts; |
|
} |
|
pictures.pop_front(); |
|
} |
|
if (!selected.isNull()) { |
|
std::lock_guard<std::mutex> guard(m->lock); m->shown = selected; ++m->shownSerial; ++shown; lastPictureAt = now; |
|
} |
|
} |
|
// VIDEO_OFF пока отсутствует в протоколе: убираем застывшую картинку после прекращения показа. |
|
if (lastPictureAt && now - lastPictureAt >= 5000000) { |
|
std::lock_guard<std::mutex> guard(m->lock); m->shown = {}; ++m->shownSerial; lastPictureAt = 0; |
|
DEBUG_INFO(DEBUG_CATEGORY_CALL, "video playback hidden: no playable frame for 5s"); |
|
} |
|
if (pictures.size() < 16) { |
|
call_video_frame packet{}; |
|
if (call_video_port_pull(m->id, &packet) == 1) { |
|
if (packet.format.generation == rejectedGeneration) { u_free((void*)packet.data); continue; } |
|
QImage image; |
|
if (decoder.decode(packet, image)) { |
|
pictures.push_back({packet.pts_us, image}); |
|
if (reportedGeneration != packet.format.generation) { |
|
reportedGeneration = packet.format.generation; call_video_port_decoded(m->id, reportedGeneration, 1); |
|
} |
|
} else { |
|
rejectedGeneration = packet.format.generation; call_video_port_decoded(m->id, rejectedGeneration, 0); |
|
} |
|
u_free((void*)packet.data); continue; |
|
} |
|
} |
|
if (now - report >= 1000000) { |
|
DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "video playback shown=%u queue=%zu audio_valid=%d audio_pts=%lld video_pts=%lld", |
|
shown, pictures.size(), clock.valid, (long long)clock.pts_us, (long long)lastShown); report = now; |
|
} |
|
idle(); |
|
} |
|
}
|
|
|