343 lines
19 KiB
C++
343 lines
19 KiB
C++
#include "call_video_engine.h"
|
||
#include "call_video_encoder.h"
|
||
#include "call_video_decoder.h"
|
||
#include "call_camera_capture.h"
|
||
#include "../../../src/call/call_video_port.h"
|
||
#include "../../../src/call/call_video_playout.h"
|
||
#include "../../../src/call/call_audio.h"
|
||
#include "debug_config.h"
|
||
#include "mem.h"
|
||
#include <thread>
|
||
#include <mutex>
|
||
#include <condition_variable>
|
||
#include <atomic>
|
||
#include <deque>
|
||
#include <chrono>
|
||
|
||
namespace {
|
||
int64_t nowUs() { return call_video_playout_now_us(); }
|
||
}
|
||
struct CallVideoEngine::Impl {
|
||
std::mutex lock;
|
||
std::condition_variable wake;
|
||
uint64_t wakeSerial = 0;
|
||
std::thread tx, rx;
|
||
std::atomic<bool> running{false};
|
||
uint64_t id = 0, revision = 1;
|
||
bool enabled = false;
|
||
int width = 640, height = 480, codec = 0;
|
||
int64_t pts = -1;
|
||
int64_t clockCorrection = 0;
|
||
QImage capturedImage;
|
||
QByteArray cameraId;
|
||
bool captureActive = false;
|
||
uint64_t captured = 0;
|
||
QImage shown;
|
||
int64_t shownPts = -1;
|
||
uint64_t publicationEpoch = 0;
|
||
bool publicationPending = false;
|
||
int64_t shownAt = 0;
|
||
int64_t previewReportAt = 0, previewMaxUs = 0;
|
||
unsigned previewFrames = 0;
|
||
QString status;
|
||
bool statusDirty = false;
|
||
void report(const QString& text) {
|
||
std::lock_guard<std::mutex> guard(lock); status = text; statusDirty = true;
|
||
DEBUG_INFO(DEBUG_CATEGORY_CALL, "desktop video: %s", qPrintable(text));
|
||
}
|
||
#ifdef HAVE_STREAM_CAPTURE
|
||
std::unique_ptr<CallCameraCapture> capture;
|
||
#endif
|
||
};
|
||
CallVideoEngine::CallVideoEngine(QObject* parent) : QObject(parent), m(std::make_unique<Impl>()) {
|
||
m_timer.setInterval(20);
|
||
m_timer.setTimerType(Qt::PreciseTimer);
|
||
connect(&m_timer, &QTimer::timeout, this, [this] {
|
||
QString status; bool report = false;
|
||
{ std::lock_guard<std::mutex> guard(m->lock);
|
||
if (m->statusDirty) { status = m->status; m->statusDirty = false; report = true; } }
|
||
if (report) emit statusChanged(status);
|
||
});
|
||
#ifdef HAVE_STREAM_CAPTURE
|
||
m->capture = std::make_unique<CallCameraCapture>();
|
||
connect(m->capture.get(), &CallCameraCapture::devicesChanged, this, &CallVideoEngine::camerasChanged);
|
||
connect(m->capture.get(), &CallCameraCapture::failed, this, [this](const QString& error) {
|
||
{ std::lock_guard<std::mutex> guard(m->lock);
|
||
m->enabled = false; ++m->revision; ++m->wakeSerial; m->capturedImage = {}; m->pts = -1; }
|
||
m->captureActive = false; m->wake.notify_one(); emit localImageChanged({});
|
||
m->report(error); emit statusChanged(error);
|
||
});
|
||
connect(m->capture.get(), &CallCameraCapture::frameReady, this, [this](const QImage& image, qint64 pts, qint64 correction) {
|
||
const int64_t now = nowUs();
|
||
{ std::lock_guard<std::mutex> guard(m->lock);
|
||
if (!m->running || !m->enabled || pts <= m->pts) return;
|
||
m->capturedImage = image; m->pts = pts; m->clockCorrection = correction; ++m->captured; ++m->wakeSerial; }
|
||
m->wake.notify_one();
|
||
emit localImageChanged(image.scaled(320, 240, Qt::KeepAspectRatio, Qt::FastTransformation));
|
||
++m->previewFrames; m->previewMaxUs = std::max(m->previewMaxUs, nowUs() - now);
|
||
if (now - m->previewReportAt >= 1000000) {
|
||
DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "video local preview call=%llu frames=%u update_max_us=%lld capture_age_us=%lld",
|
||
(unsigned long long)m->id, m->previewFrames, (long long)m->previewMaxUs, (long long)(now - pts));
|
||
m->previewReportAt = now; m->previewFrames = 0; m->previewMaxUs = 0;
|
||
}
|
||
});
|
||
#endif
|
||
}
|
||
CallVideoEngine::~CallVideoEngine() { stop(); }
|
||
// В GUI одновременно не больше одного уведомления; при паузе UI остаётся последний выбранный кадр.
|
||
void CallVideoEngine::publishImage(const QImage& image, int64_t pts) {
|
||
uint64_t epoch;
|
||
{ std::lock_guard<std::mutex> guard(m->lock);
|
||
m->shown = image; m->shownPts = pts; m->shownAt = nowUs();
|
||
if (m->publicationPending) return;
|
||
m->publicationPending = true; epoch = m->publicationEpoch; }
|
||
QMetaObject::invokeMethod(this, [this, epoch] {
|
||
QImage image; int64_t pts, selected;
|
||
{ std::lock_guard<std::mutex> guard(m->lock);
|
||
if (epoch != m->publicationEpoch) return;
|
||
image = m->shown; pts = m->shownPts; selected = m->shownAt; m->publicationPending = false; }
|
||
emit imageChanged(image);
|
||
if (!image.isNull() && pts >= 0 && pts % 1000000 < 34000)
|
||
DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "video timing stage=publish call=%llu pts=%lld mono_us=%lld ui_wait_us=%lld",
|
||
(unsigned long long)m->id, (long long)pts, (long long)nowUs(), (long long)(nowUs() - selected));
|
||
}, Qt::QueuedConnection);
|
||
}
|
||
void CallVideoEngine::start(quint64 id) {
|
||
if (m->running && m->id == id) return;
|
||
stop(); m->id = id; m->running = true; m_timer.start();
|
||
try { m->tx = std::thread(&CallVideoEngine::transmit, this); m->rx = std::thread(&CallVideoEngine::receive, this); }
|
||
catch (const std::exception& error) { DEBUG_ERROR(DEBUG_CATEGORY_CALL, "video worker start: %s", error.what()); stop(); emit statusChanged(error.what()); }
|
||
}
|
||
void CallVideoEngine::closeCamera() {
|
||
#ifdef HAVE_STREAM_CAPTURE
|
||
if (m->captureActive) m->capture->configure(false, m->cameraId, m->width, m->height);
|
||
#endif
|
||
m->captureActive = false;
|
||
{ std::lock_guard<std::mutex> guard(m->lock); m->capturedImage = {}; m->pts = -1; ++m->wakeSerial; }
|
||
m->wake.notify_one(); emit localImageChanged({});
|
||
}
|
||
void CallVideoEngine::stop() {
|
||
m->running = false; m_timer.stop(); closeCamera();
|
||
if (m->tx.joinable()) m->tx.join();
|
||
if (m->rx.joinable()) m->rx.join();
|
||
if (m->id) call_video_port_close(m->id);
|
||
{ std::lock_guard<std::mutex> guard(m->lock);
|
||
++m->publicationEpoch; m->publicationPending = false; m->shown = {}; m->shownPts = -1; }
|
||
m->id = 0; m->enabled = false; emit imageChanged({});
|
||
}
|
||
void CallVideoEngine::setVideo(bool enabled, const QByteArray& cameraId, int width, int height, int codec) {
|
||
const bool active = enabled && m->running;
|
||
const bool captureChanged = active != m->captureActive || cameraId != m->cameraId || width != m->width || height != m->height;
|
||
{ std::lock_guard<std::mutex> guard(m->lock);
|
||
if (m->enabled == enabled && cameraId == m->cameraId && width == m->width && height == m->height && codec == m->codec
|
||
&& !captureChanged) return;
|
||
m->enabled = enabled; m->width = width; m->height = height; m->codec = codec; ++m->revision; ++m->wakeSerial;
|
||
if (captureChanged) { m->capturedImage = {}; m->pts = -1; } }
|
||
m->cameraId = cameraId; m->captureActive = active; m->wake.notify_one();
|
||
if (!captureChanged) return;
|
||
emit localImageChanged({});
|
||
#ifdef HAVE_STREAM_CAPTURE
|
||
m->capture->configure(active, cameraId, width, height);
|
||
#else
|
||
if (active) m->report(tr("Video capture is unavailable in this build"));
|
||
#endif
|
||
}
|
||
|
||
void CallVideoEngine::transmit() {
|
||
call_video_caps caps = CallVideoDecoder::capabilities();
|
||
// Не связываем приём с наличием собственного encoder.
|
||
for (unsigned codec = 1; codec <= 2 && m->running; ++codec) {
|
||
call_video_format probe{1, 1280, 720, 30, 0, (uint8_t)codec, (uint8_t)(codec == 1 ? 31 : 93), 4000000,
|
||
(uint8_t)(codec == 1 ? CALL_VIDEO_CONSTRAINED_BASELINE : CALL_VIDEO_MAIN)};
|
||
CallVideoEncoder encoder;
|
||
if (encoder.open(probe, 1000000)) {
|
||
call_video_limit limit{1280, 720, 30, 1280 * 720 * 30, 4000000, probe.profile, probe.level};
|
||
caps.encode[codec - 1] = limit;
|
||
} else DEBUG_WARN(DEBUG_CATEGORY_CALL, "call codec probe unavailable codec=%u: %s", codec, qPrintable(encoder.error()));
|
||
}
|
||
if (!m->running) return;
|
||
if (call_video_port_open(m->id, &caps) < 0) { m->report(tr("Video transport is unavailable")); return; }
|
||
CallVideoEncoder encoder;
|
||
call_video_format format{};
|
||
uint64_t appliedRevision = 0, keySerial = 0, request = 0, frameId = 0;
|
||
uint64_t userRevision = 0, rejectedGeneration = 0;
|
||
unsigned excluded = 0;
|
||
int64_t lastPts = -1;
|
||
bool forceKey = true;
|
||
int64_t reportAt = nowUs(), encodeMax = 0, inputAgeMax = 0;
|
||
uint64_t captureBase = 0;
|
||
unsigned encodedCount = 0, keyCount = 0, sendBusy = 0, expired = 0;
|
||
uint64_t wakeSerial = 0;
|
||
auto idle = [this, &wakeSerial] {
|
||
std::unique_lock<std::mutex> guard(m->lock);
|
||
m->wake.wait_for(guard, std::chrono::milliseconds(5), [&] { return !m->running || m->wakeSerial != wakeSerial; });
|
||
};
|
||
while (m->running) {
|
||
struct call_video_port_status state{};
|
||
if (call_video_port_status(m->id, &state) < 0) break;
|
||
if (state.error) { m->report(tr("Core rejected video format")); break; }
|
||
uint64_t revision, captured; bool enabled; int width, height, codec; int64_t pts, clockCorrection;
|
||
QImage image;
|
||
{ std::lock_guard<std::mutex> guard(m->lock);
|
||
revision = m->revision; enabled = m->enabled; width = m->width; height = m->height; codec = m->codec; pts = m->pts;
|
||
captured = m->captured;
|
||
clockCorrection = m->clockCorrection; image = m->capturedImage; wakeSerial = m->wakeSerial;
|
||
}
|
||
int64_t now = nowUs();
|
||
if (now - reportAt >= 1000000) {
|
||
DEBUG_DEBUG(DEBUG_CATEGORY_DEBUG,
|
||
"video capture flow call=%llu interval_us=%lld captured=%llu encoded=%u keys=%u send_busy=%u "
|
||
"expired=%u encode_max_us=%lld input_age_max_us=%lld capture_age_us=%lld pts=%lld clock_correction_us=%lld",
|
||
(unsigned long long)m->id, (long long)(now - reportAt), (unsigned long long)(captured - captureBase),
|
||
encodedCount, keyCount, sendBusy, expired, (long long)encodeMax, (long long)inputAgeMax,
|
||
(long long)(pts >= 0 ? now - pts : 0), (long long)pts, (long long)clockCorrection);
|
||
reportAt = now; captureBase = captured; encodedCount = keyCount = sendBusy = expired = 0;
|
||
encodeMax = inputAgeMax = 0;
|
||
}
|
||
if (revision != userRevision) { userRevision = revision; excluded = 0; }
|
||
if (format.generation && state.decoded_generation == format.generation && !state.decoded_accepted
|
||
&& rejectedGeneration != format.generation) {
|
||
rejectedGeneration = format.generation; excluded |= 1u << format.codec;
|
||
if (!codec) { appliedRevision = 0; m->report(tr("Receiver rejected video; selecting fallback")); }
|
||
else {
|
||
m->report(tr("Receiver cannot decode selected codec"));
|
||
std::lock_guard<std::mutex> guard(m->lock); m->enabled = false; continue;
|
||
}
|
||
}
|
||
if (format.generation && state.have_peer && !call_video_format_supported(&caps, &state.peer, &format)) appliedRevision = 0;
|
||
if (!enabled || !state.have_peer || pts <= lastPts) { idle(); continue; }
|
||
if (revision != appliedRevision) {
|
||
call_video_format next{format.generation + 1, (uint16_t)width, (uint16_t)height, 30, 0, (uint8_t)codec, 0, 4000000, 0};
|
||
bool opened = false;
|
||
uint32_t minimum = state.minimum_bps ? state.minimum_bps : 300000;
|
||
uint32_t maximum = state.maximum_bps ? state.maximum_bps : 4000000;
|
||
uint32_t initialRate = std::max(minimum, std::min(maximum, encoder.bitrate() ? encoder.bitrate() : 1000000));
|
||
for (int candidate = 2; candidate >= 1; --candidate) {
|
||
if ((codec && codec != candidate) || (excluded & (1u << candidate))) continue;
|
||
next.codec = candidate; next.level = next.profile = 0;
|
||
if (call_video_choose(&caps, &state.peer, &next) > 0 && encoder.open(next, initialRate)) { opened = true; break; }
|
||
}
|
||
if (!opened) {
|
||
m->report(tr("Selected video mode is unavailable to the peer"));
|
||
std::lock_guard<std::mutex> guard(m->lock); if (m->revision == revision) m->enabled = false;
|
||
continue;
|
||
}
|
||
format = next; appliedRevision = revision; forceKey = true; request = 0;
|
||
if (call_video_port_format(m->id, &format, encoder.bitrate()) < 0) break;
|
||
m->report(QString("%1 · %2×%3 · %4 kbit/s").arg(format.codec == 2 ? "H.265" : "H.264").arg(width).arg(height).arg(encoder.bitrate() / 1000));
|
||
}
|
||
if (state.configured_generation != format.generation) { idle(); continue; }
|
||
if (state.key_serial != keySerial) { keySerial = state.key_serial; forceKey = true; }
|
||
if (state.rate_generation == format.generation && state.rate_request > request) {
|
||
request = state.rate_request;
|
||
if (!encoder.requestBitrate(format.generation, request, state.rate_bps))
|
||
call_video_port_applied(m->id, format.generation, request, 0);
|
||
}
|
||
#ifdef HAVE_STREAM_CAPTURE
|
||
if (image.isNull() || nowUs() - pts > CallCameraMaxFrameAgeUs) {
|
||
DEBUG_WARN(DEBUG_CATEGORY_CALL, "video TX skips old capture age_us=%lld", (long long)(nowUs() - pts));
|
||
lastPts = pts; ++expired; continue;
|
||
}
|
||
QList<LiveEncodedFrame> outputs; CallVideoRateCompletion completion;
|
||
int64_t encodeStarted = nowUs();
|
||
inputAgeMax = std::max(inputAgeMax, encodeStarted - pts);
|
||
bool encoded = encoder.encode(image, pts, forceKey, outputs, completion); lastPts = pts;
|
||
encodeMax = std::max(encodeMax, nowUs() - encodeStarted);
|
||
if (completion.request) {
|
||
call_video_port_applied(m->id, completion.generation, completion.request, completion.bitrate);
|
||
m->report(QString("%1 · %2×%3 · %4 kbit/s").arg(format.codec == 2 ? "H.265" : "H.264")
|
||
.arg(format.width).arg(format.height).arg(encoder.bitrate() / 1000));
|
||
}
|
||
if (!encoded) { forceKey = true; m->report(encoder.error()); continue; }
|
||
// GPU может вернуть несколько уже готовых кадров: отправляем все, не добавляя очередь в TX worker.
|
||
if (outputs.isEmpty()) continue;
|
||
if (outputs.size() > CALL_VIDEO_PORT_TX_FRAMES) {
|
||
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "GPU output batch exceeds mailbox frames=%lld", (long long)outputs.size());
|
||
forceKey = true; continue;
|
||
}
|
||
call_video_frame packets[CALL_VIDEO_PORT_TX_FRAMES]{}; unsigned count = 0;
|
||
for (const auto& output : outputs) {
|
||
++encodedCount; if (output.key) ++keyCount;
|
||
auto& packet = packets[count++];
|
||
packet.call_id = m->id; packet.frame_id = ++frameId; packet.pts_us = output.pts; packet.format = format;
|
||
packet.size = packet.length = output.data.size(); packet.data = reinterpret_cast<const uint8_t*>(output.data.constData());
|
||
packet.flags = output.key ? CALL_VIDEO_KEY : 0;
|
||
}
|
||
{ std::lock_guard<std::mutex> guard(m->lock);
|
||
if (!m->running || !m->enabled || m->revision != revision || nowUs() - packets[0].pts_us > CallCameraMaxFrameAgeUs) {
|
||
DEBUG_WARN(DEBUG_CATEGORY_CALL, "video TX discards obsolete batch revision=%llu/%llu enabled=%d age_us=%lld frames=%u",
|
||
(unsigned long long)revision, (unsigned long long)m->revision, m->enabled,
|
||
(long long)(nowUs() - packets[0].pts_us), count);
|
||
expired += count; forceKey = true; continue;
|
||
}
|
||
forceKey = call_video_port_send(packets, count) != 1; }
|
||
for (const auto& output : outputs) {
|
||
// Одинаковое окно исходного PTS на всех этапах: примерно один кадр в секунду, без трассы каждого кадра.
|
||
if (output.pts % 1000000 < 34000)
|
||
DEBUG_DEBUG(DEBUG_CATEGORY_CALL,
|
||
"video timing stage=encode call=%llu frame=%llu pts=%lld mono_us=%lld encode_us=%lld accepted=%d",
|
||
(unsigned long long)m->id, (unsigned long long)(frameId - count + 1 + &output - outputs.constData()),
|
||
(long long)output.pts, (long long)nowUs(),
|
||
(long long)(nowUs() - encodeStarted), !forceKey);
|
||
}
|
||
if (forceKey) ++sendBusy;
|
||
#else
|
||
idle();
|
||
#endif
|
||
}
|
||
}
|
||
|
||
void CallVideoEngine::receive() {
|
||
struct Picture { call_video_playout_frame timing; QImage image; };
|
||
std::deque<Picture> pictures;
|
||
CallVideoDecoder decoder;
|
||
call_video_playout playout; call_video_playout_init(&playout, m->id);
|
||
int64_t lastPictureAt = 0;
|
||
uint64_t reportedGeneration = 0, rejectedGeneration = 0;
|
||
while (m->running) {
|
||
// Один кадр ждёт своего срока, второй позволяет пропустить устаревший показ после паузы CPU.
|
||
while (pictures.size() < 2 && m->running) {
|
||
call_video_frame packet{};
|
||
if (call_video_port_pull(m->id, &packet) == 1) {
|
||
if (packet.format.generation == rejectedGeneration) { u_free((void*)packet.data); continue; }
|
||
QImage image;
|
||
const int64_t decodeStarted = nowUs();
|
||
if (decoder.decode(packet, image)) {
|
||
pictures.push_back({{packet.pts_us, packet.received_us}, image});
|
||
if (packet.pts_us % 1000000 < 34000)
|
||
DEBUG_DEBUG(DEBUG_CATEGORY_CALL,
|
||
"video timing stage=decode call=%llu frame=%llu pts=%lld mono_us=%lld decode_us=%lld queue=%zu",
|
||
(unsigned long long)m->id, (unsigned long long)packet.frame_id, (long long)packet.pts_us,
|
||
(long long)nowUs(), (long long)(nowUs() - decodeStarted), pictures.size());
|
||
if (reportedGeneration != packet.format.generation) {
|
||
reportedGeneration = packet.format.generation; call_video_port_decoded(m->id, reportedGeneration, 1);
|
||
}
|
||
} else {
|
||
rejectedGeneration = packet.format.generation; call_video_port_decoded(m->id, rejectedGeneration, 0);
|
||
}
|
||
u_free((void*)packet.data);
|
||
} else break;
|
||
}
|
||
int64_t now = nowUs(); call_clock_snapshot clock{};
|
||
call_audio_get_clock(m->id, now, &clock);
|
||
auto action = call_video_playout_poll(&playout, pictures.empty() ? nullptr : &pictures.front().timing,
|
||
pictures.size() > 1 ? &pictures[1].timing : nullptr, now, clock.valid, clock.pts_us);
|
||
if (action != CALL_VIDEO_PLAYOUT_WAIT) {
|
||
if (action == CALL_VIDEO_PLAYOUT_SHOW) {
|
||
const auto& f = pictures.front(); publishImage(f.image, f.timing.pts_us); lastPictureAt = now;
|
||
if (f.timing.pts_us % 1000000 < 34000)
|
||
DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "video timing stage=select call=%llu pts=%lld mono_us=%lld audio_pts=%lld",
|
||
(unsigned long long)m->id, (long long)f.timing.pts_us, (long long)now, (long long)clock.pts_us);
|
||
}
|
||
pictures.pop_front();
|
||
continue;
|
||
}
|
||
// VIDEO_OFF пока отсутствует в протоколе: убираем застывшую картинку после прекращения показа.
|
||
if (lastPictureAt && now - lastPictureAt >= 5000000) {
|
||
publishImage({}, -1); lastPictureAt = 0;
|
||
DEBUG_INFO(DEBUG_CATEGORY_CALL, "video playback hidden: no playable frame for 5s");
|
||
}
|
||
std::this_thread::sleep_for(std::chrono::microseconds(std::max<int64_t>(100, std::min<int64_t>(5000, playout.deadline_us - nowUs()))));
|
||
}
|
||
}
|