diff --git a/tools/chatgui/src/call_video_engine.cpp b/tools/chatgui/src/call_video_engine.cpp new file mode 100644 index 00000000..67f6bb95 --- /dev/null +++ b/tools/chatgui/src/call_video_engine.cpp @@ -0,0 +1,242 @@ +#include "call_video_engine.h" +#include "call_video_encoder.h" +#include "call_video_decoder.h" +#include "../../../src/call/call_video_port.h" +#include "../../../src/call/call_audio.h" +#include "debug_config.h" +#include "mem.h" +#include +#include +#include +#include +#include +#ifdef HAVE_STREAM_CAPTURE +#include +#include +#include +#include +#include +#include +#endif + +namespace { +int64_t nowUs() { return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); } +void idle() { std::this_thread::sleep_for(std::chrono::milliseconds(5)); } +} +struct CallVideoEngine::Impl { + std::mutex lock; + std::thread tx, rx; + std::atomic running{false}; + uint64_t id = 0, revision = 1; + bool enabled = false; + int width = 640, height = 480, codec = 0; + int64_t pts = -1, backend = -1, anchor = -1; + QImage shown; + uint64_t shownSerial = 0, deliveredSerial = 0; + QString status; + bool statusDirty = false; + void report(const QString& text) { + std::lock_guard guard(lock); status = text; statusDirty = true; + DEBUG_INFO(DEBUG_CATEGORY_CALL, "desktop video: %s", qPrintable(text)); + } +#ifdef HAVE_STREAM_CAPTURE + std::unique_ptr camera; + std::unique_ptr capture; + std::unique_ptr sink; + QVideoFrame raw; +#endif +}; +CallVideoEngine::CallVideoEngine(QObject* parent) : QObject(parent), m(std::make_unique()) { + m_timer.setInterval(20); + connect(&m_timer, &QTimer::timeout, this, [this] { + QImage image; QString status; bool changed = false, report = false; + { std::lock_guard guard(m->lock); + if (m->shownSerial != m->deliveredSerial) { image = m->shown; m->deliveredSerial = m->shownSerial; changed = true; } + if (m->statusDirty) { status = m->status; m->statusDirty = false; report = true; } } + if (changed) emit imageChanged(image); + if (report) emit statusChanged(status); + }); +} +CallVideoEngine::~CallVideoEngine() { stop(); } +QList> CallVideoEngine::cameras() { + QList> result; +#ifdef HAVE_STREAM_CAPTURE + for (const auto& camera : QMediaDevices::videoInputs()) result.append({camera.description(), camera.id()}); +#endif + return result; +} +void CallVideoEngine::start(quint64 id) { + if (m->running && m->id == id) return; + stop(); m->id = id; m->running = true; m_timer.start(); + try { m->tx = std::thread(&CallVideoEngine::transmit, this); m->rx = std::thread(&CallVideoEngine::receive, this); } + catch (const std::exception& error) { DEBUG_ERROR(DEBUG_CATEGORY_CALL, "video worker start: %s", error.what()); stop(); emit statusChanged(error.what()); } +} +void CallVideoEngine::closeCamera() { +#ifdef HAVE_STREAM_CAPTURE + if (m->sink) disconnect(m->sink.get(), nullptr, this, nullptr); + if (m->camera) { disconnect(m->camera.get(), nullptr, this, nullptr); m->camera->stop(); } + m->capture.reset(); m->camera.reset(); m->sink.reset(); + std::lock_guard guard(m->lock); m->raw = {}; m->pts = m->backend = m->anchor = -1; +#endif +} +void CallVideoEngine::stop() { + m->running = false; m_timer.stop(); closeCamera(); + if (m->tx.joinable()) m->tx.join(); + if (m->rx.joinable()) m->rx.join(); + if (m->id) call_video_port_close(m->id); + m->id = 0; m->enabled = false; m->shown = {}; emit imageChanged({}); +} +void CallVideoEngine::setVideo(bool enabled, const QByteArray& cameraId, int width, int height, int codec) { + closeCamera(); + uint64_t revision; + { std::lock_guard guard(m->lock); + m->enabled = enabled; m->width = width; m->height = height; m->codec = codec; revision = ++m->revision; } + if (!enabled || !m->running) return; +#ifdef HAVE_STREAM_CAPTURE + QCameraDevice selected; + for (const auto& camera : QMediaDevices::videoInputs()) if (camera.id() == cameraId) selected = camera; + if (selected.isNull()) { m->report(tr("Camera is unavailable")); return; } + m->capture = std::make_unique(); m->sink = std::make_unique(); + m->camera = std::make_unique(selected); m->capture->setCamera(m->camera.get()); m->capture->setVideoSink(m->sink.get()); + QCameraFormat best; int distance = INT_MAX; + for (const auto& f : selected.videoFormats()) { + if (f.maxFrameRate() < 29.9) continue; + int score = std::abs(f.resolution().width() - width) + std::abs(f.resolution().height() - height); + if (score < distance) { best = f; distance = score; } + } + if (!best.isNull()) m->camera->setCameraFormat(best); + connect(m->sink.get(), &QVideoSink::videoFrameChanged, this, [this, revision](const QVideoFrame& frame) { + if (!frame.isValid() || !m->running) return; + std::lock_guard guard(m->lock); + if (revision != m->revision) return; + int64_t pts = nowUs(); + if (frame.startTime() >= 0) { + if (m->backend < 0) { + m->backend = frame.startTime(); m->anchor = pts; + DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera clock source=%lld monotonic=%lld calibration=arrival-estimate", + (long long)m->backend, (long long)m->anchor); + } + pts = m->anchor + frame.startTime() - m->backend; + } + if (pts > m->pts) { m->raw = frame; m->pts = pts; } + }, Qt::DirectConnection); + connect(m->camera.get(), &QCamera::errorOccurred, this, [this](QCamera::Error, const QString& error) { + DEBUG_ERROR(DEBUG_CATEGORY_CALL, "call camera failed: %s", qPrintable(error)); m->report(error); + }); + m->camera->start(); +#else + Q_UNUSED(cameraId); Q_UNUSED(revision); m->report(tr("Video capture is unavailable in this build")); +#endif +} + +void CallVideoEngine::transmit() { + call_video_caps caps{}; + // Каждый объявленный codec проверяется реальным encoder + независимым decoder при open. + for (unsigned codec = 1; codec <= 2 && m->running; ++codec) { + call_video_format probe{1, 1280, 720, 30, 0, (uint8_t)codec, (uint8_t)(codec == 1 ? 31 : 93), 4000000, 1000000}; + CallVideoEncoder encoder; + if (encoder.open(probe, 1000000)) { + call_video_limit limit{1280, 720, 30, 1280 * 720 * 30, 4000000, 1000000, probe.level}; + caps.encode[codec - 1] = caps.decode[codec - 1] = limit; + } else DEBUG_WARN(DEBUG_CATEGORY_CALL, "call codec probe unavailable codec=%u: %s", codec, qPrintable(encoder.error())); + } + if (!m->running) return; + if (call_video_port_open(m->id, &caps) < 0) { m->report(tr("Video transport is unavailable")); return; } + CallVideoEncoder encoder; + call_video_format format{}; + uint64_t appliedRevision = 0, keySerial = 0, request = 0, frameId = 0; + int64_t lastPts = -1, lastKey = -1; + bool forceKey = true; + while (m->running) { + call_video_port_status state{}; + if (call_video_port_status(m->id, &state) < 0) break; + uint64_t revision; bool enabled; int width, height, codec; int64_t pts; +#ifdef HAVE_STREAM_CAPTURE + QVideoFrame raw; +#endif + { std::lock_guard guard(m->lock); + revision = m->revision; enabled = m->enabled; width = m->width; height = m->height; codec = m->codec; pts = m->pts; +#ifdef HAVE_STREAM_CAPTURE + raw = m->raw; +#endif + } + if (!enabled || !state.have_peer || pts <= lastPts || (lastPts >= 0 && pts - lastPts < 33333)) { idle(); continue; } + if (revision != appliedRevision) { + if (lastKey >= 0 && pts - lastKey < 1000000) { idle(); continue; } + call_video_format next{format.generation + 1, (uint16_t)width, (uint16_t)height, 30, 0, (uint8_t)codec, 0, 4000000, 1000000}; + if (call_video_choose(&caps, &state.peer, &next) <= 0 || !encoder.open(next, 1000000)) { + m->report(tr("Selected video mode is unavailable to the peer")); + std::lock_guard guard(m->lock); if (m->revision == revision) m->enabled = false; + continue; + } + format = next; appliedRevision = revision; forceKey = true; request = 0; + if (call_video_port_format(m->id, &format, encoder.bitrate()) < 0) break; + m->report(QString("%1 · %2×%3 · %4 kbit/s").arg(format.codec == 2 ? "H.265" : "H.264").arg(width).arg(height).arg(encoder.bitrate() / 1000)); + } + if (state.configured_generation != format.generation) { idle(); continue; } + if (state.key_serial != keySerial) { keySerial = state.key_serial; forceKey = true; } + if (state.rate_generation == format.generation && state.rate_request > request) { + request = state.rate_request; + if (!encoder.requestBitrate(format.generation, request, state.rate_bps)) + call_video_port_applied(m->id, format.generation, request, 0); + } +#ifdef HAVE_STREAM_CAPTURE + QImage image = raw.toImage(); + if (image.isNull()) { DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera frame conversion failed"); lastPts = pts; continue; } + LiveEncodedFrame output; CallVideoRateCompletion completion; + bool encoded = encoder.encode(image, pts, forceKey, output, completion); lastPts = pts; + if (completion.request) { + call_video_port_applied(m->id, completion.generation, completion.request, completion.bitrate); + m->report(QString("%1 · %2×%3 · %4 kbit/s").arg(format.codec == 2 ? "H.265" : "H.264") + .arg(format.width).arg(format.height).arg(encoder.bitrate() / 1000)); + } + if (!encoded) { forceKey = true; m->report(encoder.error()); continue; } + if (output.key) lastKey = pts; + call_video_frame packet{}; + packet.call_id = m->id; packet.frame_id = ++frameId; packet.pts_us = pts; packet.format = format; + packet.size = packet.length = output.data.size(); packet.data = reinterpret_cast(output.data.constData()); + packet.flags = output.key ? CALL_VIDEO_KEY : 0; + forceKey = call_video_port_send(&packet) != 1; +#else + idle(); +#endif + } +} + +void CallVideoEngine::receive() { + struct Picture { int64_t pts; QImage image; }; + std::deque pictures; + CallVideoDecoder decoder; + uint64_t sourceEpoch = 0; int64_t lastShown = -1, report = 0; unsigned shown = 0; + while (m->running) { + int64_t now = nowUs(); call_clock_snapshot clock{}; + call_audio_get_clock(m->id, now, &clock); + if (clock.valid) { + if (sourceEpoch && sourceEpoch != clock.source_epoch) { + pictures.clear(); decoder.close(); call_video_port_request_key(m->id); lastShown = -1; + DEBUG_INFO(DEBUG_CATEGORY_CALL, "video playback audio source epoch changed"); + } + sourceEpoch = clock.source_epoch; + QImage selected; + while (!pictures.empty() && pictures.front().pts <= clock.pts_us) { + if (pictures.front().pts > lastShown) { selected = pictures.front().image; lastShown = pictures.front().pts; } + pictures.pop_front(); + } + if (!selected.isNull()) { std::lock_guard guard(m->lock); m->shown = selected; ++m->shownSerial; ++shown; } + } + if (pictures.size() < 16) { + call_video_frame packet{}; + if (call_video_port_pull(m->id, &packet) == 1) { + QImage image; + if (decoder.decode(packet, image)) pictures.push_back({packet.pts_us, image}); + else call_video_port_request_key(m->id); + u_free((void*)packet.data); continue; + } + } + if (now - report >= 1000000) { + DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "video playback shown=%u queue=%zu audio_valid=%d audio_pts=%lld video_pts=%lld", + shown, pictures.size(), clock.valid, (long long)clock.pts_us, (long long)lastShown); report = now; + } + idle(); + } +} diff --git a/tools/chatgui/src/call_video_engine.h b/tools/chatgui/src/call_video_engine.h new file mode 100644 index 00000000..6fd36a4d --- /dev/null +++ b/tools/chatgui/src/call_video_engine.h @@ -0,0 +1,27 @@ +#pragma once +#include +#include +#include +#include + +// GUI владеет камерой, TX/RX workers — codec; отображение читает фактический audio clock. +class CallVideoEngine : public QObject { + Q_OBJECT +public: + explicit CallVideoEngine(QObject* parent = nullptr); + ~CallVideoEngine() override; + void start(quint64 id); + void stop(); + void setVideo(bool enabled, const QByteArray& camera, int width, int height, int codec); + static QList> cameras(); +signals: + void imageChanged(const QImage& image); + void statusChanged(const QString& status); +private: + struct Impl; + std::unique_ptr m; + QTimer m_timer; + void closeCamera(); + void transmit(); + void receive(); +};