251 lines
14 KiB
C++
251 lines
14 KiB
C++
#include "call_camera_source.h"
|
|
#include "call_camera_protocol.h"
|
|
#include "call_camera_buffer.h"
|
|
#include "call_video_capture_clock.h"
|
|
#include "call_video_cadence.h"
|
|
#include "call_camera_rate.h"
|
|
#include "../../../src/call/call_video_playout.h"
|
|
#include "debug_config.h"
|
|
#include <QGuiApplication>
|
|
#include <QTimer>
|
|
#include <mutex>
|
|
#include <climits>
|
|
#include <algorithm>
|
|
#include <atomic>
|
|
#include <cstring>
|
|
#ifdef HAVE_STREAM_CAPTURE
|
|
#include <QCamera>
|
|
#include <QMediaDevices>
|
|
#include <QMediaCaptureSession>
|
|
#include <QVideoSink>
|
|
|
|
using namespace CallCameraProtocol;
|
|
|
|
class QtCallCameraSource : public CallCameraSource {
|
|
std::unique_ptr<QCamera> camera;
|
|
std::unique_ptr<QMediaCaptureSession> session;
|
|
std::unique_ptr<QVideoSink> sink;
|
|
QMediaDevices devicesMonitor;
|
|
CallCameraRate rate;
|
|
public:
|
|
QtCallCameraSource() {
|
|
connect(&devicesMonitor, &QMediaDevices::videoInputsChanged, this, &CallCameraSource::devicesChanged);
|
|
}
|
|
~QtCallCameraSource() override { stop(); }
|
|
CallCameraDevices devices() override {
|
|
CallCameraDevices result;
|
|
for (const auto& device : QMediaDevices::videoInputs()) {
|
|
CallCameraDevice entry{device.description(), device.id(), {}};
|
|
for (const auto& format : device.videoFormats()) {
|
|
const QSize size = format.resolution();
|
|
DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "camera capability device=%s size=%dx%d pixel=%d fps=%.3f..%.3f",
|
|
device.id().constData(), size.width(), size.height(), format.pixelFormat(), format.minFrameRate(), format.maxFrameRate());
|
|
if (validSize(size.width(), size.height()))
|
|
stream_video_add_camera_modes(entry.modes, size, format.minFrameRate(), format.maxFrameRate());
|
|
}
|
|
DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera modes device=%s count=%lld limit=1920x1080 output_fps<=60",
|
|
device.id().constData(), (long long)entry.modes.size());
|
|
result.append(entry);
|
|
}
|
|
return result;
|
|
}
|
|
bool start(const QByteArray& device, int width, int height, int fps) override {
|
|
QCameraDevice selected;
|
|
for (const auto& item : QMediaDevices::videoInputs()) if (item.id() == device) selected = item;
|
|
if (selected.isNull()) { emit error(QStringLiteral("Camera is unavailable")); return false; }
|
|
session = std::make_unique<QMediaCaptureSession>(); sink = std::make_unique<QVideoSink>();
|
|
camera = std::make_unique<QCamera>(selected);
|
|
session->setCamera(camera.get()); session->setVideoSink(sink.get());
|
|
QCameraFormat best;
|
|
for (const auto& format : selected.videoFormats()) {
|
|
if (!stream_video_camera_supports(format.resolution(), format.minFrameRate(), format.maxFrameRate(),
|
|
{QSize(width, height), fps})) continue;
|
|
if (best.isNull() || format.maxFrameRate() < best.maxFrameRate()) best = format;
|
|
}
|
|
if (best.isNull()) {
|
|
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera mode unavailable device=%s size=%dx%d output_fps=%d", device.constData(), width, height, fps);
|
|
emit error(QStringLiteral("Selected camera mode is unavailable")); return false;
|
|
}
|
|
camera->setCameraFormat(best);
|
|
const auto format = camera->cameraFormat();
|
|
DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera selected device=%s size=%dx%d pixel=%d capture_fps=%.3f..%.3f output_fps=%d",
|
|
device.constData(), format.resolution().width(), format.resolution().height(), format.pixelFormat(),
|
|
format.minFrameRate(), format.maxFrameRate(), fps);
|
|
connect(camera.get(), &QCamera::activeChanged, this, [this, device](bool active) {
|
|
if (active) rate.activate(device); else rate.restore();
|
|
});
|
|
connect(camera.get(), &QCamera::errorOccurred, this, [this](QCamera::Error, const QString& message) { emit error(message); });
|
|
connect(sink.get(), &QVideoSink::videoFrameChanged, this, &CallCameraSource::frame, Qt::DirectConnection);
|
|
camera->start(); return true;
|
|
}
|
|
void stop() override {
|
|
if (sink) disconnect(sink.get(), nullptr, this, nullptr);
|
|
if (camera) { disconnect(camera.get(), nullptr, this, nullptr); camera->stop(); }
|
|
rate.restore(); session.reset(); camera.reset(); sink.reset();
|
|
}
|
|
};
|
|
|
|
class CameraWorker : public QObject {
|
|
struct Pending {
|
|
std::mutex lock;
|
|
QVideoFrame raw;
|
|
qint64 arrival = 0;
|
|
bool posted = false;
|
|
};
|
|
QLocalSocket socket;
|
|
CallCameraBuffer buffer;
|
|
uchar* pixels = nullptr;
|
|
Reader reader;
|
|
std::unique_ptr<CallCameraSource> source;
|
|
std::shared_ptr<Pending> pending;
|
|
QMetaObject::Connection captureConnection, errorConnection;
|
|
CallVideoCaptureClock clock;
|
|
CallVideoCadence cadence;
|
|
quint64 generation = 0, inFlight = 0;
|
|
bool sending = false;
|
|
int width = 640, height = 480, fps = 30;
|
|
qint64 lastPts = -1, reportAt = 0, convertMax = 0;
|
|
unsigned frames = 0, paced = 0;
|
|
|
|
void send(const QByteArray& data) {
|
|
if (socket.write(data) != data.size()) {
|
|
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera worker IPC write failed: %s", qPrintable(socket.errorString()));
|
|
QCoreApplication::exit(2);
|
|
}
|
|
socket.flush(); // Стадия должна уйти до потенциально блокирующего вызова драйвера.
|
|
}
|
|
void stage(const QString& name) { send(packet(Stage, generation, [&](QDataStream& out) { out << name; })); }
|
|
void enumerate() {
|
|
if (socket.state() != QLocalSocket::ConnectedState) return; // Первичный опрос повторит connected.
|
|
stage(QStringLiteral("enumerate"));
|
|
const auto devices = source->devices();
|
|
send(packet(Devices, 0, [&](QDataStream& out) {
|
|
out << quint32(devices.size());
|
|
for (const auto& device : devices) {
|
|
out << device.name << device.id << quint32(device.modes.size());
|
|
for (const auto& mode : device.modes) out << qint32(mode.size.width()) << qint32(mode.size.height()) << qint32(mode.fps);
|
|
}
|
|
}));
|
|
stage(pending ? QStringLiteral("running") : QStringLiteral("idle"));
|
|
}
|
|
void stop() {
|
|
stage(QStringLiteral("stop"));
|
|
disconnect(captureConnection); disconnect(errorConnection); source->stop(); pending.reset();
|
|
DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera stopped generation=%llu", (unsigned long long)generation);
|
|
stage(QStringLiteral("idle"));
|
|
}
|
|
void convert(std::shared_ptr<Pending> slot, quint64 revision) {
|
|
if (slot != pending || revision != generation || sending) return;
|
|
QVideoFrame raw; qint64 arrival;
|
|
{ std::lock_guard<std::mutex> guard(slot->lock);
|
|
raw.swap(slot->raw); arrival = slot->arrival; slot->posted = false; }
|
|
if (!raw.isValid()) return;
|
|
const qint64 started = call_video_playout_now_us(), sourcePts = raw.startTime();
|
|
if (clock.source >= 0 && sourcePts < clock.source)
|
|
DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera clock reset source=%lld previous=%lld", (long long)sourcePts, (long long)clock.source);
|
|
const qint64 pts = clock.map(sourcePts, arrival);
|
|
if (pts <= lastPts || started - pts > CallCameraMaxFrameAgeUs) {
|
|
DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera skips old frame age_us=%lld pts=%lld last=%lld",
|
|
(long long)(started - pts), (long long)pts, (long long)lastPts); return;
|
|
}
|
|
// Ограничиваем поток до дорогой конвертации и IPC; preview и TX получают выбранную частоту.
|
|
if (!cadence.accept(pts)) { ++paced; return; }
|
|
stage(QStringLiteral("convert"));
|
|
QImage image = raw.toImage();
|
|
if (image.isNull()) {
|
|
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera frame conversion failed");
|
|
send(packet(Error, generation, [](QDataStream& out) { out << QStringLiteral("Camera frame conversion failed"); })); return;
|
|
}
|
|
if (image.size() != QSize(width, height)) {
|
|
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera geometry changed requested=%dx%d actual=%dx%d", width, height, image.width(), image.height());
|
|
send(packet(Error, generation, [](QDataStream& out) { out << QStringLiteral("Camera changed the selected resolution"); })); return;
|
|
}
|
|
image = image.convertToFormat(QImage::Format_RGB32);
|
|
const qint64 converted = call_video_playout_now_us();
|
|
if (converted - pts <= CallCameraMaxFrameAgeUs) {
|
|
for (int row = 0; row < height; ++row) std::memcpy(pixels + size_t(row) * width * 4, image.constScanLine(row), size_t(width) * 4);
|
|
std::atomic_thread_fence(std::memory_order_release);
|
|
auto bytes = packet(Frame, generation, [&](QDataStream& out) {
|
|
out << pts << qint64(clock.correction) << qint32(image.width()) << qint32(image.height());
|
|
});
|
|
sending = true; inFlight = generation; lastPts = pts; send(bytes); ++frames;
|
|
} else DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera conversion exceeded frame age budget age_us=%lld", (long long)(converted - pts));
|
|
stage(QStringLiteral("release")); image = {}; raw = {}; stage(QStringLiteral("running"));
|
|
convertMax = std::max(convertMax, call_video_playout_now_us() - started);
|
|
if (started - reportAt >= 1000000) {
|
|
DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "camera flow output_fps=%d frames=%u paced=%u convert_max_us=%lld age_us=%lld clock_correction_us=%lld",
|
|
fps, frames, paced, (long long)convertMax, (long long)(converted - pts), (long long)clock.correction);
|
|
reportAt = started; convertMax = 0; frames = paced = 0;
|
|
}
|
|
}
|
|
bool receive(quint8 type, quint64 revision, QDataStream& in) {
|
|
if (type == Buffer) {
|
|
QString name; in >> name;
|
|
if (pixels) return false;
|
|
if (!buffer.attach(name)) {
|
|
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera frame mapping failed: %s", qPrintable(buffer.errorString())); return false;
|
|
}
|
|
pixels = buffer.data();
|
|
} else if (type == Quit) { stop(); QCoreApplication::quit(); }
|
|
else if (type == Ack) {
|
|
if (sending && revision == inFlight) { sending = false; if (pending) convert(pending, generation); }
|
|
} else if (type == Configure) {
|
|
bool enabled = false; QByteArray device; qint32 w = 0, h = 0, rate = 0; in >> enabled >> device >> w >> h >> rate;
|
|
if (in.status() != QDataStream::Ok || !validSize(w, h) || rate < 1 || rate > 60 || !pixels) return false;
|
|
stop(); generation = revision; width = w; height = h; fps = rate; clock = {}; cadence = CallVideoCadence(rate); lastPts = -1;
|
|
if (!enabled) { stage(QStringLiteral("idle")); return true; }
|
|
stage(QStringLiteral("start"));
|
|
pending = std::make_shared<Pending>();
|
|
errorConnection = connect(source.get(), &CallCameraSource::error, this, [this, revision](const QString& error) {
|
|
if (revision != generation) return;
|
|
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera backend: %s", qPrintable(error));
|
|
send(packet(Error, revision, [&](QDataStream& out) { out << error; }));
|
|
}, Qt::QueuedConnection);
|
|
captureConnection = connect(source.get(), &CallCameraSource::frame, this,
|
|
[this, slot = pending, revision](const QVideoFrame& frame) {
|
|
if (!frame.isValid()) return;
|
|
const qint64 arrival = call_video_playout_now_us();
|
|
QVideoFrame previous; bool post;
|
|
{ std::lock_guard<std::mutex> guard(slot->lock);
|
|
previous.swap(slot->raw); slot->raw = frame; slot->arrival = arrival;
|
|
post = !slot->posted; slot->posted = true; }
|
|
if (post) QMetaObject::invokeMethod(this, [this, slot, revision] { convert(slot, revision); }, Qt::QueuedConnection);
|
|
}, Qt::DirectConnection);
|
|
if (source->start(device, width, height, fps)) stage(QStringLiteral("running"));
|
|
} else return false;
|
|
return true;
|
|
}
|
|
public:
|
|
explicit CameraWorker(std::unique_ptr<CallCameraSource> backend) : source(std::move(backend)) {
|
|
connect(&socket, &QLocalSocket::connected, this, &CameraWorker::enumerate);
|
|
connect(source.get(), &CallCameraSource::devicesChanged, this, &CameraWorker::enumerate);
|
|
connect(&socket, &QLocalSocket::readyRead, this, [this] {
|
|
if (!reader.read(socket, [this](quint8 type, quint64 revision, QDataStream& in) { return receive(type, revision, in); })) {
|
|
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera worker invalid command"); QCoreApplication::exit(2);
|
|
}
|
|
});
|
|
connect(&socket, &QLocalSocket::disconnected, qApp, &QCoreApplication::quit);
|
|
connect(&socket, &QLocalSocket::errorOccurred, this, [this](QLocalSocket::LocalSocketError) {
|
|
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera worker IPC: %s", qPrintable(socket.errorString())); QCoreApplication::exit(2);
|
|
});
|
|
}
|
|
~CameraWorker() override { disconnect(captureConnection); source->stop(); }
|
|
void connectTo(const QString& endpoint) { socket.connectToServer(endpoint); }
|
|
};
|
|
|
|
int call_camera_worker_run(const QString& endpoint, std::unique_ptr<CallCameraSource> source) {
|
|
CameraWorker worker(std::move(source)); worker.connectTo(endpoint); return QCoreApplication::exec();
|
|
}
|
|
#endif
|
|
|
|
int call_camera_worker_main(int argc, char** argv) {
|
|
#ifdef HAVE_STREAM_CAPTURE
|
|
QGuiApplication app(argc, argv); debug_config_init(); debug_set_level(DEBUG_LEVEL_INFO);
|
|
debug_set_category_level(DEBUG_CATEGORY_CALL, DEBUG_LEVEL_DEBUG);
|
|
if (argc != 3) { DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera worker endpoint missing"); return 2; }
|
|
return call_camera_worker_run(QString::fromLocal8Bit(argv[2]), std::make_unique<QtCallCameraSource>());
|
|
#else
|
|
Q_UNUSED(argc); Q_UNUSED(argv); return 2;
|
|
#endif
|
|
}
|