Isolate screen capture lifecycle and report worker close stages
This commit is contained in:
1 parent
48998b440f
commit
e299a9b30b
3 files changed
+289
-84
No files matched your search
@@ -23,14 +23,17 @@ struct VideoCapture::Session : QObject {
|
||||
VideoCaptureBuffer buffer;
|
||||
uchar* pixels = nullptr;
|
||||
Reader reader;
|
||||
Usage usage;
|
||||
VideoCaptureKind kind = VideoCaptureKind::Camera;
|
||||
bool cursor = true, stopping = false, forced = false;
|
||||
quint64 generation = 0;
|
||||
bool enabled = false, retiring = false, failureReported = false, restartRequested = false, exiting = false;
|
||||
QByteArray device;
|
||||
int width = 640, height = 480, fps = 30;
|
||||
QString phase = QStringLiteral("launch");
|
||||
qint64 progressAt = 0, frameAt = 0, stopAt = 0;
|
||||
qint64 progressAt = 0, frameAt = 0, stopAt = 0, stopStarted = 0, stageAt = 0;
|
||||
|
||||
explicit Session(VideoCapture* target) : owner(target) {
|
||||
explicit Session(VideoCapture* target, Usage use) : owner(target), usage(use) {
|
||||
server.setSocketOptions(QLocalServer::UserAccessOption);
|
||||
connect(&server, &QLocalServer::newConnection, this, [this] {
|
||||
auto* connection = server.nextPendingConnection();
|
||||
@@ -43,7 +46,9 @@ struct VideoCapture::Session : QObject {
|
||||
});
|
||||
if (retiring) send(packet(Quit, generation));
|
||||
else {
|
||||
send(packet(Buffer, 0, [this](QDataStream& out) { out << buffer.name(); }));
|
||||
send(packet(Buffer, 0, [this](QDataStream& out) {
|
||||
out << buffer.name() << quint32(usage == Usage::Stream ? MaxFrameBytes : FrameBytes);
|
||||
}));
|
||||
sendConfiguration();
|
||||
}
|
||||
});
|
||||
@@ -65,22 +70,30 @@ struct VideoCapture::Session : QObject {
|
||||
}
|
||||
});
|
||||
connect(&process, QOverload<int, QProcess::ExitStatus>::of(&QProcess::finished), this, [this](int code, QProcess::ExitStatus status) {
|
||||
DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera worker exited code=%d status=%d retiring=%d", code, int(status), retiring);
|
||||
DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "capture process exited pid=%lld generation=%llu code=%d status=%d expected=%d forced=%d "
|
||||
"last_stage=%s stop_ms=%lld", (long long)process.processId(), (unsigned long long)generation, code, int(status),
|
||||
stopping, forced, qPrintable(phase), stopStarted ? (long long)((call_video_playout_now_us() - stopStarted) / 1000) : 0);
|
||||
if (socket) { socket->disconnect(this); socket->deleteLater(); socket = nullptr; }
|
||||
server.close(); watchdog.stop();
|
||||
const bool expected = stopping; stopping = false; stopAt = 0;
|
||||
if (owner && expected) emit owner->stopped();
|
||||
if (retiring) deleteLater();
|
||||
else if (restartRequested) { restartRequested = false; launch(); }
|
||||
else if (!failureReported) fail(QStringLiteral("Camera worker stopped"));
|
||||
else if (!expected && !failureReported) fail(QStringLiteral("Capture worker stopped unexpectedly"));
|
||||
});
|
||||
watchdog.setInterval(250);
|
||||
connect(&watchdog, &QTimer::timeout, this, [this] {
|
||||
const qint64 now = call_video_playout_now_us();
|
||||
if (stopAt && now >= stopAt) {
|
||||
DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera stop timed out phase=%s; killing worker", qPrintable(phase));
|
||||
DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "capture stop timeout pid=%lld generation=%llu elapsed_ms=%lld "
|
||||
"last_stage=%s stage_age_ms=%lld; killing worker", (long long)process.processId(),
|
||||
(unsigned long long)generation, (long long)((now - stopStarted) / 1000), qPrintable(phase),
|
||||
(long long)((now - stageAt) / 1000));
|
||||
stopAt = 0;
|
||||
if (!retiring) fail(QStringLiteral("Camera did not stop")); else process.kill();
|
||||
} else if (!retiring && !failureReported && ((phase != "idle" && now - progressAt >= 5000000)
|
||||
|| (enabled && now - frameAt >= 5000000))) {
|
||||
forced = true;
|
||||
if (!stopping && !retiring) fail(QStringLiteral("Capture did not stop")); else process.kill();
|
||||
} else if (!stopping && !retiring && !failureReported && ((phase != "idle" && now - progressAt >= 5000000)
|
||||
|| (enabled && kind == VideoCaptureKind::Camera && now - frameAt >= 5000000))) {
|
||||
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera stalled generation=%llu phase=%s operation_age_us=%lld frame_age_us=%lld",
|
||||
(unsigned long long)generation, qPrintable(phase), (long long)(now - progressAt), (long long)(now - frameAt));
|
||||
fail(QStringLiteral("Camera stopped responding"));
|
||||
@@ -98,15 +111,18 @@ struct VideoCapture::Session : QObject {
|
||||
server.close();
|
||||
reader = {}; failureReported = false; phase = QStringLiteral("launch"); progressAt = call_video_playout_now_us();
|
||||
if (!pixels) {
|
||||
if (!buffer.create()) {
|
||||
if (!buffer.create(usage == Usage::Stream ? MaxFrameBytes : FrameBytes)) {
|
||||
fail(QStringLiteral("Cannot allocate camera frame buffer: %1").arg(buffer.errorString())); return;
|
||||
}
|
||||
pixels = buffer.data();
|
||||
}
|
||||
const QString name = QStringLiteral("vibechat-camera-") + QUuid::createUuid().toString(QUuid::Id128);
|
||||
if (!server.listen(name)) { fail(QStringLiteral("Cannot create camera IPC: %1").arg(server.errorString())); return; }
|
||||
DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera worker starting; hardware isolated from GUI");
|
||||
process.start(QCoreApplication::applicationFilePath(), {QStringLiteral("--video-capture-worker"), server.fullServerName()});
|
||||
stopping = forced = false; stopAt = stopStarted = 0;
|
||||
DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "capture process starting generation=%llu usage=%s buffer_bytes=%u",
|
||||
(unsigned long long)generation, usage == Usage::Stream ? "stream" : "call", usage == Usage::Stream ? MaxFrameBytes : FrameBytes);
|
||||
process.start(QCoreApplication::applicationFilePath(), {QStringLiteral("--video-capture-worker"), server.fullServerName(),
|
||||
usage == Usage::Stream ? QStringLiteral("stream") : QStringLiteral("call")});
|
||||
watchdog.start();
|
||||
}
|
||||
void send(const QByteArray& data) {
|
||||
@@ -116,27 +132,42 @@ struct VideoCapture::Session : QObject {
|
||||
}
|
||||
void sendConfiguration() {
|
||||
send(packet(Configure, generation, [this](QDataStream& out) {
|
||||
out << enabled << device << qint32(width) << qint32(height) << qint32(fps);
|
||||
out << enabled << device << qint32(width) << qint32(height) << qint32(fps) << quint8(kind) << cursor;
|
||||
}));
|
||||
}
|
||||
void configure(bool active, const QByteArray& id, int w, int h, int rate) {
|
||||
void configure(const VideoCaptureConfig& config) {
|
||||
if (exiting) return;
|
||||
if (!validSize(w, h) || rate < 1 || rate > 60) { fail(QStringLiteral("Invalid camera size/frame rate")); return; }
|
||||
if (enabled == active && device == id && width == w && height == h && fps == rate && !failureReported) return;
|
||||
const auto [active, id, w, h, rate, nextKind, showCursor] = config;
|
||||
if (stopping) { DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "capture configure refused while process is closing"); return; }
|
||||
if (!(usage == Usage::Stream ? validStreamSize(w, h) : validSize(w, h)) || rate < 1 || rate > 60
|
||||
|| quint8(nextKind) > quint8(VideoCaptureKind::Window) || (usage == Usage::Call && nextKind != VideoCaptureKind::Camera)) {
|
||||
fail(QStringLiteral("Invalid capture source/size/frame rate")); return;
|
||||
}
|
||||
if (enabled == active && device == id && width == w && height == h && fps == rate
|
||||
&& kind == nextKind && cursor == showCursor && !failureReported) return;
|
||||
kind = nextKind; cursor = showCursor;
|
||||
enabled = active; device = id; width = w; height = h; fps = rate; ++generation;
|
||||
progressAt = call_video_playout_now_us();
|
||||
frameAt = progressAt;
|
||||
stopAt = active ? 0 : progressAt + 1500000;
|
||||
stopStarted = active ? 0 : progressAt; stopAt = active ? 0 : progressAt + 1500000;
|
||||
DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera request generation=%llu enabled=%d size=%dx%d output_fps=%d device=%s",
|
||||
(unsigned long long)generation, active, w, h, rate, id.constData());
|
||||
if (failureReported && active) restartRequested = true;
|
||||
if (process.state() == QProcess::NotRunning) { restartRequested = false; launch(); }
|
||||
if (socket) sendConfiguration();
|
||||
}
|
||||
void stop() {
|
||||
if (stopping || process.state() == QProcess::NotRunning) return;
|
||||
stopping = true; enabled = false; ++generation;
|
||||
stopStarted = call_video_playout_now_us(); stopAt = stopStarted + 1500000;
|
||||
DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "capture stop requested pid=%lld generation=%llu last_stage=%s deadline_ms=1500",
|
||||
(long long)process.processId(), (unsigned long long)generation, qPrintable(phase));
|
||||
watchdog.start(); send(packet(Quit, generation));
|
||||
}
|
||||
void shutdown() {
|
||||
owner = nullptr; retiring = true; enabled = false; ++generation;
|
||||
owner = nullptr; retiring = true;
|
||||
if (process.state() == QProcess::NotRunning) { deleteLater(); return; }
|
||||
stopAt = call_video_playout_now_us() + 1500000; watchdog.start(); send(packet(Quit, generation));
|
||||
stop();
|
||||
}
|
||||
void fail(const QString& error) {
|
||||
if (failureReported) return;
|
||||
@@ -151,11 +182,13 @@ struct VideoCapture::Session : QObject {
|
||||
if (in.status() != QDataStream::Ok || count > 128) return false;
|
||||
VideoCaptureDevices devices;
|
||||
for (quint32 i = 0; i < count; ++i) {
|
||||
VideoCaptureDevice device; quint32 modes = 0; in >> device.name >> device.id >> modes;
|
||||
if (in.status() != QDataStream::Ok || modes > 256) return false;
|
||||
VideoCaptureDevice device; quint32 modes = 0; quint8 kind = 0;
|
||||
in >> device.name >> device.id >> modes >> kind >> device.size;
|
||||
if (in.status() != QDataStream::Ok || modes > 256 || kind > quint8(VideoCaptureKind::Window)) return false;
|
||||
device.kind = VideoCaptureKind(kind);
|
||||
for (quint32 j = 0; j < modes; ++j) {
|
||||
qint32 w = 0, h = 0, rate = 0; in >> w >> h >> rate;
|
||||
if (!validSize(w, h) || rate < 1 || rate > 60) return false;
|
||||
if (!(usage == Usage::Stream ? validStreamSize(w, h) : validSize(w, h)) || rate < 1 || rate > 60) return false;
|
||||
device.modes.append({QSize(w, h), rate});
|
||||
}
|
||||
devices.append(device);
|
||||
@@ -163,14 +196,21 @@ struct VideoCapture::Session : QObject {
|
||||
if (owner) emit owner->devicesChanged(devices);
|
||||
} else if (type == Stage) {
|
||||
QString next; in >> next;
|
||||
phase = next; progressAt = call_video_playout_now_us();
|
||||
if (revision == generation && next == "idle") stopAt = 0;
|
||||
const qint64 now = call_video_playout_now_us();
|
||||
if (next != phase) stageAt = now;
|
||||
phase = next; progressAt = now;
|
||||
if (stopping) DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "capture closing pid=%lld generation=%llu stage=%s elapsed_ms=%lld",
|
||||
(long long)process.processId(), (unsigned long long)generation, qPrintable(phase), (long long)((now - stopStarted) / 1000));
|
||||
if (!stopping && revision == generation && next == "idle") stopAt = 0;
|
||||
} else if (type == Heartbeat) {
|
||||
progressAt = call_video_playout_now_us();
|
||||
} else if (type == Error) {
|
||||
QString error; in >> error;
|
||||
if (revision == generation) fail(error);
|
||||
} else if (type == Frame) {
|
||||
qint64 pts = -1, correction = 0; qint32 w = 0, h = 0; in >> pts >> correction >> w >> h;
|
||||
if (in.status() != QDataStream::Ok || pts < 0 || correction < 0 || !validSize(w, h) || !pixels) return false;
|
||||
if (in.status() != QDataStream::Ok || pts < 0 || correction < 0
|
||||
|| !(usage == Usage::Stream ? validStreamSize(w, h) : validSize(w, h)) || !pixels) return false;
|
||||
const qint64 now = call_video_playout_now_us();
|
||||
const bool current = owner && enabled && revision == generation && !failureReported;
|
||||
if (current && pts <= now && now - pts <= VideoCaptureMaxFrameAgeUs) {
|
||||
@@ -191,8 +231,8 @@ struct VideoCapture::Session : QObject {
|
||||
}
|
||||
};
|
||||
|
||||
VideoCapture::VideoCapture(QObject* parent) : QObject(parent), m(new Session(this)) {}
|
||||
VideoCapture::VideoCapture(QObject* parent, Usage usage) : QObject(parent), m(new Session(this, usage)) {}
|
||||
VideoCapture::~VideoCapture() { m->shutdown(); }
|
||||
void VideoCapture::configure(bool enabled, const QByteArray& device, int width, int height, int fps) {
|
||||
m->configure(enabled, device, width, height, fps);
|
||||
}
|
||||
void VideoCapture::configure(const VideoCaptureConfig& config) { m->configure(config); }
|
||||
void VideoCapture::stop() { m->stop(); }
|
||||
bool VideoCapture::closing() const { return m->stopping; }
|
||||
@@ -12,11 +12,12 @@ class VideoCaptureSource : public QObject {
|
||||
Q_OBJECT
|
||||
public:
|
||||
/* Возвращает текущий полный список с аппаратными размерами/FPS передачи, допустимыми для звонка. */
|
||||
virtual VideoCaptureDevices devices() = 0;
|
||||
virtual VideoCaptureDevices devices(bool stream) = 0;
|
||||
/* Запрашивает точный device/размер и предел FPS выдачи; worker вызывает stop перед каждым start.
|
||||
* true — запуск принят, готовность наступает с frame; отказ/асинхронная ошибка сообщаются также сигналом error.
|
||||
* Сам return false не создаёт IPC-ошибку. Qt backend выбирает подходящий формат без масштабирования кадров. */
|
||||
virtual bool start(const QByteArray& device, int width, int height, int fps) = 0;
|
||||
virtual bool start(const VideoCaptureConfig& config) = 0;
|
||||
virtual bool prepareImage(QImage&) { return true; } // Наложение курсора до IPC; только worker event loop.
|
||||
/* Освобождает захват; должен допускать вызов без успешного start и повтор при уничтожении worker. */
|
||||
virtual void stop() = 0;
|
||||
signals:
|
||||
@@ -26,9 +27,10 @@ signals:
|
||||
void frame(const QVideoFrame& frame);
|
||||
/* Текст ошибки, передаётся GUI с поколением активной конфигурации. */
|
||||
void error(const QString& message);
|
||||
void stage(const QString& name); // Перед аппаратным вызовом; supervisor сохраняет последний этап при зависании.
|
||||
};
|
||||
/* Забирает ненулевой source, подключается к endpoint GUI и выполняет QCoreApplication::exec.
|
||||
* Qt application уже должен существовать, event loop ещё не запущен; возвращает его exit code.
|
||||
* При выходе останавливает/уничтожает backend; предназначен только для отдельного процесса. */
|
||||
int video_capture_worker_run(const QString& endpoint, std::unique_ptr<VideoCaptureSource> source);
|
||||
int video_capture_worker_run(const QString& endpoint, std::unique_ptr<VideoCaptureSource> source, bool stream = false);
|
||||
#endif
|
||||
@@ -21,70 +21,197 @@
|
||||
#include <QMediaDevices>
|
||||
#include <QMediaCaptureSession>
|
||||
#include <QVideoSink>
|
||||
#include <QScreenCapture>
|
||||
#include <QWindowCapture>
|
||||
#include <QScreen>
|
||||
#include <QPixmap>
|
||||
#include <QJsonArray>
|
||||
#include <QJsonDocument>
|
||||
#include <QHash>
|
||||
#ifdef UTUN_STREAM_PORTAL
|
||||
#include "stream_portal_capture.h"
|
||||
#endif
|
||||
#if defined(UTUN_STREAM_X11_CURSOR) || defined(UTUN_STREAM_WIN_CURSOR)
|
||||
#include "stream_screen_cursor.h"
|
||||
#endif
|
||||
|
||||
using namespace VideoCaptureProtocol;
|
||||
|
||||
/* Все Qt/native объекты создаются и закрываются в главном потоке дочернего процесса. */
|
||||
class QtVideoCaptureSource : public VideoCaptureSource {
|
||||
std::unique_ptr<QCamera> camera;
|
||||
std::unique_ptr<QScreenCapture> screen;
|
||||
std::unique_ptr<QWindowCapture> window;
|
||||
std::unique_ptr<QMediaCaptureSession> session;
|
||||
std::unique_ptr<QVideoSink> sink;
|
||||
QMediaDevices devicesMonitor;
|
||||
QTimer grabTimer;
|
||||
QPointer<QScreen> selectedScreen;
|
||||
QHash<QByteArray, QCapturableWindow> windows;
|
||||
quint64 nextWindow = 0;
|
||||
CallCameraRate rate;
|
||||
#ifdef UTUN_STREAM_PORTAL
|
||||
std::unique_ptr<StreamPortalCapture> portal;
|
||||
#endif
|
||||
#if defined(UTUN_STREAM_X11_CURSOR) || defined(UTUN_STREAM_WIN_CURSOR)
|
||||
std::unique_ptr<StreamScreenCursor> cursor;
|
||||
#endif
|
||||
static QByteArray screenId(QScreen* selected) {
|
||||
int duplicate = 0;
|
||||
for (auto* item : QGuiApplication::screens()) {
|
||||
if (item == selected) break;
|
||||
if (item->name() == selected->name() && item->manufacturer() == selected->manufacturer()
|
||||
&& item->model() == selected->model() && item->serialNumber() == selected->serialNumber()) ++duplicate;
|
||||
}
|
||||
return QJsonDocument(QJsonArray{selected->name(), selected->manufacturer(), selected->model(),
|
||||
selected->serialNumber(), duplicate}).toJson(QJsonDocument::Compact);
|
||||
}
|
||||
void fail(const QString& message) {
|
||||
DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "capture backend: %s", qUtf8Printable(message)); emit error(message);
|
||||
}
|
||||
public:
|
||||
QtVideoCaptureSource() {
|
||||
connect(&devicesMonitor, &QMediaDevices::videoInputsChanged, this, &VideoCaptureSource::devicesChanged);
|
||||
connect(qGuiApp, &QGuiApplication::screenAdded, this, &VideoCaptureSource::devicesChanged);
|
||||
connect(qGuiApp, &QGuiApplication::screenRemoved, this, [this](QScreen* removed) {
|
||||
if (selectedScreen == removed) fail(QStringLiteral("Selected screen was disconnected"));
|
||||
emit devicesChanged();
|
||||
});
|
||||
}
|
||||
~QtVideoCaptureSource() override { stop(); }
|
||||
VideoCaptureDevices devices() override {
|
||||
VideoCaptureDevices devices(bool stream) override {
|
||||
VideoCaptureDevices result;
|
||||
for (const auto& device : QMediaDevices::videoInputs()) {
|
||||
VideoCaptureDevice entry{device.description(), device.id(), {}};
|
||||
for (const auto& format : device.videoFormats()) {
|
||||
const QSize size = format.resolution();
|
||||
DEBUG_DEBUG(DEBUG_CATEGORY_CALL, "camera capability device=%s size=%dx%d pixel=%d fps=%.3f..%.3f",
|
||||
device.id().constData(), size.width(), size.height(), format.pixelFormat(), format.minFrameRate(), format.maxFrameRate());
|
||||
if (validSize(size.width(), size.height()))
|
||||
if (stream ? validStreamSize(size.width(), size.height()) : validSize(size.width(), size.height()))
|
||||
stream_video_add_camera_modes(entry.modes, size, format.minFrameRate(), format.maxFrameRate());
|
||||
}
|
||||
DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera modes device=%s count=%lld limit=1920x1080 output_fps<=60",
|
||||
device.id().constData(), (long long)entry.modes.size());
|
||||
DEBUG_DEBUG(DEBUG_CATEGORY_VIDEO, "capture camera enumerated device=%s modes=%lld stream=%d",
|
||||
device.id().constData(), (long long)entry.modes.size(), stream);
|
||||
result.append(entry);
|
||||
}
|
||||
if (!stream) return result;
|
||||
if (qEnvironmentVariableIsSet("WAYLAND_DISPLAY") || qEnvironmentVariable("XDG_SESSION_TYPE") == "wayland") {
|
||||
result.append({QStringLiteral("Screen (system selection)"), "portal", {}, VideoCaptureKind::Screen, {}});
|
||||
} else {
|
||||
for (auto* item : QGuiApplication::screens())
|
||||
result.append({item->name(), screenId(item), {}, VideoCaptureKind::Screen, item->size() * item->devicePixelRatio()});
|
||||
}
|
||||
QHash<QByteArray, QCapturableWindow> current;
|
||||
for (const auto& item : QWindowCapture::capturableWindows()) {
|
||||
QByteArray id;
|
||||
for (auto it = windows.cbegin(); it != windows.cend(); ++it) if (it.value() == item) { id = it.key(); break; }
|
||||
if (id.isEmpty()) id = "window-" + QByteArray::number(++nextWindow);
|
||||
current.insert(id, item);
|
||||
result.append({item.description(), id, {}, VideoCaptureKind::Window, {}});
|
||||
}
|
||||
windows = std::move(current);
|
||||
return result;
|
||||
}
|
||||
bool start(const QByteArray& device, int width, int height, int fps) override {
|
||||
QCameraDevice selected;
|
||||
for (const auto& item : QMediaDevices::videoInputs()) if (item.id() == device) selected = item;
|
||||
if (selected.isNull()) { emit error(QStringLiteral("Camera is unavailable")); return false; }
|
||||
bool start(const VideoCaptureConfig& config) override {
|
||||
emit stage(QStringLiteral("session.create"));
|
||||
session = std::make_unique<QMediaCaptureSession>(); sink = std::make_unique<QVideoSink>();
|
||||
camera = std::make_unique<QCamera>(selected);
|
||||
session->setCamera(camera.get()); session->setVideoSink(sink.get());
|
||||
QCameraFormat best;
|
||||
for (const auto& format : selected.videoFormats()) {
|
||||
if (!stream_video_camera_supports(format.resolution(), format.minFrameRate(), format.maxFrameRate(),
|
||||
{QSize(width, height), fps})) continue;
|
||||
if (best.isNull() || format.maxFrameRate() < best.maxFrameRate()) best = format;
|
||||
}
|
||||
if (best.isNull()) {
|
||||
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera mode unavailable device=%s size=%dx%d output_fps=%d", device.constData(), width, height, fps);
|
||||
emit error(QStringLiteral("Selected camera mode is unavailable")); return false;
|
||||
}
|
||||
camera->setCameraFormat(best);
|
||||
const auto format = camera->cameraFormat();
|
||||
DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera selected device=%s size=%dx%d pixel=%d capture_fps=%.3f..%.3f output_fps=%d",
|
||||
device.constData(), format.resolution().width(), format.resolution().height(), format.pixelFormat(),
|
||||
format.minFrameRate(), format.maxFrameRate(), fps);
|
||||
connect(camera.get(), &QCamera::activeChanged, this, [this, device](bool active) {
|
||||
if (active) rate.activate(device); else rate.restore();
|
||||
});
|
||||
connect(camera.get(), &QCamera::errorOccurred, this, [this](QCamera::Error, const QString& message) { emit error(message); });
|
||||
session->setVideoSink(sink.get());
|
||||
connect(sink.get(), &QVideoSink::videoFrameChanged, this, &VideoCaptureSource::frame, Qt::DirectConnection);
|
||||
camera->start(); return true;
|
||||
if (config.kind == VideoCaptureKind::Camera) {
|
||||
QCameraDevice selected;
|
||||
for (const auto& item : QMediaDevices::videoInputs()) if (item.id() == config.device) selected = item;
|
||||
if (selected.isNull()) { fail(QStringLiteral("Camera is unavailable")); return false; }
|
||||
camera = std::make_unique<QCamera>(selected);
|
||||
QCameraFormat best;
|
||||
for (const auto& format : selected.videoFormats()) {
|
||||
if (!stream_video_camera_supports(format.resolution(), format.minFrameRate(), format.maxFrameRate(),
|
||||
{QSize(config.width, config.height), config.fps})) continue;
|
||||
if (best.isNull() || format.maxFrameRate() < best.maxFrameRate()) best = format;
|
||||
}
|
||||
if (best.isNull()) { fail(QStringLiteral("Selected camera mode is unavailable")); return false; }
|
||||
camera->setCameraFormat(best); session->setCamera(camera.get());
|
||||
connect(camera.get(), &QCamera::activeChanged, this, [this, id = config.device](bool active) {
|
||||
if (active) rate.activate(id); else rate.restore();
|
||||
});
|
||||
connect(camera.get(), &QCamera::errorOccurred, this, [this](QCamera::Error, const QString& message) { fail(message); });
|
||||
emit stage(QStringLiteral("camera.start")); camera->start();
|
||||
} else if (config.kind == VideoCaptureKind::Screen) {
|
||||
#ifdef UTUN_STREAM_PORTAL
|
||||
if (config.device == "portal") {
|
||||
portal = std::make_unique<StreamPortalCapture>();
|
||||
connect(portal.get(), &StreamPortalCapture::frameReady, sink.get(), &QVideoSink::setVideoFrame);
|
||||
connect(portal.get(), &StreamPortalCapture::errorOccurred, this, &VideoCaptureSource::error);
|
||||
emit stage(QStringLiteral("portal.start")); portal->start(config.cursor); return true;
|
||||
}
|
||||
#endif
|
||||
for (auto* item : QGuiApplication::screens()) if (screenId(item) == config.device) selectedScreen = item;
|
||||
if (!selectedScreen) { fail(QStringLiteral("Selected screen is unavailable")); return false; }
|
||||
#if defined(UTUN_STREAM_X11_CURSOR) || defined(UTUN_STREAM_WIN_CURSOR)
|
||||
if (config.cursor) cursor = std::make_unique<StreamScreenCursor>(selectedScreen);
|
||||
#endif
|
||||
if (QGuiApplication::platformName() == "xcb") {
|
||||
grabTimer.setTimerType(Qt::PreciseTimer);
|
||||
connect(&grabTimer, &QTimer::timeout, this, [this] {
|
||||
QImage image = selectedScreen ? selectedScreen->grabWindow(0).toImage() : QImage();
|
||||
if (image.isNull()) { grabTimer.stop(); fail(QStringLiteral("Screen capture failed")); return; }
|
||||
#if QT_VERSION >= QT_VERSION_CHECK(6, 8, 0)
|
||||
QVideoFrame raw(image);
|
||||
#else
|
||||
image = image.convertToFormat(QImage::Format_RGBA8888);
|
||||
QVideoFrame raw(QVideoFrameFormat(image.size(), QVideoFrameFormat::Format_RGBA8888));
|
||||
if (!raw.map(QVideoFrame::WriteOnly)) { fail(QStringLiteral("Screen frame mapping failed")); return; }
|
||||
for (int row = 0; row < image.height(); ++row)
|
||||
std::memcpy(raw.bits(0) + row * raw.bytesPerLine(0), image.constScanLine(row), size_t(image.width()) * 4);
|
||||
raw.unmap();
|
||||
#endif
|
||||
raw.setStartTime(call_video_playout_now_us()); emit frame(raw);
|
||||
});
|
||||
emit stage(QStringLiteral("screen.start")); grabTimer.start(std::max(1, 1000 / config.fps));
|
||||
} else {
|
||||
screen = std::make_unique<QScreenCapture>(); screen->setScreen(selectedScreen); session->setScreenCapture(screen.get());
|
||||
connect(screen.get(), &QScreenCapture::errorOccurred, this,
|
||||
[this](QScreenCapture::Error, const QString& message) { fail(message); });
|
||||
emit stage(QStringLiteral("screen.start")); screen->start();
|
||||
}
|
||||
} else {
|
||||
if (!windows.contains(config.device) || !windows.value(config.device).isValid()) {
|
||||
fail(QStringLiteral("Selected window is unavailable")); return false;
|
||||
}
|
||||
window = std::make_unique<QWindowCapture>(); window->setWindow(windows.value(config.device));
|
||||
session->setWindowCapture(window.get());
|
||||
connect(window.get(), &QWindowCapture::errorOccurred, this,
|
||||
[this](QWindowCapture::Error, const QString& message) { fail(message); });
|
||||
emit stage(QStringLiteral("window.start")); window->start();
|
||||
}
|
||||
DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "capture started kind=%u source=%s size=%dx%d fps=%d cursor=%d",
|
||||
unsigned(config.kind), config.device.constData(), config.width, config.height, config.fps, config.cursor);
|
||||
return true;
|
||||
}
|
||||
bool prepareImage(QImage& image) override {
|
||||
#if defined(UTUN_STREAM_X11_CURSOR) || defined(UTUN_STREAM_WIN_CURSOR)
|
||||
if (cursor && !cursor->draw(image)) { fail(QStringLiteral("Mouse cursor capture failed")); return false; }
|
||||
#else
|
||||
Q_UNUSED(image);
|
||||
#endif
|
||||
return true;
|
||||
}
|
||||
void stop() override {
|
||||
grabTimer.stop(); disconnect(&grabTimer, nullptr, this, nullptr);
|
||||
if (sink) disconnect(sink.get(), nullptr, this, nullptr);
|
||||
if (camera) { disconnect(camera.get(), nullptr, this, nullptr); camera->stop(); }
|
||||
rate.restore(); session.reset(); camera.reset(); sink.reset();
|
||||
#ifdef UTUN_STREAM_PORTAL
|
||||
if (portal) { emit stage(QStringLiteral("portal.destroy")); portal.reset(); }
|
||||
#endif
|
||||
if (camera) { disconnect(camera.get(), nullptr, this, nullptr); emit stage(QStringLiteral("camera.stop")); camera->stop(); }
|
||||
if (screen) { disconnect(screen.get(), nullptr, this, nullptr); emit stage(QStringLiteral("screen.stop")); screen->stop(); }
|
||||
if (window) { disconnect(window.get(), nullptr, this, nullptr); emit stage(QStringLiteral("window.stop")); window->stop(); }
|
||||
emit stage(QStringLiteral("camera-rate.restore")); rate.restore();
|
||||
emit stage(QStringLiteral("session.destroy")); session.reset();
|
||||
emit stage(QStringLiteral("camera.destroy")); camera.reset();
|
||||
emit stage(QStringLiteral("screen.destroy")); screen.reset();
|
||||
emit stage(QStringLiteral("window.destroy")); window.reset();
|
||||
emit stage(QStringLiteral("sink.destroy")); sink.reset();
|
||||
#if defined(UTUN_STREAM_X11_CURSOR) || defined(UTUN_STREAM_WIN_CURSOR)
|
||||
cursor.reset();
|
||||
#endif
|
||||
selectedScreen = nullptr;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -96,6 +223,7 @@ class VideoCaptureWorker : public QObject {
|
||||
bool posted = false;
|
||||
};
|
||||
QLocalSocket socket;
|
||||
QTimer heartbeat;
|
||||
VideoCaptureBuffer buffer;
|
||||
uchar* pixels = nullptr;
|
||||
Reader reader;
|
||||
@@ -106,6 +234,8 @@ class VideoCaptureWorker : public QObject {
|
||||
CallVideoCadence cadence;
|
||||
quint64 generation = 0, inFlight = 0;
|
||||
bool sending = false;
|
||||
bool stream = false;
|
||||
VideoCaptureConfig config;
|
||||
int width = 640, height = 480, fps = 30;
|
||||
qint64 lastPts = -1, reportAt = 0, convertMax = 0;
|
||||
unsigned frames = 0, paced = 0;
|
||||
@@ -121,11 +251,11 @@ class VideoCaptureWorker : public QObject {
|
||||
void enumerate() {
|
||||
if (socket.state() != QLocalSocket::ConnectedState) return; // Первичный опрос повторит connected.
|
||||
stage(QStringLiteral("enumerate"));
|
||||
const auto devices = source->devices();
|
||||
const auto devices = source->devices(stream);
|
||||
send(packet(Devices, 0, [&](QDataStream& out) {
|
||||
out << quint32(devices.size());
|
||||
for (const auto& device : devices) {
|
||||
out << device.name << device.id << quint32(device.modes.size());
|
||||
out << device.name << device.id << quint32(device.modes.size()) << quint8(device.kind) << device.size;
|
||||
for (const auto& mode : device.modes) out << qint32(mode.size.width()) << qint32(mode.size.height()) << qint32(mode.fps);
|
||||
}
|
||||
}));
|
||||
@@ -159,17 +289,34 @@ class VideoCaptureWorker : public QObject {
|
||||
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera frame conversion failed");
|
||||
send(packet(Error, generation, [](QDataStream& out) { out << QStringLiteral("Camera frame conversion failed"); })); return;
|
||||
}
|
||||
if (image.size() != QSize(width, height)) {
|
||||
if (!source->prepareImage(image)) return;
|
||||
#if QT_VERSION >= QT_VERSION_CHECK(6, 7, 0)
|
||||
if (raw.rotation() != QtVideo::Rotation::None) image = image.transformed(QTransform().rotate(static_cast<int>(raw.rotation())));
|
||||
if (raw.mirrored()) image = image.transformed(QTransform().scale(-1, 1));
|
||||
#else
|
||||
if (raw.rotationAngle()) image = image.transformed(QTransform().rotate(raw.rotationAngle()));
|
||||
#endif
|
||||
if (config.kind == VideoCaptureKind::Camera && image.size() != QSize(width, height)) {
|
||||
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera geometry changed requested=%dx%d actual=%dx%d", width, height, image.width(), image.height());
|
||||
send(packet(Error, generation, [](QDataStream& out) { out << QStringLiteral("Camera changed the selected resolution"); })); return;
|
||||
}
|
||||
if (stream && config.kind != VideoCaptureKind::Camera) {
|
||||
const QSize size = stream_video_screen_size(image.size());
|
||||
if (size.isEmpty()) { send(packet(Error, generation, [](QDataStream& out) { out << QStringLiteral("Invalid screen size"); })); return; }
|
||||
if (image.size() != size) image = image.scaled(size, Qt::IgnoreAspectRatio, Qt::SmoothTransformation);
|
||||
}
|
||||
const int outputWidth = image.width(), outputHeight = image.height();
|
||||
if (!(stream ? validStreamSize(outputWidth, outputHeight) : validSize(outputWidth, outputHeight))) {
|
||||
send(packet(Error, generation, [](QDataStream& out) { out << QStringLiteral("Invalid converted frame size"); })); return;
|
||||
}
|
||||
image = image.convertToFormat(QImage::Format_RGB32);
|
||||
const qint64 converted = call_video_playout_now_us();
|
||||
if (converted - pts <= VideoCaptureMaxFrameAgeUs) {
|
||||
for (int row = 0; row < height; ++row) std::memcpy(pixels + size_t(row) * width * 4, image.constScanLine(row), size_t(width) * 4);
|
||||
for (int row = 0; row < outputHeight; ++row)
|
||||
std::memcpy(pixels + size_t(row) * outputWidth * 4, image.constScanLine(row), size_t(outputWidth) * 4);
|
||||
std::atomic_thread_fence(std::memory_order_release);
|
||||
auto bytes = packet(Frame, generation, [&](QDataStream& out) {
|
||||
out << pts << qint64(clock.correction) << qint32(image.width()) << qint32(image.height());
|
||||
out << pts << qint64(clock.correction) << qint32(outputWidth) << qint32(outputHeight);
|
||||
});
|
||||
sending = true; inFlight = generation; lastPts = pts; send(bytes); ++frames;
|
||||
} else DEBUG_WARN(DEBUG_CATEGORY_CALL, "camera conversion exceeded frame age budget age_us=%lld", (long long)(converted - pts));
|
||||
@@ -183,19 +330,24 @@ class VideoCaptureWorker : public QObject {
|
||||
}
|
||||
bool receive(quint8 type, quint64 revision, QDataStream& in) {
|
||||
if (type == Buffer) {
|
||||
QString name; in >> name;
|
||||
QString name; quint32 capacity = 0; in >> name >> capacity;
|
||||
if (pixels) return false;
|
||||
if (!buffer.attach(name)) {
|
||||
if (capacity != (stream ? MaxFrameBytes : FrameBytes) || !buffer.attach(name, capacity)) {
|
||||
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera frame mapping failed: %s", qPrintable(buffer.errorString())); return false;
|
||||
}
|
||||
pixels = buffer.data();
|
||||
} else if (type == Quit) { stop(); QCoreApplication::quit(); }
|
||||
} else if (type == Quit) { generation = revision; stop(); stage(QStringLiteral("process.exit")); QCoreApplication::quit(); }
|
||||
else if (type == Ack) {
|
||||
if (sending && revision == inFlight) { sending = false; if (pending) convert(pending, generation); }
|
||||
} else if (type == Configure) {
|
||||
bool enabled = false; QByteArray device; qint32 w = 0, h = 0, rate = 0; in >> enabled >> device >> w >> h >> rate;
|
||||
if (in.status() != QDataStream::Ok || !validSize(w, h) || rate < 1 || rate > 60 || !pixels) return false;
|
||||
stop(); generation = revision; width = w; height = h; fps = rate; clock = {}; cadence = CallVideoCadence(rate); lastPts = -1;
|
||||
bool enabled = false, cursor = false; QByteArray device; qint32 w = 0, h = 0, rate = 0; quint8 kind = 0;
|
||||
in >> enabled >> device >> w >> h >> rate >> kind >> cursor;
|
||||
if (in.status() != QDataStream::Ok || !(stream ? validStreamSize(w, h) : validSize(w, h))
|
||||
|| rate < 1 || rate > 60 || !pixels || kind > quint8(VideoCaptureKind::Window)
|
||||
|| (!stream && kind != quint8(VideoCaptureKind::Camera))) return false;
|
||||
generation = revision; stop(); width = w; height = h; fps = rate;
|
||||
config = {enabled, device, w, h, rate, VideoCaptureKind(kind), cursor};
|
||||
clock = {}; cadence = CallVideoCadence(rate); lastPts = -1;
|
||||
if (!enabled) { stage(QStringLiteral("idle")); return true; }
|
||||
stage(QStringLiteral("start"));
|
||||
pending = std::make_shared<Pending>();
|
||||
@@ -214,12 +366,16 @@ class VideoCaptureWorker : public QObject {
|
||||
post = !slot->posted; slot->posted = true; }
|
||||
if (post) QMetaObject::invokeMethod(this, [this, slot, revision] { convert(slot, revision); }, Qt::QueuedConnection);
|
||||
}, Qt::DirectConnection);
|
||||
if (source->start(device, width, height, fps)) stage(QStringLiteral("running"));
|
||||
if (source->start(config)) stage(QStringLiteral("running"));
|
||||
} else return false;
|
||||
return true;
|
||||
}
|
||||
public:
|
||||
explicit VideoCaptureWorker(std::unique_ptr<VideoCaptureSource> backend) : source(std::move(backend)) {
|
||||
explicit VideoCaptureWorker(std::unique_ptr<VideoCaptureSource> backend, bool useStream) : source(std::move(backend)), stream(useStream) {
|
||||
connect(source.get(), &VideoCaptureSource::stage, this, &VideoCaptureWorker::stage);
|
||||
heartbeat.setInterval(1000);
|
||||
connect(&heartbeat, &QTimer::timeout, this, [this] { if (socket.state() == QLocalSocket::ConnectedState) send(packet(Heartbeat, generation)); });
|
||||
heartbeat.start();
|
||||
connect(&socket, &QLocalSocket::connected, this, &VideoCaptureWorker::enumerate);
|
||||
connect(source.get(), &VideoCaptureSource::devicesChanged, this, &VideoCaptureWorker::enumerate);
|
||||
connect(&socket, &QLocalSocket::readyRead, this, [this] {
|
||||
@@ -232,12 +388,17 @@ public:
|
||||
DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera worker IPC: %s", qPrintable(socket.errorString())); QCoreApplication::exit(2);
|
||||
});
|
||||
}
|
||||
~VideoCaptureWorker() override { disconnect(captureConnection); source->stop(); }
|
||||
~VideoCaptureWorker() override {
|
||||
disconnect(captureConnection); source->stop();
|
||||
stage(QStringLiteral("source.destroy")); source.reset();
|
||||
stage(QStringLiteral("frame.release")); pending.reset();
|
||||
stage(QStringLiteral("thread-resources.release"));
|
||||
}
|
||||
void connectTo(const QString& endpoint) { socket.connectToServer(endpoint); }
|
||||
};
|
||||
|
||||
int video_capture_worker_run(const QString& endpoint, std::unique_ptr<VideoCaptureSource> source) {
|
||||
VideoCaptureWorker worker(std::move(source)); worker.connectTo(endpoint); return QCoreApplication::exec();
|
||||
int video_capture_worker_run(const QString& endpoint, std::unique_ptr<VideoCaptureSource> source, bool stream) {
|
||||
VideoCaptureWorker worker(std::move(source), stream); worker.connectTo(endpoint); return QCoreApplication::exec();
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -249,8 +410,10 @@ int video_capture_worker_main(int argc, char** argv) {
|
||||
const UINT codePage = GetACP();
|
||||
DEBUG_INFO(DEBUG_CATEGORY_CALL, "camera worker Windows code_page=%u Qt=%s camera_errors=Unicode", codePage, qVersion());
|
||||
#endif
|
||||
if (argc != 3) { DEBUG_ERROR(DEBUG_CATEGORY_CALL, "camera worker endpoint missing"); return 2; }
|
||||
return video_capture_worker_run(QString::fromLocal8Bit(argv[2]), std::make_unique<QtVideoCaptureSource>());
|
||||
if (argc != 4 || (std::strcmp(argv[3], "stream") && std::strcmp(argv[3], "call"))) {
|
||||
DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "capture worker endpoint/usage missing"); return 2;
|
||||
}
|
||||
return video_capture_worker_run(QString::fromLocal8Bit(argv[2]), std::make_unique<QtVideoCaptureSource>(), !std::strcmp(argv[3], "stream"));
|
||||
#else
|
||||
Q_UNUSED(argc); Q_UNUSED(argv); return 2;
|
||||
#endif
|
||||
|
||||
Reference in new issue
Block a user