Keep Qt video frame conversion off the publisher worker
This commit is contained in:
evgeny committed
1 parent
c27ee36cfb
commit
97e20d2f1c
2 files changed
+58
-39
No files matched your search
@@ -144,8 +144,10 @@ stats timer и stream handle освобождаются до контекста
|
||||
|
||||
Desktop `MainWindow` владеет `StreamPublishEngine`, а окно управления лишь отображает
|
||||
его состояние. Закрытие окна сохраняет эфир, новый запуск запрещён до конца FIN drain.
|
||||
Qt capture callback хранит только последний кадр, video/audio workers выполняют encode/DSP
|
||||
вне GUI и hardware callbacks. Отдельные поколения источника и encoder позволяют менять
|
||||
Qt capture callback конвертирует кадр в `QImage` до передачи worker и сохраняет последний
|
||||
`QVideoFrame` отдельно для preview. Конвертация и освобождение media frame не выполняются
|
||||
в короткоживущем video worker, поскольку Qt multimedia TLS может задержать его exit на Windows.
|
||||
Video/audio workers выполняют encode/DSP вне GUI и hardware callbacks. Отдельные поколения источника и encoder позволяют менять
|
||||
видеокодек/bitrate/rotation без повторного выбора экрана в портале. Ошибки Qt capture
|
||||
обрабатываются через queued callback, чтобы не уничтожать backend из собственного callback.
|
||||
Stop запрещает producers, закрывает Qt capture и подписку на аудиозахват. Отдельный join worker
|
||||
|
||||
@@ -102,9 +102,10 @@ struct StreamPublishEngine::Impl {
|
||||
std::unique_ptr<StreamScreenCursor> cursor;
|
||||
#endif
|
||||
std::unique_ptr<QWindowCapture> window;
|
||||
QVideoFrame frame;
|
||||
QImage frame;
|
||||
QVideoFrame previewFrame;
|
||||
QVideoFrame presentedPreview;
|
||||
int64_t lastImageCaptureAt = 0;
|
||||
#endif
|
||||
};
|
||||
|
||||
@@ -231,30 +232,58 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) {
|
||||
#ifdef HAVE_STREAM_CAPTURE
|
||||
uint64_t generation;
|
||||
{ std::lock_guard<std::mutex> guard(m->lock); generation = ++m->sourceGeneration;
|
||||
m->backendOrigin = m->backendAnchor = -1; m->capturedFrames = 0; }
|
||||
m->backendOrigin = m->backendAnchor = -1; m->capturedFrames = 0; m->lastImageCaptureAt = 0; }
|
||||
m->session = std::make_unique<QMediaCaptureSession>(); m->sink = std::make_unique<QVideoSink>(this);
|
||||
m->session->setVideoSink(m->sink.get());
|
||||
connect(m->sink.get(), &QVideoSink::videoFrameChanged, this, [this, generation](const QVideoFrame& frame) {
|
||||
if (!frame.isValid()) { DEBUG_TRACE(DEBUG_CATEGORY_VIDEO, "desktop capture sink invalid frame generation=%llu", (unsigned long long)generation); return; }
|
||||
if (!m->running) return;
|
||||
std::lock_guard<std::mutex> guard(m->lock);
|
||||
if (generation != m->sourceGeneration) return;
|
||||
if (!m->capturedFrames++) DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop capture first raw frame kind=%d size=%dx%d pixel=%d PTS=%lld",
|
||||
m->wanted.source.kind, frame.width(), frame.height(), frame.pixelFormat(), (long long)frame.startTime());
|
||||
int64_t now = monotonicUs(), capture = frame.startTime();
|
||||
if (capture >= 0) {
|
||||
if (m->backendOrigin < 0) {
|
||||
m->backendOrigin = capture; m->backendAnchor = now;
|
||||
DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop capture timing generation=%llu source_PTS=%lld anchor=%lld calibration=arrival-estimate",
|
||||
(unsigned long long)generation, (long long)capture, (long long)now);
|
||||
int64_t callbackAt = monotonicUs(), capture = frame.startTime(), framePts = 0;
|
||||
bool convert = false;
|
||||
{
|
||||
std::lock_guard<std::mutex> guard(m->lock);
|
||||
if (generation != m->sourceGeneration) return;
|
||||
if (!m->capturedFrames++) DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop capture first raw frame kind=%d size=%dx%d pixel=%d PTS=%lld",
|
||||
m->wanted.source.kind, frame.width(), frame.height(), frame.pixelFormat(), (long long)capture);
|
||||
if (capture >= 0) {
|
||||
if (m->backendOrigin < 0) {
|
||||
m->backendOrigin = capture; m->backendAnchor = callbackAt;
|
||||
DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop capture timing generation=%llu source_PTS=%lld anchor=%lld calibration=arrival-estimate",
|
||||
(unsigned long long)generation, (long long)capture, (long long)callbackAt);
|
||||
}
|
||||
callbackAt = m->backendAnchor + capture - m->backendOrigin;
|
||||
} else if (m->backendAnchor < 0) {
|
||||
m->backendAnchor = callbackAt;
|
||||
DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "desktop capture has no source PTS; using monotonic callback estimate");
|
||||
}
|
||||
now = m->backendAnchor + capture - m->backendOrigin;
|
||||
} else if (m->backendAnchor < 0) {
|
||||
m->backendAnchor = now;
|
||||
DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "desktop capture has no source PTS; using monotonic callback estimate");
|
||||
int fps = std::clamp(m->wanted.video.fps, 1, 60);
|
||||
convert = !m->lastImageCaptureAt || callbackAt - m->lastImageCaptureAt >= 1000000 / fps;
|
||||
m->previewFrame = frame;
|
||||
if (!convert) { m->wake.notify_all(); return; }
|
||||
m->lastImageCaptureAt = callbackAt;
|
||||
if (m->framePts < 0 && m->wanted.source.id == "portal") m->started = monotonicUs();
|
||||
framePts = std::max<int64_t>(1, callbackAt - m->origin);
|
||||
}
|
||||
/* QVideoFrame conversion stays on Qt's persistent multimedia callback thread.
|
||||
* Qt worker-thread storage was preventing the short-lived publisher thread from exiting on Windows. */
|
||||
QImage image = frame.toImage();
|
||||
if (image.isNull()) {
|
||||
DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop captured frame conversion failed generation=%llu format=%d size=%dx%d",
|
||||
(unsigned long long)generation, frame.pixelFormat(), frame.width(), frame.height());
|
||||
QMetaObject::invokeMethod(this, [this, generation] { sourceFailed(generation, text("Video frame conversion failed")); });
|
||||
return;
|
||||
}
|
||||
#if QT_VERSION >= QT_VERSION_CHECK(6, 7, 0)
|
||||
if (frame.rotation() != QtVideo::Rotation::None) image = image.transformed(QTransform().rotate(static_cast<int>(frame.rotation())));
|
||||
if (frame.mirrored()) image = image.transformed(QTransform().scale(-1, 1));
|
||||
#else
|
||||
if (frame.rotationAngle()) image = image.transformed(QTransform().rotate(frame.rotationAngle()));
|
||||
#endif
|
||||
{
|
||||
std::lock_guard<std::mutex> guard(m->lock);
|
||||
if (!m->running || generation != m->sourceGeneration) return;
|
||||
m->frame = std::move(image); m->framePts = framePts; m->frameGeneration = m->generation;
|
||||
}
|
||||
if (m->framePts < 0 && m->wanted.source.id == "portal") m->started = monotonicUs();
|
||||
m->frame = m->previewFrame = frame; m->framePts = std::max<int64_t>(1, now - m->origin); m->frameGeneration = m->generation;
|
||||
m->wake.notify_all();
|
||||
}, Qt::DirectConnection);
|
||||
auto failed = [this, generation](const QString& error) {
|
||||
@@ -462,7 +491,7 @@ bool StreamPublishEngine::reconfigure(const StreamPublishConfig& config) {
|
||||
&& (config.video.width != previous.video.width || config.video.height != previous.video.height || config.video.fps != previous.video.fps));
|
||||
{ std::lock_guard<std::mutex> guard(m->lock); m->wanted = config; m->generation++; m->started = monotonicUs(); m->softwareRequest = 0;
|
||||
#ifdef HAVE_STREAM_CAPTURE
|
||||
if (!sourceChange) { m->frame = m->previewFrame; m->frameGeneration = m->generation; }
|
||||
if (!sourceChange) m->frameGeneration = m->generation;
|
||||
#endif
|
||||
}
|
||||
if (sourceChange && !openSource(config)) { sourceFailed(m->generation, text("Could not open the selected video source")); return false; }
|
||||
@@ -654,17 +683,17 @@ void StreamPublishEngine::videoLoop(StreamPublishConfig initial) {
|
||||
int64_t lastPts = -1, lastKey = -1000000, next = 0, report = 0; unsigned count = 0, skipped = 0;
|
||||
while (m->running) {
|
||||
m->videoProgress("waiting-frame");
|
||||
StreamPublishConfig config; uint64_t generation; QVideoFrame frame; int64_t capture;
|
||||
StreamPublishConfig config; uint64_t generation; QImage frame; int64_t capture;
|
||||
{ std::unique_lock<std::mutex> lock(m->lock); config = m->wanted; generation = m->generation;
|
||||
if (config.audio && !m->lastAudioAt.load(std::memory_order_acquire)) {
|
||||
m->wake.wait_for(lock, std::chrono::milliseconds(5)); continue;
|
||||
}
|
||||
int64_t now = monotonicUs();
|
||||
if (next > now) { m->wake.wait_for(lock, std::chrono::microseconds(next - now)); continue; }
|
||||
if (m->frameGeneration == generation) { frame = m->frame; capture = m->framePts; m->frame = {}; }
|
||||
if (m->frameGeneration == generation) { frame = std::move(m->frame); capture = m->framePts; }
|
||||
else capture = -1; }
|
||||
// Размер кадра точнее QScreen при HiDPI и единственный доступный после выбора Wayland portal.
|
||||
if (config.source.kind == StreamVideoSource::Screen && frame.isValid() && config.source.size != frame.size()) {
|
||||
if (config.source.kind == StreamVideoSource::Screen && !frame.isNull() && config.source.size != frame.size()) {
|
||||
QSize size = stream_video_screen_size(frame.size(), config.nativeSize ? 4096 : std::min(config.video.width, config.video.height));
|
||||
if (size.isEmpty()) {
|
||||
DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop screen invalid captured size=%dx%d", frame.width(), frame.height());
|
||||
@@ -685,7 +714,7 @@ void StreamPublishEngine::videoLoop(StreamPublishConfig initial) {
|
||||
m->wake.wait_for(lock, std::chrono::milliseconds(100), [this, generation] { return !m->running || m->generation != generation; });
|
||||
continue;
|
||||
}
|
||||
if (config.source.kind == StreamVideoSource::Screen && config.source.size.isEmpty() && !frame.isValid()) {
|
||||
if (config.source.kind == StreamVideoSource::Screen && config.source.size.isEmpty() && frame.isNull()) {
|
||||
std::unique_lock<std::mutex> lock(m->lock); m->wake.wait_for(lock, std::chrono::milliseconds(5)); continue;
|
||||
}
|
||||
if (generation != current) {
|
||||
@@ -748,22 +777,10 @@ void StreamPublishEngine::videoLoop(StreamPublishConfig initial) {
|
||||
}
|
||||
}
|
||||
bool screen = config.source.kind != StreamVideoSource::Camera;
|
||||
if (frame.isValid()) {
|
||||
if (!frame.isNull()) {
|
||||
if (!screen && (capture <= lastPts || monotonicUs() - m->origin - capture > 250000)) { skipped++; continue; }
|
||||
m->videoProgress("convert-frame-to-image");
|
||||
lastImage = frame.toImage();
|
||||
lastImage = std::move(frame);
|
||||
m->convertedFrames++;
|
||||
if (lastImage.isNull()) {
|
||||
DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop captured frame conversion failed format=%d", frame.pixelFormat());
|
||||
QMetaObject::invokeMethod(this, [this, generation] { sourceFailed(generation, text("Video frame conversion failed")); });
|
||||
continue;
|
||||
}
|
||||
#if QT_VERSION >= QT_VERSION_CHECK(6, 7, 0)
|
||||
if (frame.rotation() != QtVideo::Rotation::None) lastImage = lastImage.transformed(QTransform().rotate(static_cast<int>(frame.rotation())));
|
||||
if (frame.mirrored()) lastImage = lastImage.transformed(QTransform().scale(-1, 1));
|
||||
#else
|
||||
if (frame.rotationAngle()) lastImage = lastImage.transformed(QTransform().rotate(frame.rotationAngle()));
|
||||
#endif
|
||||
} else if (!screen || lastImage.isNull()) {
|
||||
std::unique_lock<std::mutex> lock(m->lock); m->wake.wait_for(lock, std::chrono::milliseconds(5)); continue;
|
||||
} else m->repeatedFrames++;
|
||||
|
||||
Reference in new issue
Block a user