jeka
/
utun2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Render raw preview at 30 fps and preserve fallback audio sample timing

master
evgeny 5 days ago
parent
27f3726124
commit
ba17f9908d
3 changed files with 51 additions and 25 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 65
      tools/chatgui/src/stream_publish_engine.cpp
  2. 3
      tools/chatgui/src/stream_publish_engine.h
  3. 8
      tools/chatgui/src/stream_publish_window.cpp

65
tools/chatgui/src/stream_publish_engine.cpp
Unescape View File

@ -44,13 +44,14 @@ struct StreamPublishEngine::Impl {
AudioEventQueue<Audio, 32> audio;
uint64_t id = 0, group = 0, generation = 0, sourceGeneration = 0, confirmedGeneration = 0, frameGeneration = 0, audioSequence = 0;
int64_t origin = 0, started = 0, framePts = -1, backendOrigin = -1, backendAnchor = -1;
int64_t audioAnchor = -1;
uint64_t audioFrames = 0;
int channels = 2;
uint64_t capturedFrames = 0;
int64_t captureReport = 0;
bool probeReady = false, probeDelivered = false, hasGood = false, rollingBack = false;
StreamPublishConfig wanted, good;
QList<int> codecs;
QImage preview;
stream_publisher_status stats{};
#ifdef HAVE_STREAM_CAPTURE
std::unique_ptr<QMediaCaptureSession> session;
@ -60,11 +61,25 @@ struct StreamPublishEngine::Impl {
std::unique_ptr<QWindowCapture> window;
QVideoFrame frame;
QVideoFrame previewFrame;
QVideoFrame presentedPreview;
#endif
};
StreamPublishEngine::StreamPublishEngine(QObject* parent) : QObject(parent), m(std::make_unique<Impl>()) {
m_timer.setInterval(100); connect(&m_timer, &QTimer::timeout, this, &StreamPublishEngine::tick); m_timer.start();
m_timer.setTimerType(Qt::PreciseTimer); m_timer.setInterval(100);
connect(&m_timer, &QTimer::timeout, this, &StreamPublishEngine::tick); m_timer.start();
#ifdef HAVE_STREAM_CAPTURE
m_previewTimer.setTimerType(Qt::PreciseTimer); m_previewTimer.setInterval(33);
connect(&m_previewTimer, &QTimer::timeout, this, [this] {
if (!m->running) return;
QVideoFrame frame;
{ std::lock_guard<std::mutex> guard(m->lock);
if (m->previewFrame == m->presentedPreview) return;
frame = m->presentedPreview = m->previewFrame; }
if (frame.isValid()) emit previewVideoChanged(QVariant::fromValue(frame));
});
m_previewTimer.start();
#endif
try {
m->probeThread = std::thread([this] {
auto codecs = supported() ? StreamVideoEncoder::available() : QList<int>();
@ -78,7 +93,7 @@ StreamPublishEngine::StreamPublishEngine(QObject* parent) : QObject(parent), m(s
connect(sound, &SoundManager::contextRestored, this, [this] { if (m->running && m->wanted.audio) openMicrophone(); });
}
StreamPublishEngine::~StreamPublishEngine() {
stop(); m_timer.stop();
stop(); m_timer.stop(); m_previewTimer.stop();
if (m->probeThread.joinable()) m->probeThread.join();
}
bool StreamPublishEngine::supported() {
@ -89,6 +104,7 @@ bool StreamPublishEngine::supported() {
#endif
}
bool StreamPublishEngine::active() const { return m->id != 0; }
bool StreamPublishEngine::running() const { return m->running.load(); }
uint64_t StreamPublishEngine::group() const { return m->group; }
StreamPublishConfig StreamPublishEngine::config() const { std::lock_guard<std::mutex> guard(m->lock); return m->wanted; }
QList<int> StreamPublishEngine::codecs() const { std::lock_guard<std::mutex> guard(m->lock); return m->codecs; }
@ -115,7 +131,8 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) {
closeSource();
#ifdef HAVE_STREAM_CAPTURE
uint64_t generation;
{ std::lock_guard<std::mutex> guard(m->lock); generation = ++m->sourceGeneration; m->backendOrigin = m->backendAnchor = -1; }
{ std::lock_guard<std::mutex> guard(m->lock); generation = ++m->sourceGeneration;
m->backendOrigin = m->backendAnchor = -1; m->capturedFrames = 0; }
m->session = std::make_unique<QMediaCaptureSession>(); m->sink = std::make_unique<QVideoSink>();
m->session->setVideoSink(m->sink.get());
connect(m->sink.get(), &QVideoSink::videoFrameChanged, this, [this, generation](const QVideoFrame& frame) {
@ -196,12 +213,13 @@ bool StreamPublishEngine::openSource(const StreamPublishConfig& config) {
}
void StreamPublishEngine::closeSource() {
#ifdef HAVE_STREAM_CAPTURE
emit previewVideoChanged(QVariant());
if (m->sink) disconnect(m->sink.get(), nullptr, this, nullptr);
if (m->camera) { disconnect(m->camera.get(), nullptr, this, nullptr); m->camera->stop(); }
if (m->screen) { disconnect(m->screen.get(), nullptr, this, nullptr); m->screen->stop(); }
if (m->window) { disconnect(m->window.get(), nullptr, this, nullptr); m->window->stop(); }
m->session.reset(); m->camera.reset(); m->screen.reset(); m->window.reset(); m->sink.reset();
std::lock_guard<std::mutex> guard(m->lock); m->frame = {}; m->previewFrame = {}; m->framePts = -1;
std::lock_guard<std::mutex> guard(m->lock); m->frame = {}; m->previewFrame = {}; m->presentedPreview = {}; m->framePts = -1;
#endif
}
@ -214,7 +232,7 @@ bool StreamPublishEngine::start(uint64_t group, const StreamPublishConfig& confi
}
{ std::lock_guard<std::mutex> guard(m->lock);
m->wanted = config; m->good = {}; m->hasGood = m->rollingBack = false; m->generation++; m->confirmedGeneration = 0;
m->origin = m->started = monotonicUs(); m->preview = {}; m->stats = {}; }
m->origin = m->started = monotonicUs(); m->stats = {}; }
m->id = QRandomGenerator::global()->generate64(); if (!m->id) m->id = 1;
m->group = group; m->audio.reset(); m->audioSequence = 0; m->muted = false;
stream_media_format format{(uint16_t)config.video.width, (uint16_t)config.video.height, (uint16_t)config.video.rotation,
@ -276,6 +294,7 @@ void StreamPublishEngine::stop() {
bool StreamPublishEngine::openMicrophone() {
m_capture.close();
m->audioAnchor = -1; m->audioFrames = 0;
auto config = this->config(); auto* context = SoundManager::instance()->context();
if (!context) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop stream audio context unavailable"); return false; }
ma_device_id device; bool selected = SoundManager::deviceId(config.microphone, &device);
@ -287,8 +306,10 @@ bool StreamPublishEngine::openMicrophone() {
if (!m_capture.init(context, capture, "stream-capture", m->id, callback)) continue;
m->channels = channels;
if (!m_capture.start()) continue;
if (channels == 1) DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop stream input is mono; stereo transport duplicates the input");
emit audioChannelsChanged(channels); return true;
int native = m_capture.device()->capture.internalChannels;
if (native == 1) DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop stream input is mono; stereo transport duplicates the input");
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop microphone opened requested=%d backend_channels=%d rate=%u", channels, native, m_capture.device()->sampleRate);
emit audioChannelsChanged(native == 1 ? 1 : channels); return true;
}
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "desktop stream cannot open microphone: %s", qPrintable(config.microphone)); return false;
}
@ -296,15 +317,20 @@ bool StreamPublishEngine::openMicrophone() {
/* Hardware callback копирует максимум 20ms за элемент, без codec/DSP/allocations. */
void StreamPublishEngine::captureAudio(const void* input, unsigned frames) {
const auto* pcm = static_cast<const int16_t*>(input);
if (!m->running || !pcm) return;
if (!m->running) return;
int64_t first = m_capture.callbackTimeUs() - m->origin;
if (m_capture.timingMeasured() || m->audioAnchor < 0 || m_capture.captureMissing()) {
m->audioAnchor = first; m->audioFrames = 0;
}
first = m->audioAnchor + (int64_t)m->audioFrames * 1000000 / 48000; m->audioFrames += frames;
if (!pcm) m->audioDropped++;
for (unsigned offset = 0; offset < frames;) {
Impl::Audio record; record.frames = std::min(960u, frames - offset);
record.pts = std::max<int64_t>(0, first + (int64_t)offset * 1000000 / 48000); record.created = monotonicUs();
record.sequence = ++m->audioSequence;
for (unsigned i = 0; i < record.frames; i++) {
record.pcm[2 * i] = pcm[(offset + i) * m->channels];
record.pcm[2 * i + 1] = pcm[(offset + i) * m->channels + (m->channels == 2 ? 1 : 0)];
record.pcm[2 * i] = pcm ? pcm[(offset + i) * m->channels] : 0;
record.pcm[2 * i + 1] = pcm ? pcm[(offset + i) * m->channels + (m->channels == 2 ? 1 : 0)] : 0;
}
if (!m->audio.push(record)) m->audioDropped++;
offset += record.frames;
@ -428,14 +454,17 @@ void StreamPublishEngine::videoLoop() {
if (m->generation == generation) {
m->good = config; m->hasGood = true; m->rollingBack = false; m->confirmedGeneration = generation;
} }
if (first) QMetaObject::invokeMethod(this, [this, generation] {
if (first) {
DEBUG_INFO(DEBUG_CATEGORY_VIDEO, "desktop first KEY committed generation=%llu revision=%u PTS=%llu bytes=%lld",
(unsigned long long)generation, revision, (unsigned long long)encoded.pts, (long long)encoded.data.size());
QMetaObject::invokeMethod(this, [this, generation] {
if (generation != m->generation) return;
emit configurationChanged(); emit stateChanged(text("Live"));
});
});
}
}
} else skipped++;
}
{ std::lock_guard<std::mutex> guard(m->lock); if (m->generation == generation) m->preview = lastImage; }
if (now - report >= 1000000) {
DEBUG_DEBUG(DEBUG_CATEGORY_VIDEO, "desktop capture generation=%llu encoded=%u skipped=%u frame_age=%lldus encoder=%s",
(unsigned long long)generation, count, skipped, (long long)(screen ? 0 : now - m->origin - capture), qPrintable(encoder->name()));
@ -456,14 +485,6 @@ void StreamPublishEngine::tick() {
}
{ std::lock_guard<std::mutex> guard(m->lock); m->stats = status; }
emit telemetryChanged(status);
QImage preview;
{ std::lock_guard<std::mutex> guard(m->lock); preview = m->preview; }
if (!preview.isNull()) emit previewChanged(preview);
#ifdef HAVE_STREAM_CAPTURE
QVideoFrame rawPreview;
{ std::lock_guard<std::mutex> guard(m->lock); rawPreview = m->previewFrame; }
if (rawPreview.isValid()) emit previewVideoChanged(QVariant::fromValue(rawPreview));
#endif
if (status.state == STREAM_PUBLISHER_ENDED || status.state == STREAM_PUBLISHER_FAILED) {
stop(); m->id = m->group = 0;
if (status.state == STREAM_PUBLISHER_FAILED) emit errorOccurred(text("Stream publication failed"));

3
tools/chatgui/src/stream_publish_engine.h
Unescape View File

@ -35,6 +35,7 @@ public:
bool reconfigure(const StreamPublishConfig& config);
void stop();
bool active() const;
bool running() const;
uint64_t group() const;
StreamPublishConfig config() const;
QList<int> codecs() const;
@ -45,7 +46,6 @@ signals:
void codecsChanged();
void stateChanged(const QString& state);
void configurationChanged();
void previewChanged(const QImage& frame);
void previewVideoChanged(const QVariant& frame);
void telemetryChanged(const stream_publisher_status& status);
void errorOccurred(const QString& error);
@ -56,6 +56,7 @@ private:
AudioDevice m_capture;
AudioCaptureLease m_microphone{AudioCaptureLease::Mode::Exclusive};
QTimer m_timer;
QTimer m_previewTimer;
bool openSource(const StreamPublishConfig& config);
void closeSource();
bool openMicrophone();

8
tools/chatgui/src/stream_publish_window.cpp
Unescape View File

@ -30,7 +30,8 @@ StreamPublishWindow::StreamPublishWindow(StreamPublishEngine* engine, uint64_t g
auto* preview = new QVideoWidget(this); m_preview = preview;
preview->setMinimumHeight(160); preview->setAspectRatioMode(Qt::KeepAspectRatio);
connect(engine, &StreamPublishEngine::previewVideoChanged, this, [this, preview](const QVariant& frame) {
if (isVisible()) preview->videoSink()->setVideoFrame(frame.value<QVideoFrame>());
auto video = frame.value<QVideoFrame>();
if (!video.isValid() || isVisible()) preview->videoSink()->setVideoFrame(video);
});
#else
m_preview = new QLabel(text("Video capture requires Qt Multimedia and FFmpeg"), this);
@ -38,12 +39,14 @@ StreamPublishWindow::StreamPublishWindow(StreamPublishEngine* engine, uint64_t g
layout->addWidget(m_preview, 1);
auto* scroll = new QScrollArea(this); scroll->setWidgetResizable(true);
auto* settings = new QWidget(scroll); auto* form = new QFormLayout(settings);
form->setRowWrapPolicy(QFormLayout::WrapLongRows);
scroll->setWidget(settings); scroll->setMinimumHeight(220); layout->addWidget(scroll);
m_source = new QComboBox(settings); m_source->setSizeAdjustPolicy(QComboBox::AdjustToMinimumContentsLengthWithIcon);
m_source->setMinimumContentsLength(16); m_source->setSizePolicy(QSizePolicy::Expanding, QSizePolicy::Fixed);
auto* sourceRow = new QWidget(settings); auto* sourceLayout = new QHBoxLayout(sourceRow); sourceLayout->setContentsMargins(0, 0, 0, 0);
sourceLayout->addWidget(m_source, 1); auto* refresh = new QPushButton(text("Refresh"), settings); sourceLayout->addWidget(refresh);
connect(refresh, &QPushButton::clicked, this, &StreamPublishWindow::refreshSources); form->addRow(text("Video source"), sourceRow);
connect(m_source, QOverload<int>::of(&QComboBox::currentIndexChanged), this, [this] { m_source->setToolTip(m_source->currentText()); });
m_mode = new QComboBox(settings);
for (auto mode : {QSize(640, 360), QSize(1280, 720), QSize(1920, 1080)})
for (int fps : {15, 30, 60}) m_mode->addItem(QStringLiteral("%1 × %2 · %3 fps").arg(mode.width()).arg(mode.height()).arg(fps),
@ -129,7 +132,8 @@ void StreamPublishWindow::apply() { if (!m_updating && m_engine->active()) m_eng
void StreamPublishWindow::synchronize() {
m_updating = true; bool active = m_engine->active();
m_start->setText(active ? text("Stop stream") : text("Start stream"));
m_start->setEnabled(active || (StreamPublishEngine::supported() && m_source->count() && m_codec->count()));
m_start->setEnabled((active && m_engine->running()) || (!active && StreamPublishEngine::supported() && m_source->count() && m_codec->count()));
for (auto* box : {m_source, m_mode, m_codec, m_bitrate}) box->setEnabled(!active || m_engine->running());
m_audio->setEnabled(!active); m_microphone->setEnabled(!active); m_mute->setEnabled(active && m_engine->config().audio);
if (active) {
auto config = m_engine->config();

Write Preview
Loading…
Cancel
Save