jeka/utun2
1
0 Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Preserve video frames until audio clock reaches their PTS

Browse Source
This commit is contained in:
evgeny committed 2026-10-13 12:55:56 +02:00
1 parent 1d984f0d0b
commit ef67e3ed2c
2 files changed
+29 -11
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tools/chatgui/src/streamplayer_window.cpp
+28 -10
View File
@@ -46,6 +46,7 @@ struct StreamPlayerWindow::Playback {
std::thread worker;
std::mutex lock;
std::deque<Video> videos;
size_t videoBytes = 0;
std::deque<Audio> audios;
int64_t prefillUntil = 0, videoBase = -1, videoAt = 0, audioPts = -1, audioAt = 0;
uint64_t shown = 0, underruns = 0;
@@ -130,19 +131,21 @@ void StreamPlayerWindow::openAudio() {
void StreamPlayerWindow::tick() {
auto& p = *m_playback;
if (m_audio.needsRecovery()) { m_audio.close(); p.audioAvailable = false; openAudio(); }
QImage image; bool finished = false;
QImage image; bool finished = false, first = false;
{
std::lock_guard<std::mutex> guard(p.lock);
int64_t now = nowUs(), clock = p.clock(now);
if (now >= p.prefillUntil) while (!p.videos.empty() && p.videos.front().pts <= clock) {
p.videoBytes -= static_cast<size_t>(p.videos.front().image.sizeInBytes());
image = std::move(p.videos.front().image); p.videos.pop_front();
}
if (!image.isNull()) first = p.shown++ == 0;
finished = p.ended && p.videos.empty() && (p.audios.empty() || !p.audioAvailable);
}
if (!image.isNull()) {
m_render->setFrame(image);
m_status->setText(text("Live"));
if (!p.shown++) {
if (first) {
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop first frame id=%llu %dx%d", (unsigned long long)p.id, image.width(), image.height());
}
emit framePresented();
@@ -178,13 +181,13 @@ void StreamPlayerWindow::decode() {
opus_codec_decoder_t* opus = nullptr;
bool waitKey = true, opened = false;
int64_t deadline = nowUs() + 10000000, reportAt = nowUs(), expectedAudio = -1;
uint64_t decoded = 0, discarded = 0;
uint64_t decoded = 0, discarded = 0, late = 0, pressureDropped = 0, previousDecoded = 0, previousShown = 0;
uint32_t videoRevision = 0;
auto reset = [&] {
avcodec_free_context(&decoder);
opus_codec_decoder_destroy(opus); opus = nullptr; expectedAudio = -1; waitKey = true;
std::lock_guard<std::mutex> guard(p.lock);
p.videos.clear(); p.audios.clear(); p.prefillUntil = 0; p.videoBase = p.audioPts = -1;
p.videos.clear(); p.videoBytes = 0; p.audios.clear(); p.prefillUntil = 0; p.videoBase = p.audioPts = -1;
emit statusChanged(text("Waiting for a key frame…"));
};
auto fail = [&](const QString& reason) {
@@ -244,7 +247,7 @@ void StreamPlayerWindow::decode() {
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop video revision=%u->%u dimensions=%ux%u rotation=%u; preserve audio",
videoRevision, record.format.revision, record.format.width, record.format.height, record.format.rotation);
avcodec_free_context(&decoder); av_frame_unref(frame); waitKey = true; videoRevision = record.format.revision;
{ std::lock_guard<std::mutex> guard(p.lock); p.videos.clear(); }
{ std::lock_guard<std::mutex> guard(p.lock); p.videos.clear(); p.videoBytes = 0; }
deadline = nowUs() + 10000000;
}
if (waitKey && !(record.flags & STREAM_MEDIA_KEY)) continue;
@@ -277,7 +280,7 @@ void StreamPlayerWindow::decode() {
std::lock_guard<std::mutex> guard(p.lock);
if (!p.prefillUntil) p.prefillUntil = nowUs() + 150000;
if (p.videoBase < 0) { p.videoBase = pts; p.videoAt = p.prefillUntil; }
if (pts < p.clock(nowUs()) - 100000) { discarded++; continue; }
if (pts < p.clock(nowUs()) - 100000) { discarded++; late++; continue; }
}
converter = sws_getCachedContext(converter, frame->width, frame->height, static_cast<AVPixelFormat>(frame->format),
frame->width, frame->height, AV_PIX_FMT_RGB24, SWS_BILINEAR, nullptr, nullptr, nullptr);
@@ -290,7 +293,12 @@ void StreamPlayerWindow::decode() {
}
if (record.format.rotation) image = image.transformed(QTransform().rotate(record.format.rotation));
std::lock_guard<std::mutex> guard(p.lock);
if (p.videos.size() == 6) { p.videos.pop_front(); discarded++; }
size_t bytes = static_cast<size_t>(image.sizeInBytes());
// Будущие кадры ждут аудиочасы; при пределе памяти сохраняем ближайшие к показу.
if (p.videos.size() >= 128 || p.videoBytes + bytes > 256u * 1024u * 1024u) {
discarded++; pressureDropped++; continue;
}
p.videoBytes += bytes;
p.videos.push_back({std::move(image), pts});
}
if (received != AVERROR(EAGAIN) && received != AVERROR_EOF && p.running) {
@@ -298,10 +306,20 @@ void StreamPlayerWindow::decode() {
reset(); deadline = nowUs() + 10000000;
}
if (nowUs() - reportAt >= 1000000) {
reportAt = nowUs(); std::lock_guard<std::mutex> guard(p.lock);
DEBUG_DEBUG(DEBUG_CATEGORY_STREAM, "desktop viewer id=%llu decoded=%llu discarded=%llu videoQ=%zu audioQ=%zu underruns=%llu",
int64_t now = nowUs(); double seconds = (now - reportAt) / 1000000.0; reportAt = now;
std::lock_guard<std::mutex> guard(p.lock); int64_t clock = p.clock(now);
DEBUG_DEBUG(DEBUG_CATEGORY_STREAM,
"desktop viewer id=%llu decoded=%llu discarded=%llu late=%llu pressure=%llu shown=%llu decodeFps=%.1f showFps=%.1f "
"videoQ=%zu videoBytes=%zu firstAheadUs=%lld lastAheadUs=%lld audioQ=%zu audioAtDeltaUs=%lld underruns=%llu",
(unsigned long long)p.id, (unsigned long long)decoded, (unsigned long long)discarded,
p.videos.size(), p.audios.size(), (unsigned long long)p.underruns);
(unsigned long long)late, (unsigned long long)pressureDropped, (unsigned long long)p.shown,
(decoded - previousDecoded) / seconds, (p.shown - previousShown) / seconds, p.videos.size(), p.videoBytes,
(long long)(p.videos.empty() ? 0 : p.videos.front().pts - clock),
(long long)(p.videos.empty() ? 0 : p.videos.back().pts - clock), p.audios.size(),
(long long)(p.audioPts < 0 ? 0 : now - p.audioAt), (unsigned long long)p.underruns);
if (pressureDropped) DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop video queue pressure id=%llu dropped=%llu; preserve due frames",
(unsigned long long)p.id, (unsigned long long)pressureDropped);
previousDecoded = decoded; previousShown = p.shown;
}
}
avcodec_free_context(&decoder); av_frame_free(&frame); av_packet_free(&packet);