jeka
/
utun2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Preserve video frames until audio clock reaches their PTS

stream-multiview
evgeny 1 week ago
parent
1d984f0d0b
commit
ef67e3ed2c
2 changed files with 29 additions and 11 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      src/stream/stream_buffer.c
  2. 38
      tools/chatgui/src/streamplayer_window.cpp

2
src/stream/stream_buffer.c
Unescape View File

@ -125,7 +125,7 @@ static void buffer_evict_oldest(struct stream_buffer *b) {
b->stats.blocks -= batch->count;
b->stats.stored_bytes -= batch->count * b->block_bytes;
b->stats.evicted_batches++;
DEBUG_DEBUG(DEBUG_CATEGORY_STREAM, "buffer evicted seq=%llu blocks=%u retained=%u bytes=%zu low=%llu",
DEBUG_TRACE(DEBUG_CATEGORY_STREAM, "buffer evicted seq=%llu blocks=%u retained=%u bytes=%zu low=%llu",
(unsigned long long)batch->seq, batch->count, b->stats.blocks, b->stats.stored_bytes,
(unsigned long long)b->stats.expired_before);
u_free(batch);

38
tools/chatgui/src/streamplayer_window.cpp
Unescape View File

@ -46,6 +46,7 @@ struct StreamPlayerWindow::Playback {
std::thread worker;
std::mutex lock;
std::deque<Video> videos;
size_t videoBytes = 0;
std::deque<Audio> audios;
int64_t prefillUntil = 0, videoBase = -1, videoAt = 0, audioPts = -1, audioAt = 0;
uint64_t shown = 0, underruns = 0;
@ -130,19 +131,21 @@ void StreamPlayerWindow::openAudio() {
void StreamPlayerWindow::tick() {
auto& p = *m_playback;
if (m_audio.needsRecovery()) { m_audio.close(); p.audioAvailable = false; openAudio(); }
QImage image; bool finished = false;
QImage image; bool finished = false, first = false;
{
std::lock_guard<std::mutex> guard(p.lock);
int64_t now = nowUs(), clock = p.clock(now);
if (now >= p.prefillUntil) while (!p.videos.empty() && p.videos.front().pts <= clock) {
p.videoBytes -= static_cast<size_t>(p.videos.front().image.sizeInBytes());
image = std::move(p.videos.front().image); p.videos.pop_front();
}
if (!image.isNull()) first = p.shown++ == 0;
finished = p.ended && p.videos.empty() && (p.audios.empty() || !p.audioAvailable);
}
if (!image.isNull()) {
m_render->setFrame(image);
m_status->setText(text("Live"));
if (!p.shown++) {
if (first) {
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop first frame id=%llu %dx%d", (unsigned long long)p.id, image.width(), image.height());
}
emit framePresented();
@ -178,13 +181,13 @@ void StreamPlayerWindow::decode() {
opus_codec_decoder_t* opus = nullptr;
bool waitKey = true, opened = false;
int64_t deadline = nowUs() + 10000000, reportAt = nowUs(), expectedAudio = -1;
uint64_t decoded = 0, discarded = 0;
uint64_t decoded = 0, discarded = 0, late = 0, pressureDropped = 0, previousDecoded = 0, previousShown = 0;
uint32_t videoRevision = 0;
auto reset = [&] {
avcodec_free_context(&decoder);
opus_codec_decoder_destroy(opus); opus = nullptr; expectedAudio = -1; waitKey = true;
std::lock_guard<std::mutex> guard(p.lock);
p.videos.clear(); p.audios.clear(); p.prefillUntil = 0; p.videoBase = p.audioPts = -1;
p.videos.clear(); p.videoBytes = 0; p.audios.clear(); p.prefillUntil = 0; p.videoBase = p.audioPts = -1;
emit statusChanged(text("Waiting for a key frame…"));
};
auto fail = [&](const QString& reason) {
@ -244,7 +247,7 @@ void StreamPlayerWindow::decode() {
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop video revision=%u->%u dimensions=%ux%u rotation=%u; preserve audio",
videoRevision, record.format.revision, record.format.width, record.format.height, record.format.rotation);
avcodec_free_context(&decoder); av_frame_unref(frame); waitKey = true; videoRevision = record.format.revision;
{ std::lock_guard<std::mutex> guard(p.lock); p.videos.clear(); }
{ std::lock_guard<std::mutex> guard(p.lock); p.videos.clear(); p.videoBytes = 0; }
deadline = nowUs() + 10000000;
}
if (waitKey && !(record.flags & STREAM_MEDIA_KEY)) continue;
@ -277,7 +280,7 @@ void StreamPlayerWindow::decode() {
std::lock_guard<std::mutex> guard(p.lock);
if (!p.prefillUntil) p.prefillUntil = nowUs() + 150000;
if (p.videoBase < 0) { p.videoBase = pts; p.videoAt = p.prefillUntil; }
if (pts < p.clock(nowUs()) - 100000) { discarded++; continue; }
if (pts < p.clock(nowUs()) - 100000) { discarded++; late++; continue; }
}
converter = sws_getCachedContext(converter, frame->width, frame->height, static_cast<AVPixelFormat>(frame->format),
frame->width, frame->height, AV_PIX_FMT_RGB24, SWS_BILINEAR, nullptr, nullptr, nullptr);
@ -290,7 +293,12 @@ void StreamPlayerWindow::decode() {
}
if (record.format.rotation) image = image.transformed(QTransform().rotate(record.format.rotation));
std::lock_guard<std::mutex> guard(p.lock);
if (p.videos.size() == 6) { p.videos.pop_front(); discarded++; }
size_t bytes = static_cast<size_t>(image.sizeInBytes());
// Будущие кадры ждут аудиочасы; при пределе памяти сохраняем ближайшие к показу.
if (p.videos.size() >= 128 || p.videoBytes + bytes > 256u * 1024u * 1024u) {
discarded++; pressureDropped++; continue;
}
p.videoBytes += bytes;
p.videos.push_back({std::move(image), pts});
}
if (received != AVERROR(EAGAIN) && received != AVERROR_EOF && p.running) {
@ -298,10 +306,20 @@ void StreamPlayerWindow::decode() {
reset(); deadline = nowUs() + 10000000;
}
if (nowUs() - reportAt >= 1000000) {
reportAt = nowUs(); std::lock_guard<std::mutex> guard(p.lock);
DEBUG_DEBUG(DEBUG_CATEGORY_STREAM, "desktop viewer id=%llu decoded=%llu discarded=%llu videoQ=%zu audioQ=%zu underruns=%llu",
int64_t now = nowUs(); double seconds = (now - reportAt) / 1000000.0; reportAt = now;
std::lock_guard<std::mutex> guard(p.lock); int64_t clock = p.clock(now);
DEBUG_DEBUG(DEBUG_CATEGORY_STREAM,
"desktop viewer id=%llu decoded=%llu discarded=%llu late=%llu pressure=%llu shown=%llu decodeFps=%.1f showFps=%.1f "
"videoQ=%zu videoBytes=%zu firstAheadUs=%lld lastAheadUs=%lld audioQ=%zu audioAtDeltaUs=%lld underruns=%llu",
(unsigned long long)p.id, (unsigned long long)decoded, (unsigned long long)discarded,
p.videos.size(), p.audios.size(), (unsigned long long)p.underruns);
(unsigned long long)late, (unsigned long long)pressureDropped, (unsigned long long)p.shown,
(decoded - previousDecoded) / seconds, (p.shown - previousShown) / seconds, p.videos.size(), p.videoBytes,
(long long)(p.videos.empty() ? 0 : p.videos.front().pts - clock),
(long long)(p.videos.empty() ? 0 : p.videos.back().pts - clock), p.audios.size(),
(long long)(p.audioPts < 0 ? 0 : now - p.audioAt), (unsigned long long)p.underruns);
if (pressureDropped) DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop video queue pressure id=%llu dropped=%llu; preserve due frames",
(unsigned long long)p.id, (unsigned long long)pressureDropped);
previousDecoded = decoded; previousShown = p.shown;
}
}
avcodec_free_context(&decoder); av_frame_free(&frame); av_packet_free(&packet);

Write Preview
Loading…
Cancel
Save