From 2c7bd78a646089394eacf147cf287515f72e8784 Mon Sep 17 00:00:00 2001 From: evgeny Date: Tue, 13 Oct 2026 22:37:09 +0200 Subject: [PATCH] Track stereo time stretched audio through the device playback timeline --- src/stream/stream_playback_audio.cpp | 215 +++++++++++++++++++++++++++ src/stream/stream_playback_audio.h | 36 +++++ 2 files changed, 251 insertions(+) create mode 100644 src/stream/stream_playback_audio.cpp create mode 100644 src/stream/stream_playback_audio.h diff --git a/src/stream/stream_playback_audio.cpp b/src/stream/stream_playback_audio.cpp new file mode 100644 index 00000000..e1fc26ba --- /dev/null +++ b/src/stream/stream_playback_audio.cpp @@ -0,0 +1,215 @@ +#include "stream_playback_audio.h" +#include "soundtouch/SoundTouch.h" +#include +#include +#include +#include + +extern "C" { +#include "../../lib/debug_config.h" +#include "../../lib/mem.h" +#include "../../lib/opus_codec.h" +} + +namespace { +constexpr unsigned rate = 48000, framesPerPacket = 960, maxFrames = 8192, mapCapacity = 1024; +struct Span { double first = 0, last = 0, ptsFirst = 0, ptsLast = 0; }; +double mapped(const Span& span, double position) { + if (span.last <= span.first) return span.ptsLast; + double fraction = std::clamp((position - span.first) / (span.last - span.first), 0.0, 1.0); + return span.ptsFirst + fraction * (span.ptsLast - span.ptsFirst); +} +} + +struct stream_playback_audio { + opus_codec_decoder_t* opus = nullptr; + soundtouch::SoundTouch stretch; + Span input[mapCapacity], output[mapCapacity]; + unsigned inputHead = 0, inputCount = 0, outputHead = 0, outputCount = 0; + int tempo = 1000; + int64_t startPts = 0, expectedPts = -1; + double expectedOutput = 0, producedOutput = 0, lastPts = 0; + uint64_t written = 0; + bool finished = false, failed = false; + int16_t pcm[framesPerPacket * 2]; + float samples[maxFrames * 2]; +}; + +/* Между сменами темпа соответствие линейно; точки сохраняют старый темп уже поставленного PCM. */ +static bool appendOutput(stream_playback_audio* a, const Span& span) { + if (a->outputCount) { + auto& previous = a->output[(a->outputHead + a->outputCount - 1) % mapCapacity]; + double previousSlope = (previous.ptsLast - previous.ptsFirst) / (previous.last - previous.first); + double slope = (span.ptsLast - span.ptsFirst) / (span.last - span.first); + if (std::abs(previous.last - span.first) < 0.000001 && std::abs(previous.ptsLast - span.ptsFirst) < 0.001 + && std::abs(previousSlope - slope) < 0.000001) { + previous.last = span.last; previous.ptsLast = span.ptsLast; return true; + } + } + if (a->outputCount == mapCapacity) { + DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio output map overflow written=%llu", (unsigned long long)a->written); + a->failed = true; return false; + } + a->output[(a->outputHead + a->outputCount++) % mapCapacity] = span; + return true; +} + +/* Прогрев перед точкой старта не создаёт слышимый PCM; частично перекрытый пакет обрезается по кадрам. */ +static int feedPcm(stream_playback_audio* a, int64_t pts, const uint8_t* packet, int length) { + int count = opus_codec_decode(a->opus, packet, length, a->pcm, framesPerPacket); + if (count != (int)framesPerPacket) { + DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback Opus decode failed pts=%lld result=%d", (long long)pts, count); return -1; + } + if (pts + 20000 <= a->startPts) return 0; + unsigned skip = pts < a->startPts ? (unsigned)((a->startPts - pts) * rate + 999999) / 1000000 : 0; + if (skip >= framesPerPacket) return 0; + unsigned frames = framesPerPacket - skip; + while (a->inputCount && a->input[a->inputHead].last <= a->producedOutput) { + a->inputHead = (a->inputHead + 1) % mapCapacity; a->inputCount--; + } + if (a->inputCount == mapCapacity) { + DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio input map overflow pts=%lld", (long long)pts); return -1; + } + double end = a->expectedOutput + frames * 1000.0 / a->tempo; + a->input[(a->inputHead + a->inputCount++) % mapCapacity] = + {a->expectedOutput, end, pts + skip * 1000000.0 / rate, pts + 20000.0}; + a->expectedOutput = end; + for (unsigned i = 0; i < frames * 2; i++) a->samples[i] = a->pcm[skip * 2 + i] / 32768.0f; + a->stretch.putSamples(a->samples, frames); + return 0; +} + +extern "C" { + +struct stream_playback_audio* stream_playback_audio_create(int64_t start) { + if (start < 0) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio invalid start=%lld", (long long)start); return nullptr; } + void* memory = u_malloc(sizeof(stream_playback_audio)); + if (!memory) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio allocation failed"); return nullptr; } + auto* a = new (memory) stream_playback_audio; + a->startPts = start; a->lastPts = (double)start; + a->opus = opus_codec_decoder_create(rate, 2); + if (!a->opus) { stream_playback_audio_destroy(a); return nullptr; } + a->stretch.setSampleRate(rate); a->stretch.setChannels(2); a->stretch.setTempo(1.0); + a->stretch.setSetting(SETTING_USE_QUICKSEEK, 1); + a->stretch.setSetting(SETTING_SEQUENCE_MS, 20); a->stretch.setSetting(SETTING_SEEKWINDOW_MS, 10); + a->stretch.setSetting(SETTING_OVERLAP_MS, 4); a->stretch.setSetting(SETTING_USE_AA_FILTER, 0); + DEBUG_INFO(DEBUG_CATEGORY_STREAM, "playback audio created start=%lld stereo=1 tempo=1.0..1.1", (long long)start); + return a; +} + +void stream_playback_audio_destroy(struct stream_playback_audio* a) { + if (!a) return; + DEBUG_INFO(DEBUG_CATEGORY_STREAM, "playback audio destroy written=%llu produced=%.0f pts=%.0f failed=%d", + (unsigned long long)a->written, a->producedOutput, a->lastPts, a->failed); + opus_codec_decoder_destroy(a->opus); a->~stream_playback_audio(); u_free(a); +} + +int stream_playback_audio_feed(struct stream_playback_audio* a, int64_t pts, const uint8_t* packet, size_t length) { + if (!a || !packet || !length || length > 1275 || pts < 0 || pts > INT64_MAX - 20000 || a->finished || a->failed) { + DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio invalid feed pts=%lld length=%zu", (long long)pts, length); return -1; + } + if (a->expectedPts >= 0 && pts < a->expectedPts - 10000) { + DEBUG_WARN(DEBUG_CATEGORY_STREAM, "playback audio late packet pts=%lld expected=%lld", (long long)pts, (long long)a->expectedPts); + return 1; + } + int missing = a->expectedPts < 0 ? 0 : (int)((pts - a->expectedPts + 10000) / 20000); + if (missing > 5) { + DEBUG_WARN(DEBUG_CATEGORY_STREAM, "playback audio discontinuity pts=%lld expected=%lld missing=%d", + (long long)pts, (long long)a->expectedPts, missing); return -2; + } + for (int i = 0; i < missing; i++) if (feedPcm(a, a->expectedPts + i * 20000, nullptr, 0) < 0) return -1; + if (feedPcm(a, pts, packet, (int)length) < 0) return -1; + a->expectedPts = pts + 20000; + return 0; +} + +void stream_playback_audio_set_tempo(struct stream_playback_audio* a, int tempo) { + if (!a || tempo < 1000 || tempo > 1100) { + DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio invalid tempo=%d", tempo); return; + } + if (tempo == a->tempo) return; + a->tempo = tempo; a->stretch.setTempo(tempo / 1000.0); +} + +int stream_playback_audio_needs_input(const struct stream_playback_audio* a) { + return a && !a->finished && !a->failed && a->stretch.numSamples() < framesPerPacket; +} + +/* Ограничение готовой границы применяется до выдачи PCM устройству, сохраняя общий A/V clock. */ +int stream_playback_audio_pull(struct stream_playback_audio* a, int16_t* pcm, unsigned frames, int64_t limit) { + if (!a || !pcm || frames > maxFrames || a->failed || limit < 0) { + DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio invalid pull frames=%u limit=%lld", frames, (long long)limit); return -1; + } + double until = a->producedOutput; + for (unsigned i = 0; i < a->inputCount; i++) { + const auto& span = a->input[(a->inputHead + i) % mapCapacity]; + if (span.last <= a->producedOutput) continue; + if (limit >= span.ptsLast) until = span.last; + else if (limit > span.ptsFirst) { + until = span.first + (limit - span.ptsFirst) * (span.last - span.first) / (span.ptsLast - span.ptsFirst); break; + } else break; + } + unsigned available = (unsigned)std::max(0.0, std::floor(until - a->producedOutput)); + frames = std::min(frames, std::min(available, a->stretch.numSamples())); + if (!frames) return 0; + unsigned got = a->stretch.receiveSamples(a->samples, frames); + for (unsigned i = 0; i < got * 2; i++) pcm[i] = (int16_t)std::lrint(std::clamp(a->samples[i] * 32767.0f, -32768.0f, 32767.0f)); + double position = a->producedOutput, end = position + got; + for (unsigned i = 0; i < a->inputCount && position < end; i++) { + const auto& span = a->input[(a->inputHead + i) % mapCapacity]; + if (span.last <= position) continue; + double next = std::min(end, span.last); + double firstPts = mapped(span, position), lastPts = mapped(span, next); + if (!appendOutput(a, {a->written + position - a->producedOutput, + a->written + next - a->producedOutput, firstPts, lastPts})) return -1; + a->lastPts = lastPts; position = next; + } + a->producedOutput = end; a->written += got; + return (int)got; +} + +void stream_playback_audio_silence(struct stream_playback_audio* a, unsigned frames) { + if (!a || !frames) return; + if (!appendOutput(a, {(double)a->written, (double)(a->written + frames), a->lastPts, a->lastPts})) return; + a->written += frames; +} + +int64_t stream_playback_audio_position(struct stream_playback_audio* a, uint64_t played) { + if (!a) return -1; + if (played > a->written) played = a->written; + while (a->outputCount > 1 && a->output[a->outputHead].last <= played) { + a->outputHead = (a->outputHead + 1) % mapCapacity; a->outputCount--; + } + if (!a->outputCount) return a->startPts; + return (int64_t)std::llround(mapped(a->output[a->outputHead], (double)played)); +} + +int64_t stream_playback_audio_pending_gain(struct stream_playback_audio* a, uint64_t played) { + if (!a) return 0; + double gain = 0; + for (unsigned i = 0; i < a->outputCount; i++) { + const auto& span = a->output[(a->outputHead + i) % mapCapacity]; + double begin = std::max(span.first, (double)played); + if (span.last <= begin) continue; + gain += std::max(0.0, span.ptsLast - mapped(span, begin) - (span.last - begin) * 1000000.0 / rate); + } + for (unsigned i = 0; i < a->inputCount; i++) { + const auto& span = a->input[(a->inputHead + i) % mapCapacity]; + double begin = std::max(span.first, a->producedOutput); + if (span.last <= begin) continue; + gain += std::max(0.0, span.ptsLast - mapped(span, begin) - (span.last - begin) * 1000000.0 / rate); + } + return (int64_t)std::ceil(gain); +} + +uint64_t stream_playback_audio_written(const struct stream_playback_audio* a) { return a ? a->written : 0; } + +void stream_playback_audio_finish(struct stream_playback_audio* a) { + if (!a || a->finished) return; + a->finished = true; a->stretch.flush(); +} + +int stream_playback_audio_empty(const struct stream_playback_audio* a) { + return !a || (a->finished && a->producedOutput + 0.5 >= a->expectedOutput); +} +} diff --git a/src/stream/stream_playback_audio.h b/src/stream/stream_playback_audio.h new file mode 100644 index 00000000..1f2ecbe6 --- /dev/null +++ b/src/stream/stream_playback_audio.h @@ -0,0 +1,36 @@ +/* Короткий stereo Opus/SoundTouch конвейер плеера. Кодированный запас остаётся в RS-кольце. + * Все вызовы одного экземпляра последовательны либо под внешним mutex. + * feed прогревает Opus перед start_pts, затем подаёт PCM с темпом 1.0..1.1 без изменения высоты звука. + * pull записывает соответствие кадров устройства исходному PTS; silence продлевает hardware time, но не media time. + * position/pending_gain используют реально проигранный frame counter, поэтому уже поставленный PCM сохраняет свой темп. + * Последний Opus-пакет/EOF не теряется: finish выталкивает остаток SoundTouch. */ +#ifndef STREAM_PLAYBACK_AUDIO_H +#define STREAM_PLAYBACK_AUDIO_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct stream_playback_audio; +struct stream_playback_audio *stream_playback_audio_create(int64_t start_pts_us); +void stream_playback_audio_destroy(struct stream_playback_audio *audio); +int stream_playback_audio_feed(struct stream_playback_audio *audio, int64_t pts_us, const uint8_t *packet, size_t length); +void stream_playback_audio_set_tempo(struct stream_playback_audio *audio, int tempo_milli); +int stream_playback_audio_needs_input(const struct stream_playback_audio *audio); +/* frames <=8192; limit_pts_us ограничивает продвижение исходного материала до готовой границы видео/аудио. + * Возвращает число stereo frames; оставшиеся места backend заполняет тишиной и вызывает silence. */ +int stream_playback_audio_pull(struct stream_playback_audio *audio, int16_t *pcm, unsigned frames, int64_t limit_pts_us); +void stream_playback_audio_silence(struct stream_playback_audio *audio, unsigned frames); +int64_t stream_playback_audio_position(struct stream_playback_audio *audio, uint64_t played_frames); +int64_t stream_playback_audio_pending_gain(struct stream_playback_audio *audio, uint64_t played_frames); +uint64_t stream_playback_audio_written(const struct stream_playback_audio *audio); +void stream_playback_audio_finish(struct stream_playback_audio *audio); +int stream_playback_audio_empty(const struct stream_playback_audio *audio); + +#ifdef __cplusplus +} +#endif +#endif