Track stereo time stretched audio through the device playback timeline

src/stream/stream_playback_audio.cpp

@@ -0,0 +1,215 @@
+#include "stream_playback_audio.h"
+#include "soundtouch/SoundTouch.h"
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <new>
+
+extern "C" {
+#include "../../lib/debug_config.h"
+#include "../../lib/mem.h"
+#include "../../lib/opus_codec.h"
+}
+
+namespace {
+constexpr unsigned rate = 48000, framesPerPacket = 960, maxFrames = 8192, mapCapacity = 1024;
+struct Span { double first = 0, last = 0, ptsFirst = 0, ptsLast = 0; };
+double mapped(const Span& span, double position) {
+    if (span.last <= span.first) return span.ptsLast;
+    double fraction = std::clamp((position - span.first) / (span.last - span.first), 0.0, 1.0);
+    return span.ptsFirst + fraction * (span.ptsLast - span.ptsFirst);
+}
+}
+
+struct stream_playback_audio {
+    opus_codec_decoder_t* opus = nullptr;
+    soundtouch::SoundTouch stretch;
+    Span input[mapCapacity], output[mapCapacity];
+    unsigned inputHead = 0, inputCount = 0, outputHead = 0, outputCount = 0;
+    int tempo = 1000;
+    int64_t startPts = 0, expectedPts = -1;
+    double expectedOutput = 0, producedOutput = 0, lastPts = 0;
+    uint64_t written = 0;
+    bool finished = false, failed = false;
+    int16_t pcm[framesPerPacket * 2];
+    float samples[maxFrames * 2];
+};
+
+/* Между сменами темпа соответствие линейно; точки сохраняют старый темп уже поставленного PCM. */
+static bool appendOutput(stream_playback_audio* a, const Span& span) {
+    if (a->outputCount) {
+        auto& previous = a->output[(a->outputHead + a->outputCount - 1) % mapCapacity];
+        double previousSlope = (previous.ptsLast - previous.ptsFirst) / (previous.last - previous.first);
+        double slope = (span.ptsLast - span.ptsFirst) / (span.last - span.first);
+        if (std::abs(previous.last - span.first) < 0.000001 && std::abs(previous.ptsLast - span.ptsFirst) < 0.001
+            && std::abs(previousSlope - slope) < 0.000001) {
+            previous.last = span.last; previous.ptsLast = span.ptsLast; return true;
+        }
+    }
+    if (a->outputCount == mapCapacity) {
+        DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio output map overflow written=%llu", (unsigned long long)a->written);
+        a->failed = true; return false;
+    }
+    a->output[(a->outputHead + a->outputCount++) % mapCapacity] = span;
+    return true;
+}
+
+/* Прогрев перед точкой старта не создаёт слышимый PCM; частично перекрытый пакет обрезается по кадрам. */
+static int feedPcm(stream_playback_audio* a, int64_t pts, const uint8_t* packet, int length) {
+    int count = opus_codec_decode(a->opus, packet, length, a->pcm, framesPerPacket);
+    if (count != (int)framesPerPacket) {
+        DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback Opus decode failed pts=%lld result=%d", (long long)pts, count); return -1;
+    }
+    if (pts + 20000 <= a->startPts) return 0;
+    unsigned skip = pts < a->startPts ? (unsigned)((a->startPts - pts) * rate + 999999) / 1000000 : 0;
+    if (skip >= framesPerPacket) return 0;
+    unsigned frames = framesPerPacket - skip;
+    while (a->inputCount && a->input[a->inputHead].last <= a->producedOutput) {
+        a->inputHead = (a->inputHead + 1) % mapCapacity; a->inputCount--;
+    }
+    if (a->inputCount == mapCapacity) {
+        DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio input map overflow pts=%lld", (long long)pts); return -1;
+    }
+    double end = a->expectedOutput + frames * 1000.0 / a->tempo;
+    a->input[(a->inputHead + a->inputCount++) % mapCapacity] =
+        {a->expectedOutput, end, pts + skip * 1000000.0 / rate, pts + 20000.0};
+    a->expectedOutput = end;
+    for (unsigned i = 0; i < frames * 2; i++) a->samples[i] = a->pcm[skip * 2 + i] / 32768.0f;
+    a->stretch.putSamples(a->samples, frames);
+    return 0;
+}
+
+extern "C" {
+
+struct stream_playback_audio* stream_playback_audio_create(int64_t start) {
+    if (start < 0) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio invalid start=%lld", (long long)start); return nullptr; }
+    void* memory = u_malloc(sizeof(stream_playback_audio));
+    if (!memory) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio allocation failed"); return nullptr; }
+    auto* a = new (memory) stream_playback_audio;
+    a->startPts = start; a->lastPts = (double)start;
+    a->opus = opus_codec_decoder_create(rate, 2);
+    if (!a->opus) { stream_playback_audio_destroy(a); return nullptr; }
+    a->stretch.setSampleRate(rate); a->stretch.setChannels(2); a->stretch.setTempo(1.0);
+    a->stretch.setSetting(SETTING_USE_QUICKSEEK, 1);
+    a->stretch.setSetting(SETTING_SEQUENCE_MS, 20); a->stretch.setSetting(SETTING_SEEKWINDOW_MS, 10);
+    a->stretch.setSetting(SETTING_OVERLAP_MS, 4); a->stretch.setSetting(SETTING_USE_AA_FILTER, 0);
+    DEBUG_INFO(DEBUG_CATEGORY_STREAM, "playback audio created start=%lld stereo=1 tempo=1.0..1.1", (long long)start);
+    return a;
+}
+
+void stream_playback_audio_destroy(struct stream_playback_audio* a) {
+    if (!a) return;
+    DEBUG_INFO(DEBUG_CATEGORY_STREAM, "playback audio destroy written=%llu produced=%.0f pts=%.0f failed=%d",
+               (unsigned long long)a->written, a->producedOutput, a->lastPts, a->failed);
+    opus_codec_decoder_destroy(a->opus); a->~stream_playback_audio(); u_free(a);
+}
+
+int stream_playback_audio_feed(struct stream_playback_audio* a, int64_t pts, const uint8_t* packet, size_t length) {
+    if (!a || !packet || !length || length > 1275 || pts < 0 || pts > INT64_MAX - 20000 || a->finished || a->failed) {
+        DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio invalid feed pts=%lld length=%zu", (long long)pts, length); return -1;
+    }
+    if (a->expectedPts >= 0 && pts < a->expectedPts - 10000) {
+        DEBUG_WARN(DEBUG_CATEGORY_STREAM, "playback audio late packet pts=%lld expected=%lld", (long long)pts, (long long)a->expectedPts);
+        return 1;
+    }
+    int missing = a->expectedPts < 0 ? 0 : (int)((pts - a->expectedPts + 10000) / 20000);
+    if (missing > 5) {
+        DEBUG_WARN(DEBUG_CATEGORY_STREAM, "playback audio discontinuity pts=%lld expected=%lld missing=%d",
+                   (long long)pts, (long long)a->expectedPts, missing); return -2;
+    }
+    for (int i = 0; i < missing; i++) if (feedPcm(a, a->expectedPts + i * 20000, nullptr, 0) < 0) return -1;
+    if (feedPcm(a, pts, packet, (int)length) < 0) return -1;
+    a->expectedPts = pts + 20000;
+    return 0;
+}
+
+void stream_playback_audio_set_tempo(struct stream_playback_audio* a, int tempo) {
+    if (!a || tempo < 1000 || tempo > 1100) {
+        DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio invalid tempo=%d", tempo); return;
+    }
+    if (tempo == a->tempo) return;
+    a->tempo = tempo; a->stretch.setTempo(tempo / 1000.0);
+}
+
+int stream_playback_audio_needs_input(const struct stream_playback_audio* a) {
+    return a && !a->finished && !a->failed && a->stretch.numSamples() < framesPerPacket;
+}
+
+/* Ограничение готовой границы применяется до выдачи PCM устройству, сохраняя общий A/V clock. */
+int stream_playback_audio_pull(struct stream_playback_audio* a, int16_t* pcm, unsigned frames, int64_t limit) {
+    if (!a || !pcm || frames > maxFrames || a->failed || limit < 0) {
+        DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio invalid pull frames=%u limit=%lld", frames, (long long)limit); return -1;
+    }
+    double until = a->producedOutput;
+    for (unsigned i = 0; i < a->inputCount; i++) {
+        const auto& span = a->input[(a->inputHead + i) % mapCapacity];
+        if (span.last <= a->producedOutput) continue;
+        if (limit >= span.ptsLast) until = span.last;
+        else if (limit > span.ptsFirst) {
+            until = span.first + (limit - span.ptsFirst) * (span.last - span.first) / (span.ptsLast - span.ptsFirst); break;
+        } else break;
+    }
+    unsigned available = (unsigned)std::max(0.0, std::floor(until - a->producedOutput));
+    frames = std::min(frames, std::min(available, a->stretch.numSamples()));
+    if (!frames) return 0;
+    unsigned got = a->stretch.receiveSamples(a->samples, frames);
+    for (unsigned i = 0; i < got * 2; i++) pcm[i] = (int16_t)std::lrint(std::clamp(a->samples[i] * 32767.0f, -32768.0f, 32767.0f));
+    double position = a->producedOutput, end = position + got;
+    for (unsigned i = 0; i < a->inputCount && position < end; i++) {
+        const auto& span = a->input[(a->inputHead + i) % mapCapacity];
+        if (span.last <= position) continue;
+        double next = std::min(end, span.last);
+        double firstPts = mapped(span, position), lastPts = mapped(span, next);
+        if (!appendOutput(a, {a->written + position - a->producedOutput,
+                              a->written + next - a->producedOutput, firstPts, lastPts})) return -1;
+        a->lastPts = lastPts; position = next;
+    }
+    a->producedOutput = end; a->written += got;
+    return (int)got;
+}
+
+void stream_playback_audio_silence(struct stream_playback_audio* a, unsigned frames) {
+    if (!a || !frames) return;
+    if (!appendOutput(a, {(double)a->written, (double)(a->written + frames), a->lastPts, a->lastPts})) return;
+    a->written += frames;
+}
+
+int64_t stream_playback_audio_position(struct stream_playback_audio* a, uint64_t played) {
+    if (!a) return -1;
+    if (played > a->written) played = a->written;
+    while (a->outputCount > 1 && a->output[a->outputHead].last <= played) {
+        a->outputHead = (a->outputHead + 1) % mapCapacity; a->outputCount--;
+    }
+    if (!a->outputCount) return a->startPts;
+    return (int64_t)std::llround(mapped(a->output[a->outputHead], (double)played));
+}
+
+int64_t stream_playback_audio_pending_gain(struct stream_playback_audio* a, uint64_t played) {
+    if (!a) return 0;
+    double gain = 0;
+    for (unsigned i = 0; i < a->outputCount; i++) {
+        const auto& span = a->output[(a->outputHead + i) % mapCapacity];
+        double begin = std::max(span.first, (double)played);
+        if (span.last <= begin) continue;
+        gain += std::max(0.0, span.ptsLast - mapped(span, begin) - (span.last - begin) * 1000000.0 / rate);
+    }
+    for (unsigned i = 0; i < a->inputCount; i++) {
+        const auto& span = a->input[(a->inputHead + i) % mapCapacity];
+        double begin = std::max(span.first, a->producedOutput);
+        if (span.last <= begin) continue;
+        gain += std::max(0.0, span.ptsLast - mapped(span, begin) - (span.last - begin) * 1000000.0 / rate);
+    }
+    return (int64_t)std::ceil(gain);
+}
+
+uint64_t stream_playback_audio_written(const struct stream_playback_audio* a) { return a ? a->written : 0; }
+
+void stream_playback_audio_finish(struct stream_playback_audio* a) {
+    if (!a || a->finished) return;
+    a->finished = true; a->stretch.flush();
+}
+
+int stream_playback_audio_empty(const struct stream_playback_audio* a) {
+    return !a || (a->finished && a->producedOutput + 0.5 >= a->expectedOutput);
+}
+}

src/stream/stream_playback_audio.h

@@ -0,0 +1,36 @@
+/* Короткий stereo Opus/SoundTouch конвейер плеера. Кодированный запас остаётся в RS-кольце.
+ * Все вызовы одного экземпляра последовательны либо под внешним mutex.
+ * feed прогревает Opus перед start_pts, затем подаёт PCM с темпом 1.0..1.1 без изменения высоты звука.
+ * pull записывает соответствие кадров устройства исходному PTS; silence продлевает hardware time, но не media time.
+ * position/pending_gain используют реально проигранный frame counter, поэтому уже поставленный PCM сохраняет свой темп.
+ * Последний Opus-пакет/EOF не теряется: finish выталкивает остаток SoundTouch. */
+#ifndef STREAM_PLAYBACK_AUDIO_H
+#define STREAM_PLAYBACK_AUDIO_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct stream_playback_audio;
+struct stream_playback_audio *stream_playback_audio_create(int64_t start_pts_us);
+void stream_playback_audio_destroy(struct stream_playback_audio *audio);
+int stream_playback_audio_feed(struct stream_playback_audio *audio, int64_t pts_us, const uint8_t *packet, size_t length);
+void stream_playback_audio_set_tempo(struct stream_playback_audio *audio, int tempo_milli);
+int stream_playback_audio_needs_input(const struct stream_playback_audio *audio);
+/* frames <=8192; limit_pts_us ограничивает продвижение исходного материала до готовой границы видео/аудио.
+ * Возвращает число stereo frames; оставшиеся места backend заполняет тишиной и вызывает silence. */
+int stream_playback_audio_pull(struct stream_playback_audio *audio, int16_t *pcm, unsigned frames, int64_t limit_pts_us);
+void stream_playback_audio_silence(struct stream_playback_audio *audio, unsigned frames);
+int64_t stream_playback_audio_position(struct stream_playback_audio *audio, uint64_t played_frames);
+int64_t stream_playback_audio_pending_gain(struct stream_playback_audio *audio, uint64_t played_frames);
+uint64_t stream_playback_audio_written(const struct stream_playback_audio *audio);
+void stream_playback_audio_finish(struct stream_playback_audio *audio);
+int stream_playback_audio_empty(const struct stream_playback_audio *audio);
+
+#ifdef __cplusplus
+}
+#endif
+#endif