jeka
/
utun2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Track stereo time stretched audio through the device playback timeline

master
evgeny 6 days ago
parent
9f37c54a3c
commit
2c7bd78a64
2 changed files with 251 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 215
      src/stream/stream_playback_audio.cpp
  2. 36
      src/stream/stream_playback_audio.h

215
src/stream/stream_playback_audio.cpp
Unescape View File

@ -0,0 +1,215 @@
#include "stream_playback_audio.h"
#include "soundtouch/SoundTouch.h"
#include <algorithm>
#include <cmath>
#include <cstring>
#include <new>
extern "C" {
#include "../../lib/debug_config.h"
#include "../../lib/mem.h"
#include "../../lib/opus_codec.h"
}
namespace {
constexpr unsigned rate = 48000, framesPerPacket = 960, maxFrames = 8192, mapCapacity = 1024;
struct Span { double first = 0, last = 0, ptsFirst = 0, ptsLast = 0; };
double mapped(const Span& span, double position) {
if (span.last <= span.first) return span.ptsLast;
double fraction = std::clamp((position - span.first) / (span.last - span.first), 0.0, 1.0);
return span.ptsFirst + fraction * (span.ptsLast - span.ptsFirst);
}
}
struct stream_playback_audio {
opus_codec_decoder_t* opus = nullptr;
soundtouch::SoundTouch stretch;
Span input[mapCapacity], output[mapCapacity];
unsigned inputHead = 0, inputCount = 0, outputHead = 0, outputCount = 0;
int tempo = 1000;
int64_t startPts = 0, expectedPts = -1;
double expectedOutput = 0, producedOutput = 0, lastPts = 0;
uint64_t written = 0;
bool finished = false, failed = false;
int16_t pcm[framesPerPacket * 2];
float samples[maxFrames * 2];
};
/* Между сменами темпа соответствие линейно; точки сохраняют старый темп уже поставленного PCM. */
static bool appendOutput(stream_playback_audio* a, const Span& span) {
if (a->outputCount) {
auto& previous = a->output[(a->outputHead + a->outputCount - 1) % mapCapacity];
double previousSlope = (previous.ptsLast - previous.ptsFirst) / (previous.last - previous.first);
double slope = (span.ptsLast - span.ptsFirst) / (span.last - span.first);
if (std::abs(previous.last - span.first) < 0.000001 && std::abs(previous.ptsLast - span.ptsFirst) < 0.001
&& std::abs(previousSlope - slope) < 0.000001) {
previous.last = span.last; previous.ptsLast = span.ptsLast; return true;
}
}
if (a->outputCount == mapCapacity) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio output map overflow written=%llu", (unsigned long long)a->written);
a->failed = true; return false;
}
a->output[(a->outputHead + a->outputCount++) % mapCapacity] = span;
return true;
}
/* Прогрев перед точкой старта не создаёт слышимый PCM; частично перекрытый пакет обрезается по кадрам. */
static int feedPcm(stream_playback_audio* a, int64_t pts, const uint8_t* packet, int length) {
int count = opus_codec_decode(a->opus, packet, length, a->pcm, framesPerPacket);
if (count != (int)framesPerPacket) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback Opus decode failed pts=%lld result=%d", (long long)pts, count); return -1;
}
if (pts + 20000 <= a->startPts) return 0;
unsigned skip = pts < a->startPts ? (unsigned)((a->startPts - pts) * rate + 999999) / 1000000 : 0;
if (skip >= framesPerPacket) return 0;
unsigned frames = framesPerPacket - skip;
while (a->inputCount && a->input[a->inputHead].last <= a->producedOutput) {
a->inputHead = (a->inputHead + 1) % mapCapacity; a->inputCount--;
}
if (a->inputCount == mapCapacity) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio input map overflow pts=%lld", (long long)pts); return -1;
}
double end = a->expectedOutput + frames * 1000.0 / a->tempo;
a->input[(a->inputHead + a->inputCount++) % mapCapacity] =
{a->expectedOutput, end, pts + skip * 1000000.0 / rate, pts + 20000.0};
a->expectedOutput = end;
for (unsigned i = 0; i < frames * 2; i++) a->samples[i] = a->pcm[skip * 2 + i] / 32768.0f;
a->stretch.putSamples(a->samples, frames);
return 0;
}
extern "C" {
struct stream_playback_audio* stream_playback_audio_create(int64_t start) {
if (start < 0) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio invalid start=%lld", (long long)start); return nullptr; }
void* memory = u_malloc(sizeof(stream_playback_audio));
if (!memory) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio allocation failed"); return nullptr; }
auto* a = new (memory) stream_playback_audio;
a->startPts = start; a->lastPts = (double)start;
a->opus = opus_codec_decoder_create(rate, 2);
if (!a->opus) { stream_playback_audio_destroy(a); return nullptr; }
a->stretch.setSampleRate(rate); a->stretch.setChannels(2); a->stretch.setTempo(1.0);
a->stretch.setSetting(SETTING_USE_QUICKSEEK, 1);
a->stretch.setSetting(SETTING_SEQUENCE_MS, 20); a->stretch.setSetting(SETTING_SEEKWINDOW_MS, 10);
a->stretch.setSetting(SETTING_OVERLAP_MS, 4); a->stretch.setSetting(SETTING_USE_AA_FILTER, 0);
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "playback audio created start=%lld stereo=1 tempo=1.0..1.1", (long long)start);
return a;
}
void stream_playback_audio_destroy(struct stream_playback_audio* a) {
if (!a) return;
DEBUG_INFO(DEBUG_CATEGORY_STREAM, "playback audio destroy written=%llu produced=%.0f pts=%.0f failed=%d",
(unsigned long long)a->written, a->producedOutput, a->lastPts, a->failed);
opus_codec_decoder_destroy(a->opus); a->~stream_playback_audio(); u_free(a);
}
int stream_playback_audio_feed(struct stream_playback_audio* a, int64_t pts, const uint8_t* packet, size_t length) {
if (!a || !packet || !length || length > 1275 || pts < 0 || pts > INT64_MAX - 20000 || a->finished || a->failed) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio invalid feed pts=%lld length=%zu", (long long)pts, length); return -1;
}
if (a->expectedPts >= 0 && pts < a->expectedPts - 10000) {
DEBUG_WARN(DEBUG_CATEGORY_STREAM, "playback audio late packet pts=%lld expected=%lld", (long long)pts, (long long)a->expectedPts);
return 1;
}
int missing = a->expectedPts < 0 ? 0 : (int)((pts - a->expectedPts + 10000) / 20000);
if (missing > 5) {
DEBUG_WARN(DEBUG_CATEGORY_STREAM, "playback audio discontinuity pts=%lld expected=%lld missing=%d",
(long long)pts, (long long)a->expectedPts, missing); return -2;
}
for (int i = 0; i < missing; i++) if (feedPcm(a, a->expectedPts + i * 20000, nullptr, 0) < 0) return -1;
if (feedPcm(a, pts, packet, (int)length) < 0) return -1;
a->expectedPts = pts + 20000;
return 0;
}
void stream_playback_audio_set_tempo(struct stream_playback_audio* a, int tempo) {
if (!a || tempo < 1000 || tempo > 1100) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio invalid tempo=%d", tempo); return;
}
if (tempo == a->tempo) return;
a->tempo = tempo; a->stretch.setTempo(tempo / 1000.0);
}
int stream_playback_audio_needs_input(const struct stream_playback_audio* a) {
return a && !a->finished && !a->failed && a->stretch.numSamples() < framesPerPacket;
}
/* Ограничение готовой границы применяется до выдачи PCM устройству, сохраняя общий A/V clock. */
int stream_playback_audio_pull(struct stream_playback_audio* a, int16_t* pcm, unsigned frames, int64_t limit) {
if (!a || !pcm || frames > maxFrames || a->failed || limit < 0) {
DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "playback audio invalid pull frames=%u limit=%lld", frames, (long long)limit); return -1;
}
double until = a->producedOutput;
for (unsigned i = 0; i < a->inputCount; i++) {
const auto& span = a->input[(a->inputHead + i) % mapCapacity];
if (span.last <= a->producedOutput) continue;
if (limit >= span.ptsLast) until = span.last;
else if (limit > span.ptsFirst) {
until = span.first + (limit - span.ptsFirst) * (span.last - span.first) / (span.ptsLast - span.ptsFirst); break;
} else break;
}
unsigned available = (unsigned)std::max(0.0, std::floor(until - a->producedOutput));
frames = std::min(frames, std::min(available, a->stretch.numSamples()));
if (!frames) return 0;
unsigned got = a->stretch.receiveSamples(a->samples, frames);
for (unsigned i = 0; i < got * 2; i++) pcm[i] = (int16_t)std::lrint(std::clamp(a->samples[i] * 32767.0f, -32768.0f, 32767.0f));
double position = a->producedOutput, end = position + got;
for (unsigned i = 0; i < a->inputCount && position < end; i++) {
const auto& span = a->input[(a->inputHead + i) % mapCapacity];
if (span.last <= position) continue;
double next = std::min(end, span.last);
double firstPts = mapped(span, position), lastPts = mapped(span, next);
if (!appendOutput(a, {a->written + position - a->producedOutput,
a->written + next - a->producedOutput, firstPts, lastPts})) return -1;
a->lastPts = lastPts; position = next;
}
a->producedOutput = end; a->written += got;
return (int)got;
}
void stream_playback_audio_silence(struct stream_playback_audio* a, unsigned frames) {
if (!a || !frames) return;
if (!appendOutput(a, {(double)a->written, (double)(a->written + frames), a->lastPts, a->lastPts})) return;
a->written += frames;
}
int64_t stream_playback_audio_position(struct stream_playback_audio* a, uint64_t played) {
if (!a) return -1;
if (played > a->written) played = a->written;
while (a->outputCount > 1 && a->output[a->outputHead].last <= played) {
a->outputHead = (a->outputHead + 1) % mapCapacity; a->outputCount--;
}
if (!a->outputCount) return a->startPts;
return (int64_t)std::llround(mapped(a->output[a->outputHead], (double)played));
}
int64_t stream_playback_audio_pending_gain(struct stream_playback_audio* a, uint64_t played) {
if (!a) return 0;
double gain = 0;
for (unsigned i = 0; i < a->outputCount; i++) {
const auto& span = a->output[(a->outputHead + i) % mapCapacity];
double begin = std::max(span.first, (double)played);
if (span.last <= begin) continue;
gain += std::max(0.0, span.ptsLast - mapped(span, begin) - (span.last - begin) * 1000000.0 / rate);
}
for (unsigned i = 0; i < a->inputCount; i++) {
const auto& span = a->input[(a->inputHead + i) % mapCapacity];
double begin = std::max(span.first, a->producedOutput);
if (span.last <= begin) continue;
gain += std::max(0.0, span.ptsLast - mapped(span, begin) - (span.last - begin) * 1000000.0 / rate);
}
return (int64_t)std::ceil(gain);
}
uint64_t stream_playback_audio_written(const struct stream_playback_audio* a) { return a ? a->written : 0; }
void stream_playback_audio_finish(struct stream_playback_audio* a) {
if (!a || a->finished) return;
a->finished = true; a->stretch.flush();
}
int stream_playback_audio_empty(const struct stream_playback_audio* a) {
return !a || (a->finished && a->producedOutput + 0.5 >= a->expectedOutput);
}
}

36
src/stream/stream_playback_audio.h
Unescape View File

@ -0,0 +1,36 @@
/* Короткий stereo Opus/SoundTouch конвейер плеера. Кодированный запас остаётся в RS-кольце.
* Все вызовы одного экземпляра последовательны либо под внешним mutex.
* feed прогревает Opus перед start_pts, затем подаёт PCM с темпом 1.0..1.1 без изменения высоты звука.
* pull записывает соответствие кадров устройства исходному PTS; silence продлевает hardware time, но не media time.
* position/pending_gain используют реально проигранный frame counter, поэтому уже поставленный PCM сохраняет свой темп.
* Последний Opus-пакет/EOF не теряется: finish выталкивает остаток SoundTouch. */
#ifndef STREAM_PLAYBACK_AUDIO_H
#define STREAM_PLAYBACK_AUDIO_H
#include <stddef.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
struct stream_playback_audio;
struct stream_playback_audio *stream_playback_audio_create(int64_t start_pts_us);
void stream_playback_audio_destroy(struct stream_playback_audio *audio);
int stream_playback_audio_feed(struct stream_playback_audio *audio, int64_t pts_us, const uint8_t *packet, size_t length);
void stream_playback_audio_set_tempo(struct stream_playback_audio *audio, int tempo_milli);
int stream_playback_audio_needs_input(const struct stream_playback_audio *audio);
/* frames <=8192; limit_pts_us ограничивает продвижение исходного материала до готовой границы видео/аудио.
* Возвращает число stereo frames; оставшиеся места backend заполняет тишиной и вызывает silence. */
int stream_playback_audio_pull(struct stream_playback_audio *audio, int16_t *pcm, unsigned frames, int64_t limit_pts_us);
void stream_playback_audio_silence(struct stream_playback_audio *audio, unsigned frames);
int64_t stream_playback_audio_position(struct stream_playback_audio *audio, uint64_t played_frames);
int64_t stream_playback_audio_pending_gain(struct stream_playback_audio *audio, uint64_t played_frames);
uint64_t stream_playback_audio_written(const struct stream_playback_audio *audio);
void stream_playback_audio_finish(struct stream_playback_audio *audio);
int stream_playback_audio_empty(const struct stream_playback_audio *audio);
#ifdef __cplusplus
}
#endif
#endif
Write Preview
Loading…
Cancel
Save