jeka
/
utun2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Warm Android Opus preroll and start audio at the first keyframe

master
evgeny 6 days ago
parent
79701feef1
commit
8206e53b5c
2 changed files with 55 additions and 16 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      src/stream/stream_optimizer.c
  2. 67
      tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt

4
src/stream/stream_optimizer.c
Unescape View File

@ -55,6 +55,10 @@ struct stream_optimizer {
/* Точный срок гонки не зависит от фазы общего 100-мс таймера; selector остаётся отложенным. */
static void startup_timeout(void *arg) {
struct stream_optimizer *o = arg; o->startup_timer = NULL;
DEBUG_DEBUG(DEBUG_CATEGORY_STREAM, "stream bootstrap relay deadline group=%llx source=%llx epoch=%llx elapsed_ms=%llu delay_ms=%u",
(unsigned long long)o->descriptor.group_id, (unsigned long long)o->descriptor.source_node_id,
(unsigned long long)o->descriptor.source_epoch, (unsigned long long)(get_time_tb() / 10 - o->started_ms),
STREAM_BOOTSTRAP_RELAY_MS);
stream_network_wake(stream_network_get(o->inst));
}

67
tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt
Unescape View File

@ -66,6 +66,11 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
var endDrainUntil = 0L
var headEpoch = 0L
var previousRawHead = 0L
var prefillUntil = 0L
var firstAudioSkipFrames = 0
var prerollFrames = 0L
var warmAudioPts = -1L
var connectDeadline = System.nanoTime() + 10_000_000_000
fun resetVideo() {
try { decoder?.stop() }
@ -83,6 +88,22 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
track = null
if (opus != 0L) NativeLib.streamOpusClose(opus)
opus = 0; audioBase = -1; written = 0; lastHead = -1; headEpoch = 0; previousRawHead = 0; audios.clear()
firstAudioSkipFrames = 0; prerollFrames = 0; warmAudioPts = -1
}
fun resetTimeline() {
resetAudio(); resetVideo(); videoBase = -1; prefillUntil = 0
connectDeadline = System.nanoTime() + 10_000_000_000
}
// Отброшенный звук всё равно прогревает Opus; остающиеся пакеты будут декодированы один раз при воспроизведении.
fun warmAudio(frame: Frame) {
if (warmAudioPts >= 0 && frame.pts != warmAudioPts) {
LogManager.addLog("WARN", "stream", "preroll audio discontinuity id=$id expected=$warmAudioPts pts=${frame.pts}")
if (opus != 0L) NativeLib.streamOpusClose(opus)
opus = 0
}
if (opus == 0L) { opus = NativeLib.streamOpusCreate(false); check(opus != 0L) }
check(NativeLib.streamOpusDecode(opus, frame.data) != null) { "Preroll Opus decode failed" }
warmAudioPts = frame.pts + 20000; prerollFrames += 960
}
// PlaybackHeadPosition — unsigned 32 bit: при 48kHz оборачивается примерно через 25 часов.
fun playedFrames(t: AudioTrack): Long {
@ -111,9 +132,9 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
}
return if (videoBase >= 0) videoBase + (now - videoStartNs) / 1000 else 0
}
fun writePcm(t: AudioTrack, pcm: ShortArray) {
fun writePcm(t: AudioTrack, pcm: ShortArray, initialOffset: Int = 0) {
val deadline = System.nanoTime() + 100_000_000
var offset = 0
var offset = initialOffset
while (running && offset < pcm.size) {
val count = t.write(pcm, offset, pcm.size - offset, AudioTrack.WRITE_NON_BLOCKING)
check(count >= 0) { "AudioTrack write=$count" }
@ -129,9 +150,7 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
}
try {
check(NativeLib.streamOpen(id, group, source, epoch = epoch)) { "Viewer open refused" }
val openDeadline = System.nanoTime() + 10_000_000_000
var opened = false
var prefillUntil = 0L
while (running && surface.isValid) {
if (!NativeLib.isReady()) error("Core stopped")
repeat(64) {
@ -154,7 +173,6 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
val display = StreamVideoDisplay.forFrame(broadcastRotation, camera, displayDegrees)
val rotation = display.rotation
val frame = Frame(pts, key, width, height, rotation, display.top, revision, ByteArray(size).also { b.get(it) })
if (prefillUntil == 0L) prefillUntil = System.nanoTime() + 150_000_000
if (kind == 1) {
if (revision != videoRevision) {
check(revision > videoRevision) { "Video revision regressed" }
@ -174,20 +192,32 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
mutableAspect.value = if (rotation % 180 == 0) width.toFloat() / height else height.toFloat() / width
mutableTop.value = frame.top
}
if (videoBase < 0) {
while (audios.isNotEmpty() && audios.first.pts + 20000 <= pts) warmAudio(audios.removeFirst())
if (audios.isNotEmpty() && audios.first.pts < pts)
firstAudioSkipFrames = ((pts - audios.first.pts) * 48000 + 999999).div(1000000).toInt()
prefillUntil = System.nanoTime() + 150_000_000
videoBase = pts; videoStartNs = prefillUntil
LogManager.addLog("INFO", "stream", "viewer playback start id=$id pts=$pts prefillMs=150 " +
"prerollFrames=$prerollFrames skipFrames=$firstAudioSkipFrames audioQ=${audios.size}")
}
videos.addLast(frame); videoBytes += size
}
} else {
if (localPreview != null) return@repeat
if (audios.size >= 16) {
LogManager.addLog("WARN", "stream", "audio queue overflow id=$id; reset timeline")
resetAudio(); resetVideo()
if (videoBase < 0) warmAudio(audios.removeFirst())
else {
LogManager.addLog("WARN", "stream", "audio queue overflow id=$id; reset timeline")
resetTimeline()
}
}
audios.addLast(frame)
}
}
3 -> { LogManager.addLog("INFO", "stream", "viewer GAP id=$id"); resetVideo(); resetAudio(); videoBase = -1; prefillUntil = 0 }
3 -> { LogManager.addLog("INFO", "stream", "viewer GAP id=$id"); resetTimeline() }
4 -> opened = true
5 -> ended = true
5 -> { ended = true; if (videoBase < 0) audios.clear() }
6 -> {
mutableStatus.value = when (flags) {
1 -> R.string.stream_group_closed
@ -203,18 +233,23 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
}
}
}
if (!opened && System.nanoTime() > openDeadline) error("Viewer open timeout")
if (!ended && (!opened || videoBase < 0) && System.nanoTime() > connectDeadline) error("Viewer open timeout")
val now = System.nanoTime()
if (prefillUntil != 0L && now >= prefillUntil) {
if (audios.isNotEmpty() && track == null) {
opus = NativeLib.streamOpusCreate(false); check(opus != 0L)
if (warmAudioPts >= 0 && audios.first.pts != warmAudioPts) {
LogManager.addLog("WARN", "stream", "audio startup discontinuity id=$id expected=$warmAudioPts pts=${audios.first.pts}")
if (opus != 0L) NativeLib.streamOpusClose(opus)
opus = 0
}
if (opus == 0L) { opus = NativeLib.streamOpusCreate(false); check(opus != 0L) }
track = AudioTrack.Builder().setAudioAttributes(AudioAttributes.Builder().setUsage(AudioAttributes.USAGE_MEDIA)
.setContentType(AudioAttributes.CONTENT_TYPE_MOVIE).build()).setAudioFormat(AudioFormat.Builder()
.setSampleRate(48000).setChannelMask(AudioFormat.CHANNEL_OUT_STEREO).setEncoding(AudioFormat.ENCODING_PCM_16BIT).build())
.setBufferSizeInBytes(AudioTrack.getMinBufferSize(48000, AudioFormat.CHANNEL_OUT_STEREO, AudioFormat.ENCODING_PCM_16BIT)
.coerceAtLeast(19200)).setTransferMode(AudioTrack.MODE_STREAM).build()
check(track!!.state == AudioTrack.STATE_INITIALIZED)
audioBase = audios.first.pts
audioBase = audios.first.pts + firstAudioSkipFrames * 1_000_000L / 48000
track!!.play()
LogManager.addLog("INFO", "stream", "audio playback starts id=$id PTS=$audioBase rate=48000 channels=${track!!.channelCount}")
}
@ -223,21 +258,21 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
val head = playedFrames(t)
if (written - head < 3840) {
val f = audios.removeFirst()
val missing = ((f.pts - audioBase - written * 1_000_000 / 48000).toDouble() / 20000).roundToInt()
val pts = f.pts + firstAudioSkipFrames * 1_000_000L / 48000
val missing = ((pts - audioBase - written * 1_000_000 / 48000).toDouble() / 20000).roundToInt()
if (missing > 5 || missing < -5) {
LogManager.addLog("WARN", "stream", "audio discontinuity id=$id missing=$missing packets")
resetAudio(); audios.addFirst(f); resetVideo()
resetTimeline(); audios.addFirst(f)
} else if (missing >= 0) {
repeat(missing) {
val conceal = NativeLib.streamOpusDecode(opus, byteArrayOf()) ?: error("Opus PLC failed")
writePcm(t, conceal)
}
val pcm = NativeLib.streamOpusDecode(opus, f.data) ?: error("Opus decode failed")
writePcm(t, pcm)
writePcm(t, pcm, firstAudioSkipFrames * 2); firstAudioSkipFrames = 0
} else LogManager.addLog("DEBUG", "stream", "late audio discarded id=$id pts=${f.pts}")
}
}
if (videoBase < 0 && videos.isNotEmpty()) { videoBase = videos.first.pts; videoStartNs = now }
val clock = currentPts(now)
repeat(8) {
if (videos.isEmpty() || videos.first.pts > clock + 80000) return@repeat

Write Preview
Loading…
Cancel
Save