From 8206e53b5c7aec593eda2b2f100116de1072dfee Mon Sep 17 00:00:00 2001 From: evgeny Date: Tue, 13 Oct 2026 16:30:41 +0200 Subject: [PATCH] Warm Android Opus preroll and start audio at the first keyframe --- src/stream/stream_optimizer.c | 4 ++ .../com/utun/chat/data/StreamPlayerEngine.kt | 67 ++++++++++++++----- 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/src/stream/stream_optimizer.c b/src/stream/stream_optimizer.c index 22e555ed..a10e6c80 100644 --- a/src/stream/stream_optimizer.c +++ b/src/stream/stream_optimizer.c @@ -55,6 +55,10 @@ struct stream_optimizer { /* Точный срок гонки не зависит от фазы общего 100-мс таймера; selector остаётся отложенным. */ static void startup_timeout(void *arg) { struct stream_optimizer *o = arg; o->startup_timer = NULL; + DEBUG_DEBUG(DEBUG_CATEGORY_STREAM, "stream bootstrap relay deadline group=%llx source=%llx epoch=%llx elapsed_ms=%llu delay_ms=%u", + (unsigned long long)o->descriptor.group_id, (unsigned long long)o->descriptor.source_node_id, + (unsigned long long)o->descriptor.source_epoch, (unsigned long long)(get_time_tb() / 10 - o->started_ms), + STREAM_BOOTSTRAP_RELAY_MS); stream_network_wake(stream_network_get(o->inst)); } diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt index 56062a07..64cbc5e0 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt @@ -66,6 +66,11 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv var endDrainUntil = 0L var headEpoch = 0L var previousRawHead = 0L + var prefillUntil = 0L + var firstAudioSkipFrames = 0 + var prerollFrames = 0L + var warmAudioPts = -1L + var connectDeadline = System.nanoTime() + 10_000_000_000 fun resetVideo() { try { decoder?.stop() } @@ -83,6 +88,22 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv track = null if (opus != 0L) NativeLib.streamOpusClose(opus) opus = 0; audioBase = -1; written = 0; lastHead = -1; headEpoch = 0; previousRawHead = 0; audios.clear() + firstAudioSkipFrames = 0; prerollFrames = 0; warmAudioPts = -1 + } + fun resetTimeline() { + resetAudio(); resetVideo(); videoBase = -1; prefillUntil = 0 + connectDeadline = System.nanoTime() + 10_000_000_000 + } + // Отброшенный звук всё равно прогревает Opus; остающиеся пакеты будут декодированы один раз при воспроизведении. + fun warmAudio(frame: Frame) { + if (warmAudioPts >= 0 && frame.pts != warmAudioPts) { + LogManager.addLog("WARN", "stream", "preroll audio discontinuity id=$id expected=$warmAudioPts pts=${frame.pts}") + if (opus != 0L) NativeLib.streamOpusClose(opus) + opus = 0 + } + if (opus == 0L) { opus = NativeLib.streamOpusCreate(false); check(opus != 0L) } + check(NativeLib.streamOpusDecode(opus, frame.data) != null) { "Preroll Opus decode failed" } + warmAudioPts = frame.pts + 20000; prerollFrames += 960 } // PlaybackHeadPosition — unsigned 32 bit: при 48kHz оборачивается примерно через 25 часов. fun playedFrames(t: AudioTrack): Long { @@ -111,9 +132,9 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv } return if (videoBase >= 0) videoBase + (now - videoStartNs) / 1000 else 0 } - fun writePcm(t: AudioTrack, pcm: ShortArray) { + fun writePcm(t: AudioTrack, pcm: ShortArray, initialOffset: Int = 0) { val deadline = System.nanoTime() + 100_000_000 - var offset = 0 + var offset = initialOffset while (running && offset < pcm.size) { val count = t.write(pcm, offset, pcm.size - offset, AudioTrack.WRITE_NON_BLOCKING) check(count >= 0) { "AudioTrack write=$count" } @@ -129,9 +150,7 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv } try { check(NativeLib.streamOpen(id, group, source, epoch = epoch)) { "Viewer open refused" } - val openDeadline = System.nanoTime() + 10_000_000_000 var opened = false - var prefillUntil = 0L while (running && surface.isValid) { if (!NativeLib.isReady()) error("Core stopped") repeat(64) { @@ -154,7 +173,6 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv val display = StreamVideoDisplay.forFrame(broadcastRotation, camera, displayDegrees) val rotation = display.rotation val frame = Frame(pts, key, width, height, rotation, display.top, revision, ByteArray(size).also { b.get(it) }) - if (prefillUntil == 0L) prefillUntil = System.nanoTime() + 150_000_000 if (kind == 1) { if (revision != videoRevision) { check(revision > videoRevision) { "Video revision regressed" } @@ -174,20 +192,32 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv mutableAspect.value = if (rotation % 180 == 0) width.toFloat() / height else height.toFloat() / width mutableTop.value = frame.top } + if (videoBase < 0) { + while (audios.isNotEmpty() && audios.first.pts + 20000 <= pts) warmAudio(audios.removeFirst()) + if (audios.isNotEmpty() && audios.first.pts < pts) + firstAudioSkipFrames = ((pts - audios.first.pts) * 48000 + 999999).div(1000000).toInt() + prefillUntil = System.nanoTime() + 150_000_000 + videoBase = pts; videoStartNs = prefillUntil + LogManager.addLog("INFO", "stream", "viewer playback start id=$id pts=$pts prefillMs=150 " + + "prerollFrames=$prerollFrames skipFrames=$firstAudioSkipFrames audioQ=${audios.size}") + } videos.addLast(frame); videoBytes += size } } else { if (localPreview != null) return@repeat if (audios.size >= 16) { - LogManager.addLog("WARN", "stream", "audio queue overflow id=$id; reset timeline") - resetAudio(); resetVideo() + if (videoBase < 0) warmAudio(audios.removeFirst()) + else { + LogManager.addLog("WARN", "stream", "audio queue overflow id=$id; reset timeline") + resetTimeline() + } } audios.addLast(frame) } } - 3 -> { LogManager.addLog("INFO", "stream", "viewer GAP id=$id"); resetVideo(); resetAudio(); videoBase = -1; prefillUntil = 0 } + 3 -> { LogManager.addLog("INFO", "stream", "viewer GAP id=$id"); resetTimeline() } 4 -> opened = true - 5 -> ended = true + 5 -> { ended = true; if (videoBase < 0) audios.clear() } 6 -> { mutableStatus.value = when (flags) { 1 -> R.string.stream_group_closed @@ -203,18 +233,23 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv } } } - if (!opened && System.nanoTime() > openDeadline) error("Viewer open timeout") + if (!ended && (!opened || videoBase < 0) && System.nanoTime() > connectDeadline) error("Viewer open timeout") val now = System.nanoTime() if (prefillUntil != 0L && now >= prefillUntil) { if (audios.isNotEmpty() && track == null) { - opus = NativeLib.streamOpusCreate(false); check(opus != 0L) + if (warmAudioPts >= 0 && audios.first.pts != warmAudioPts) { + LogManager.addLog("WARN", "stream", "audio startup discontinuity id=$id expected=$warmAudioPts pts=${audios.first.pts}") + if (opus != 0L) NativeLib.streamOpusClose(opus) + opus = 0 + } + if (opus == 0L) { opus = NativeLib.streamOpusCreate(false); check(opus != 0L) } track = AudioTrack.Builder().setAudioAttributes(AudioAttributes.Builder().setUsage(AudioAttributes.USAGE_MEDIA) .setContentType(AudioAttributes.CONTENT_TYPE_MOVIE).build()).setAudioFormat(AudioFormat.Builder() .setSampleRate(48000).setChannelMask(AudioFormat.CHANNEL_OUT_STEREO).setEncoding(AudioFormat.ENCODING_PCM_16BIT).build()) .setBufferSizeInBytes(AudioTrack.getMinBufferSize(48000, AudioFormat.CHANNEL_OUT_STEREO, AudioFormat.ENCODING_PCM_16BIT) .coerceAtLeast(19200)).setTransferMode(AudioTrack.MODE_STREAM).build() check(track!!.state == AudioTrack.STATE_INITIALIZED) - audioBase = audios.first.pts + audioBase = audios.first.pts + firstAudioSkipFrames * 1_000_000L / 48000 track!!.play() LogManager.addLog("INFO", "stream", "audio playback starts id=$id PTS=$audioBase rate=48000 channels=${track!!.channelCount}") } @@ -223,21 +258,21 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv val head = playedFrames(t) if (written - head < 3840) { val f = audios.removeFirst() - val missing = ((f.pts - audioBase - written * 1_000_000 / 48000).toDouble() / 20000).roundToInt() + val pts = f.pts + firstAudioSkipFrames * 1_000_000L / 48000 + val missing = ((pts - audioBase - written * 1_000_000 / 48000).toDouble() / 20000).roundToInt() if (missing > 5 || missing < -5) { LogManager.addLog("WARN", "stream", "audio discontinuity id=$id missing=$missing packets") - resetAudio(); audios.addFirst(f); resetVideo() + resetTimeline(); audios.addFirst(f) } else if (missing >= 0) { repeat(missing) { val conceal = NativeLib.streamOpusDecode(opus, byteArrayOf()) ?: error("Opus PLC failed") writePcm(t, conceal) } val pcm = NativeLib.streamOpusDecode(opus, f.data) ?: error("Opus decode failed") - writePcm(t, pcm) + writePcm(t, pcm, firstAudioSkipFrames * 2); firstAudioSkipFrames = 0 } else LogManager.addLog("DEBUG", "stream", "late audio discarded id=$id pts=${f.pts}") } } - if (videoBase < 0 && videos.isNotEmpty()) { videoBase = videos.first.pts; videoStartNs = now } val clock = currentPts(now) repeat(8) { if (videos.isEmpty() || videos.first.pts > clock + 80000) return@repeat