Warm Android Opus preroll and start audio at the first keyframe
This commit is contained in:
evgeny committed
1 parent
79701feef1
commit
8206e53b5c
2 files changed
+55
-16
No files matched your search
@@ -66,6 +66,11 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
|
||||
var endDrainUntil = 0L
|
||||
var headEpoch = 0L
|
||||
var previousRawHead = 0L
|
||||
var prefillUntil = 0L
|
||||
var firstAudioSkipFrames = 0
|
||||
var prerollFrames = 0L
|
||||
var warmAudioPts = -1L
|
||||
var connectDeadline = System.nanoTime() + 10_000_000_000
|
||||
|
||||
fun resetVideo() {
|
||||
try { decoder?.stop() }
|
||||
@@ -83,6 +88,22 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
|
||||
track = null
|
||||
if (opus != 0L) NativeLib.streamOpusClose(opus)
|
||||
opus = 0; audioBase = -1; written = 0; lastHead = -1; headEpoch = 0; previousRawHead = 0; audios.clear()
|
||||
firstAudioSkipFrames = 0; prerollFrames = 0; warmAudioPts = -1
|
||||
}
|
||||
fun resetTimeline() {
|
||||
resetAudio(); resetVideo(); videoBase = -1; prefillUntil = 0
|
||||
connectDeadline = System.nanoTime() + 10_000_000_000
|
||||
}
|
||||
// Отброшенный звук всё равно прогревает Opus; остающиеся пакеты будут декодированы один раз при воспроизведении.
|
||||
fun warmAudio(frame: Frame) {
|
||||
if (warmAudioPts >= 0 && frame.pts != warmAudioPts) {
|
||||
LogManager.addLog("WARN", "stream", "preroll audio discontinuity id=$id expected=$warmAudioPts pts=${frame.pts}")
|
||||
if (opus != 0L) NativeLib.streamOpusClose(opus)
|
||||
opus = 0
|
||||
}
|
||||
if (opus == 0L) { opus = NativeLib.streamOpusCreate(false); check(opus != 0L) }
|
||||
check(NativeLib.streamOpusDecode(opus, frame.data) != null) { "Preroll Opus decode failed" }
|
||||
warmAudioPts = frame.pts + 20000; prerollFrames += 960
|
||||
}
|
||||
// PlaybackHeadPosition — unsigned 32 bit: при 48kHz оборачивается примерно через 25 часов.
|
||||
fun playedFrames(t: AudioTrack): Long {
|
||||
@@ -111,9 +132,9 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
|
||||
}
|
||||
return if (videoBase >= 0) videoBase + (now - videoStartNs) / 1000 else 0
|
||||
}
|
||||
fun writePcm(t: AudioTrack, pcm: ShortArray) {
|
||||
fun writePcm(t: AudioTrack, pcm: ShortArray, initialOffset: Int = 0) {
|
||||
val deadline = System.nanoTime() + 100_000_000
|
||||
var offset = 0
|
||||
var offset = initialOffset
|
||||
while (running && offset < pcm.size) {
|
||||
val count = t.write(pcm, offset, pcm.size - offset, AudioTrack.WRITE_NON_BLOCKING)
|
||||
check(count >= 0) { "AudioTrack write=$count" }
|
||||
@@ -129,9 +150,7 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
|
||||
}
|
||||
try {
|
||||
check(NativeLib.streamOpen(id, group, source, epoch = epoch)) { "Viewer open refused" }
|
||||
val openDeadline = System.nanoTime() + 10_000_000_000
|
||||
var opened = false
|
||||
var prefillUntil = 0L
|
||||
while (running && surface.isValid) {
|
||||
if (!NativeLib.isReady()) error("Core stopped")
|
||||
repeat(64) {
|
||||
@@ -154,7 +173,6 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
|
||||
val display = StreamVideoDisplay.forFrame(broadcastRotation, camera, displayDegrees)
|
||||
val rotation = display.rotation
|
||||
val frame = Frame(pts, key, width, height, rotation, display.top, revision, ByteArray(size).also { b.get(it) })
|
||||
if (prefillUntil == 0L) prefillUntil = System.nanoTime() + 150_000_000
|
||||
if (kind == 1) {
|
||||
if (revision != videoRevision) {
|
||||
check(revision > videoRevision) { "Video revision regressed" }
|
||||
@@ -174,20 +192,32 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
|
||||
mutableAspect.value = if (rotation % 180 == 0) width.toFloat() / height else height.toFloat() / width
|
||||
mutableTop.value = frame.top
|
||||
}
|
||||
if (videoBase < 0) {
|
||||
while (audios.isNotEmpty() && audios.first.pts + 20000 <= pts) warmAudio(audios.removeFirst())
|
||||
if (audios.isNotEmpty() && audios.first.pts < pts)
|
||||
firstAudioSkipFrames = ((pts - audios.first.pts) * 48000 + 999999).div(1000000).toInt()
|
||||
prefillUntil = System.nanoTime() + 150_000_000
|
||||
videoBase = pts; videoStartNs = prefillUntil
|
||||
LogManager.addLog("INFO", "stream", "viewer playback start id=$id pts=$pts prefillMs=150 " +
|
||||
"prerollFrames=$prerollFrames skipFrames=$firstAudioSkipFrames audioQ=${audios.size}")
|
||||
}
|
||||
videos.addLast(frame); videoBytes += size
|
||||
}
|
||||
} else {
|
||||
if (localPreview != null) return@repeat
|
||||
if (audios.size >= 16) {
|
||||
LogManager.addLog("WARN", "stream", "audio queue overflow id=$id; reset timeline")
|
||||
resetAudio(); resetVideo()
|
||||
if (videoBase < 0) warmAudio(audios.removeFirst())
|
||||
else {
|
||||
LogManager.addLog("WARN", "stream", "audio queue overflow id=$id; reset timeline")
|
||||
resetTimeline()
|
||||
}
|
||||
}
|
||||
audios.addLast(frame)
|
||||
}
|
||||
}
|
||||
3 -> { LogManager.addLog("INFO", "stream", "viewer GAP id=$id"); resetVideo(); resetAudio(); videoBase = -1; prefillUntil = 0 }
|
||||
3 -> { LogManager.addLog("INFO", "stream", "viewer GAP id=$id"); resetTimeline() }
|
||||
4 -> opened = true
|
||||
5 -> ended = true
|
||||
5 -> { ended = true; if (videoBase < 0) audios.clear() }
|
||||
6 -> {
|
||||
mutableStatus.value = when (flags) {
|
||||
1 -> R.string.stream_group_closed
|
||||
@@ -203,18 +233,23 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!opened && System.nanoTime() > openDeadline) error("Viewer open timeout")
|
||||
if (!ended && (!opened || videoBase < 0) && System.nanoTime() > connectDeadline) error("Viewer open timeout")
|
||||
val now = System.nanoTime()
|
||||
if (prefillUntil != 0L && now >= prefillUntil) {
|
||||
if (audios.isNotEmpty() && track == null) {
|
||||
opus = NativeLib.streamOpusCreate(false); check(opus != 0L)
|
||||
if (warmAudioPts >= 0 && audios.first.pts != warmAudioPts) {
|
||||
LogManager.addLog("WARN", "stream", "audio startup discontinuity id=$id expected=$warmAudioPts pts=${audios.first.pts}")
|
||||
if (opus != 0L) NativeLib.streamOpusClose(opus)
|
||||
opus = 0
|
||||
}
|
||||
if (opus == 0L) { opus = NativeLib.streamOpusCreate(false); check(opus != 0L) }
|
||||
track = AudioTrack.Builder().setAudioAttributes(AudioAttributes.Builder().setUsage(AudioAttributes.USAGE_MEDIA)
|
||||
.setContentType(AudioAttributes.CONTENT_TYPE_MOVIE).build()).setAudioFormat(AudioFormat.Builder()
|
||||
.setSampleRate(48000).setChannelMask(AudioFormat.CHANNEL_OUT_STEREO).setEncoding(AudioFormat.ENCODING_PCM_16BIT).build())
|
||||
.setBufferSizeInBytes(AudioTrack.getMinBufferSize(48000, AudioFormat.CHANNEL_OUT_STEREO, AudioFormat.ENCODING_PCM_16BIT)
|
||||
.coerceAtLeast(19200)).setTransferMode(AudioTrack.MODE_STREAM).build()
|
||||
check(track!!.state == AudioTrack.STATE_INITIALIZED)
|
||||
audioBase = audios.first.pts
|
||||
audioBase = audios.first.pts + firstAudioSkipFrames * 1_000_000L / 48000
|
||||
track!!.play()
|
||||
LogManager.addLog("INFO", "stream", "audio playback starts id=$id PTS=$audioBase rate=48000 channels=${track!!.channelCount}")
|
||||
}
|
||||
@@ -223,21 +258,21 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv
|
||||
val head = playedFrames(t)
|
||||
if (written - head < 3840) {
|
||||
val f = audios.removeFirst()
|
||||
val missing = ((f.pts - audioBase - written * 1_000_000 / 48000).toDouble() / 20000).roundToInt()
|
||||
val pts = f.pts + firstAudioSkipFrames * 1_000_000L / 48000
|
||||
val missing = ((pts - audioBase - written * 1_000_000 / 48000).toDouble() / 20000).roundToInt()
|
||||
if (missing > 5 || missing < -5) {
|
||||
LogManager.addLog("WARN", "stream", "audio discontinuity id=$id missing=$missing packets")
|
||||
resetAudio(); audios.addFirst(f); resetVideo()
|
||||
resetTimeline(); audios.addFirst(f)
|
||||
} else if (missing >= 0) {
|
||||
repeat(missing) {
|
||||
val conceal = NativeLib.streamOpusDecode(opus, byteArrayOf()) ?: error("Opus PLC failed")
|
||||
writePcm(t, conceal)
|
||||
}
|
||||
val pcm = NativeLib.streamOpusDecode(opus, f.data) ?: error("Opus decode failed")
|
||||
writePcm(t, pcm)
|
||||
writePcm(t, pcm, firstAudioSkipFrames * 2); firstAudioSkipFrames = 0
|
||||
} else LogManager.addLog("DEBUG", "stream", "late audio discarded id=$id pts=${f.pts}")
|
||||
}
|
||||
}
|
||||
if (videoBase < 0 && videos.isNotEmpty()) { videoBase = videos.first.pts; videoStartNs = now }
|
||||
val clock = currentPts(now)
|
||||
repeat(8) {
|
||||
if (videos.isEmpty() || videos.first.pts > clock + 80000) return@repeat
|
||||
|
||||
Reference in new issue
Block a user