Use Android low-latency output and presentation timestamps for call clocks
This commit is contained in:
evgeny committed
1 parent
b6b0726bfe
commit
cb9852e8d5
11 files changed
+142
-24
No files matched your search
@@ -25,7 +25,7 @@ import com.utun.chat.ChatApplication
|
||||
* - play-поток: NativeLib.callAudioPull (C: джиттер-буфер → декод → SoundTouch) → AudioTrack.write
|
||||
*
|
||||
* Воспроизведение тянет PCM из единого голосового стека в C (call_audio): адаптивный
|
||||
* джиттер-буфер (кодированные кадры) + time-stretch держат задержку ~60мс и догоняют
|
||||
* джиттер-буфер (кодированные кадры) с минимальным запасом 20мс + CallDelay догоняют
|
||||
* после сетевых бёрстов. На Kotlin-стороне своего джиттера больше нет.
|
||||
*/
|
||||
class CallAudioEngine {
|
||||
@@ -71,13 +71,19 @@ class CallAudioEngine {
|
||||
@Volatile private var playFramesPlayed = 0L
|
||||
@Volatile private var capFramesRead = 0L
|
||||
private var playbackBuffer: CallPlaybackBuffer? = null // под trackLock
|
||||
@Volatile private var routeGeneration = 0L
|
||||
private data class OutputSample(val atUs: Long, val generation: Long, val latency: CallOutputLatency)
|
||||
@Volatile private var outputSample: OutputSample? = null
|
||||
@Volatile private var startElapsedMs = 0L
|
||||
@Volatile private var captureElapsedMs = 0L
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
private val trackRouteListener = AudioRouting.OnRoutingChangedListener { output ->
|
||||
synchronized(trackLock) {
|
||||
val track = audioTrack
|
||||
if (active && track != null && output === track) configurePlaybackBuffer(track, "route")
|
||||
if (active && track != null && output === track) {
|
||||
routeGeneration++
|
||||
configurePlaybackBuffer(track, "route")
|
||||
}
|
||||
}
|
||||
}
|
||||
private val aecDelayRunnable = Runnable { measureAecDelay() }
|
||||
@@ -86,6 +92,12 @@ class CallAudioEngine {
|
||||
|
||||
fun isActive(): Boolean = active
|
||||
|
||||
fun outputLatency(id: Long): CallOutputLatency {
|
||||
val sample = outputSample
|
||||
return if (active && id == callId && sample != null && sample.generation == routeGeneration &&
|
||||
System.nanoTime() / 1000 - sample.atUs in 0..500_000) sample.latency else CallOutputLatency()
|
||||
}
|
||||
|
||||
/** Позднее создание Telecom Connection: сначала снимаем собственную заявку на audio mode. */
|
||||
fun bindTelecom(connection: UtunConnection) {
|
||||
if (!active || router.telecom === connection) return
|
||||
@@ -128,6 +140,7 @@ class CallAudioEngine {
|
||||
val nativeMs = SystemClock.elapsedRealtime() - startElapsedMs
|
||||
|
||||
callId = id
|
||||
routeGeneration = 0; outputSample = null
|
||||
router.bindTelecom(TelecomCallManager.connectionFor(id))
|
||||
muted = false
|
||||
failed.set(false)
|
||||
@@ -194,12 +207,13 @@ class CallAudioEngine {
|
||||
val minQueryMs = SystemClock.elapsedRealtime() - createdAt
|
||||
val trackBuf = (trackMin * 2).coerceAtLeast(FRAME_SAMPLES * 4)
|
||||
val constructorAt = SystemClock.elapsedRealtime()
|
||||
val track = AudioTrack(
|
||||
AudioAttributes.Builder().setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION)
|
||||
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH).build(),
|
||||
AudioFormat.Builder().setSampleRate(SAMPLE_RATE).setEncoding(AudioFormat.ENCODING_PCM_16BIT)
|
||||
.setChannelMask(AudioFormat.CHANNEL_OUT_MONO).build(),
|
||||
trackBuf, AudioTrack.MODE_STREAM, AudioManager.AUDIO_SESSION_ID_GENERATE)
|
||||
val track = AudioTrack.Builder()
|
||||
.setAudioAttributes(AudioAttributes.Builder().setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION)
|
||||
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH).build())
|
||||
.setAudioFormat(AudioFormat.Builder().setSampleRate(SAMPLE_RATE).setEncoding(AudioFormat.ENCODING_PCM_16BIT)
|
||||
.setChannelMask(AudioFormat.CHANNEL_OUT_MONO).build())
|
||||
.setBufferSizeInBytes(trackBuf).setTransferMode(AudioTrack.MODE_STREAM)
|
||||
.setPerformanceMode(AudioTrack.PERFORMANCE_MODE_LOW_LATENCY).build()
|
||||
val constructorMs = SystemClock.elapsedRealtime() - constructorAt
|
||||
audioTrack = track
|
||||
if (!active) return@runWorker
|
||||
@@ -489,7 +503,7 @@ class CallAudioEngine {
|
||||
val stamp = AudioTimestamp()
|
||||
while (!Thread.currentThread().isInterrupted) {
|
||||
if (active) adjustPlaybackBuffer(track)
|
||||
if (trace.enabled && active) trace.device = samplePlaybackDevice(track, stamp)
|
||||
if (active) trace.device = samplePlaybackDevice(track, stamp)
|
||||
trace.flush()
|
||||
Thread.sleep(200)
|
||||
}
|
||||
@@ -518,6 +532,7 @@ class CallAudioEngine {
|
||||
/** Только reporter: Binder/внутренние locks AudioTrack не должны вызываться здесь из playback. */
|
||||
private fun samplePlaybackDevice(track: AudioTrack, stamp: AudioTimestamp): CallAudioTrace.DeviceSample {
|
||||
val beginNs = System.nanoTime()
|
||||
val generation = routeGeneration
|
||||
val stampOk = track.getTimestamp(stamp)
|
||||
val stampEndNs = System.nanoTime()
|
||||
val route = track.routedDevice
|
||||
@@ -525,12 +540,13 @@ class CallAudioEngine {
|
||||
val underruns = track.underrunCount.toLong()
|
||||
val bufferFrames = track.bufferSizeInFrames.toLong()
|
||||
val startFrames = if (Build.VERSION.SDK_INT >= 31) track.startThresholdInFrames.toLong() else bufferFrames
|
||||
val performanceMode = track.performanceMode
|
||||
val endNs = System.nanoTime()
|
||||
return CallAudioTrace.DeviceSample(
|
||||
route?.id?.toLong() ?: -1, route?.type?.toLong() ?: -1, underruns, bufferFrames, startFrames,
|
||||
if (stampOk) stamp.framePosition else -1, if (stampOk) stamp.nanoTime else -1, stampOk,
|
||||
beginNs / 1000, endNs / 1000, (stampEndNs - beginNs) / 1000,
|
||||
(routeEndNs - stampEndNs) / 1000, (endNs - routeEndNs) / 1000)
|
||||
(routeEndNs - stampEndNs) / 1000, (endNs - routeEndNs) / 1000, generation, performanceMode)
|
||||
}
|
||||
|
||||
private fun playLoop(trace: CallAudioTrace) {
|
||||
@@ -546,6 +562,9 @@ class CallAudioEngine {
|
||||
}
|
||||
val epoch = snapshot[0]
|
||||
val head = CallPlaybackHead()
|
||||
val playbackClock = CallPlaybackClock()
|
||||
var clockMode = ""
|
||||
var outputUs = -1L
|
||||
var clockFailed = false
|
||||
val diagnostic = LongArray(8)
|
||||
val row = LongArray(CallAudioTrace.WIDTH)
|
||||
@@ -590,6 +609,7 @@ class CallAudioEngine {
|
||||
return
|
||||
}
|
||||
Thread.sleep(2)
|
||||
continue
|
||||
} else {
|
||||
blockedAt = 0L
|
||||
offset += written
|
||||
@@ -604,13 +624,26 @@ class CallAudioEngine {
|
||||
observedPlayed = played
|
||||
playFramesPlayed = played
|
||||
operationNs = System.nanoTime()
|
||||
NativeLib.callAudioProgress(callId, epoch, played, nowUs)
|
||||
val device = trace.device
|
||||
val anchor = playbackClock.update(playFramesWritten, played, nowUs, routeGeneration,
|
||||
device.generation, device.stampFrame, device.stampNs / 1000)
|
||||
outputUs = if (anchor?.measured == true) (anchor.atUs - nowUs).coerceAtLeast(0) else -1
|
||||
outputSample = OutputSample(nowUs, routeGeneration, CallOutputLatency(
|
||||
((playFramesWritten - played) * 1000 / SAMPLE_RATE).toInt(),
|
||||
if (outputUs >= 0) (outputUs / 1000).toInt() else -1))
|
||||
if (anchor != null) NativeLib.callAudioProgress(callId, epoch, anchor.position, anchor.atUs, nowUs, anchor.measured)
|
||||
progressCallNs += System.nanoTime() - operationNs
|
||||
val mode = if (anchor == null) "unavailable" else if (anchor.measured) "timestamp" else "head-estimate"
|
||||
if (mode != clockMode) {
|
||||
LogManager.addLog(if (mode == "unavailable") "WARN" else "INFO", "CallAudio",
|
||||
"playback clock id=$callId mode=$mode routeGeneration=$routeGeneration outputUs=$outputUs")
|
||||
clockMode = mode
|
||||
}
|
||||
} else if (!clockFailed) {
|
||||
clockFailed = true
|
||||
playFramesPlayed = -1
|
||||
// Отрицательная позиция немедленно инвалидирует снимок в native.
|
||||
NativeLib.callAudioProgress(callId, epoch, -1, nowUs)
|
||||
NativeLib.callAudioProgress(callId, epoch, -1, nowUs, nowUs, false)
|
||||
LogManager.addLog("ERROR", "CallAudio",
|
||||
"playback clock invalid id=$callId epoch=$epoch written=$playFramesWritten; audio continues")
|
||||
}
|
||||
@@ -627,6 +660,7 @@ class CallAudioEngine {
|
||||
diagnostic.copyInto(row, 16); row[24] = offset.toLong()
|
||||
row[25] = headCallNs / 1000; row[26] = progressCallNs / 1000; row[27] = writeSpanUs
|
||||
row[28] = diagnosticUs; row[29] = nowUs - pullAtNs / 1000
|
||||
row[35] = outputUs
|
||||
trace.record(row)
|
||||
}
|
||||
|
||||
@@ -680,7 +714,8 @@ class CallAudioEngine {
|
||||
val level = if (actual == requested && (change == null || change.underruns == 0)) "INFO" else "WARN"
|
||||
LogManager.addLog(level, "CallAudio",
|
||||
"playback buffering id=$callId reason=$reason requestedFrames=$requested actualFrames=$actual " +
|
||||
"previousFrames=$previous startFrames=$start capacityFrames=$capacity underruns=${change?.underruns ?: 0} pacing=device")
|
||||
"previousFrames=$previous startFrames=$start capacityFrames=$capacity underruns=${change?.underruns ?: 0} " +
|
||||
"performanceRequested=${AudioTrack.PERFORMANCE_MODE_LOW_LATENCY} performanceActual=${track.performanceMode} pacing=device")
|
||||
if (playReady && (previous != actual || reason == "route")) {
|
||||
// Очередь сначала принимает новый размер; устаревшая задача не затронет следующий звонок.
|
||||
mainHandler.post {
|
||||
|
||||
@@ -39,3 +39,38 @@ internal class CallPlaybackHead {
|
||||
return position
|
||||
}
|
||||
}
|
||||
|
||||
/** Якорь конца записанного PCM: позиция не регрессирует при переходе head → аппаратный timestamp. */
|
||||
internal class CallPlaybackClock {
|
||||
data class Anchor(val position: Long, val atUs: Long, val measured: Boolean)
|
||||
private var generation = -1L
|
||||
private var stampFrame = -1L
|
||||
private var stampUs = -1L
|
||||
private var measuredOnce = false
|
||||
|
||||
fun update(written: Long, head: Long, nowUs: Long, routeGeneration: Long,
|
||||
sampleGeneration: Long, frame: Long, timeUs: Long): Anchor? {
|
||||
if (generation != routeGeneration) {
|
||||
generation = routeGeneration
|
||||
stampFrame = -1; stampUs = -1
|
||||
}
|
||||
if (head < 0 || head > written) return null
|
||||
// AudioTimestamp может содержать только младшие 32 бита; разворачиваем относительно written.
|
||||
var extended = frame
|
||||
if (frame in 0..0xffffffffL) {
|
||||
extended = (written and -0x100000000L) or frame
|
||||
if (extended > written) extended -= 0x100000000L
|
||||
}
|
||||
if (sampleGeneration == generation && extended in 0..written && timeUs > 0 &&
|
||||
timeUs in (nowUs - 500_000)..(nowUs + 200_000) && written - extended <= 96_000) {
|
||||
stampFrame = extended; stampUs = timeUs; measuredOnce = true
|
||||
}
|
||||
if (stampFrame >= 0 && stampUs in (nowUs - 500_000)..(nowUs + 200_000)) {
|
||||
val at = stampUs + (written - stampFrame) * 1_000_000 / 48_000
|
||||
if (at in (nowUs - 20_000)..(nowUs + 2_000_000)) return Anchor(written, at, true)
|
||||
}
|
||||
// Исчезнувший аппаратный timestamp не заменяем head: это сдвинуло бы видео вперёд на задержку HAL.
|
||||
if (measuredOnce) return null
|
||||
return Anchor(written, nowUs + (written - head) * 1_000_000 / 48_000, false)
|
||||
}
|
||||
}
|
||||
@@ -3,13 +3,13 @@ package com.utun.chat.data
|
||||
/** Только числа в playback-потоке; форматирование и вывод выполняет отдельный reporter. */
|
||||
internal class CallAudioTrace(private val id: Long, private val emit: (String, String) -> Unit) {
|
||||
companion object {
|
||||
const val WIDTH = 35
|
||||
const val WIDTH = 37
|
||||
const val CAPACITY = 512
|
||||
// monoUs, routeId, routeType, written, played, pullUs, writeCallUs, pullGapExcessUs, zeroWrites,
|
||||
// deviceUnderruns, bufferFrames, startFrames, stampFrame, stampNs, stampReadUs, stampOk,
|
||||
// encodedMs, processingMs, reserveMs, spreadMs, pcmSamples, tempo100, jbUnderruns, waiting, accepted
|
||||
// headCallUs, progressCallUs, writeSpanUs, jbSnapshotUs, blockUs,
|
||||
// deviceBeginUs, deviceEndUs, stampQueryUs, routeQueryUs, propertiesQueryUs
|
||||
// deviceBeginUs, deviceEndUs, stampQueryUs, routeQueryUs, propertiesQueryUs, outputUs, performanceMode
|
||||
private const val PRE_US = 2_000_000L
|
||||
private const val POST_US = 3_000_000L
|
||||
}
|
||||
@@ -20,7 +20,8 @@ internal class CallAudioTrace(private val id: Long, private val emit: (String, S
|
||||
val bufferFrames: Long = -1, val startFrames: Long = -1,
|
||||
val stampFrame: Long = -1, val stampNs: Long = -1, val stampOk: Boolean = false,
|
||||
val beginUs: Long = -1, val endUs: Long = -1,
|
||||
val stampQueryUs: Long = 0, val routeQueryUs: Long = 0, val propertiesQueryUs: Long = 0
|
||||
val stampQueryUs: Long = 0, val routeQueryUs: Long = 0, val propertiesQueryUs: Long = 0,
|
||||
val generation: Long = -1, val performanceMode: Int = -1
|
||||
) {
|
||||
fun copyTo(row: LongArray) {
|
||||
row[1] = routeId; row[2] = routeType; row[9] = underruns
|
||||
@@ -28,6 +29,7 @@ internal class CallAudioTrace(private val id: Long, private val emit: (String, S
|
||||
row[12] = stampFrame; row[13] = stampNs; row[14] = beginUs; row[15] = if (stampOk) 1 else 0
|
||||
row[30] = beginUs; row[31] = endUs
|
||||
row[32] = stampQueryUs; row[33] = routeQueryUs; row[34] = propertiesQueryUs
|
||||
row[36] = performanceMode.toLong()
|
||||
}
|
||||
}
|
||||
@Volatile var device = DeviceSample()
|
||||
@@ -139,7 +141,8 @@ internal class CallAudioTrace(private val id: Long, private val emit: (String, S
|
||||
"stamp=${last[12]}@${last[13]} readUs=${last[14]} ok=${last[15]} " +
|
||||
"deviceAgeUs=${if (last[31] >= 0) last[0] - last[31] else -1} " +
|
||||
"stampQueryUs=${range { it[32] }} routeQueryUs=${range { it[33] }} propertiesQueryUs=${range { it[34] }} " +
|
||||
"buffer=${last[10]} start=${last[11]} fault=$faults suppressed=$suppressed")
|
||||
"buffer=${last[10]} start=${last[11]} outputUs=${range { it[35] }} performanceMode=${last[36]} " +
|
||||
"outputIncludesQueue=1 fault=$faults suppressed=$suppressed")
|
||||
reportedDeviceUnderruns = last[9]; reportedJbUnderruns = last[22]
|
||||
summary.clear()
|
||||
}
|
||||
@@ -151,7 +154,7 @@ internal class CallAudioTrace(private val id: Long, private val emit: (String, S
|
||||
"bufferFrames,startFrames,stampFrame,stampNs,stampReadUs,stampOk," +
|
||||
"encodedMs,processingMs,reserveMs,spreadMs,pcmSamples,tempo100,jbUnderruns,waiting,accepted," +
|
||||
"headCallUs,progressCallUs,writeSpanUs,jbSnapshotUs,blockUs," +
|
||||
"deviceBeginUs,deviceEndUs,stampQueryUs,routeQueryUs,propertiesQueryUs")
|
||||
"deviceBeginUs,deviceEndUs,stampQueryUs,routeQueryUs,propertiesQueryUs,outputUs,performanceMode")
|
||||
// До пяти секунд, по четыре снимка в строке, с явным временем каждого.
|
||||
history.filter { it[0] in (eventAt - PRE_US)..(eventAt + POST_US) }.chunked(4).forEach { group ->
|
||||
emit("DEBUG", "audio detail call=$id rows=" + group.joinToString(";") { it.joinToString(",") })
|
||||
|
||||
@@ -403,7 +403,8 @@ object CallController : TelecomCallManager.Listener {
|
||||
39 -> if (st.phase == CallPhase.ACTIVE) {
|
||||
val buf = ByteBuffer.wrap(data, 8, 16).order(ByteOrder.BIG_ENDIAN)
|
||||
fun metric() = buf.short.toInt() and 0xFFFF
|
||||
_state.value = st.copy(stats = CallStats(metric(), metric(), metric(), metric(), metric(), metric(), metric(), metric()))
|
||||
_state.value = st.copy(stats = CallStats(metric(), metric(), metric(), metric(), metric(), metric(), metric(), metric(),
|
||||
service?.audio?.outputLatency(id) ?: CallOutputLatency()))
|
||||
}
|
||||
42 -> _state.value = st.copy(bgpPath = String(data, 8, data.size - 8))
|
||||
44 -> {
|
||||
|
||||
@@ -15,9 +15,13 @@ enum class CallConnectionType(val labelRes: Int) {
|
||||
|
||||
data class CallStats(
|
||||
val rttMs: Int, val bufferMs: Int, val tempoX100: Int, val dropped: Int,
|
||||
val underruns: Int, val minMs: Int, val maxMs: Int, val reserveMs: Int
|
||||
val underruns: Int, val minMs: Int, val maxMs: Int, val reserveMs: Int,
|
||||
val output: CallOutputLatency = CallOutputLatency()
|
||||
)
|
||||
|
||||
/** outputMs включает queueMs; это задержка конца записанного PCM, не сумма двух буферов. */
|
||||
data class CallOutputLatency(val queueMs: Int = -1, val outputMs: Int = -1)
|
||||
|
||||
data class CallState(
|
||||
val callId: Long = 0,
|
||||
val channelId: String = "",
|
||||
|
||||
@@ -268,7 +268,8 @@ object NativeLib {
|
||||
fun callAudioStart(callId: Long): Boolean = nativeCallAudioStart(callId)
|
||||
fun callAudioFeed(callId: Long, samples: ShortArray, capturePtsUs: Long): Boolean = nativeCallAudioFeed(callId, samples, capturePtsUs)
|
||||
fun callAudioPull(callId: Long, out: ShortArray): Int = nativeCallAudioPull(callId, out)
|
||||
fun callAudioProgress(callId: Long, epoch: Long, position: Long, nowUs: Long) = nativeCallAudioProgress(callId, epoch, position, nowUs)
|
||||
fun callAudioProgress(callId: Long, epoch: Long, position: Long, atUs: Long, nowUs: Long, measured: Boolean) =
|
||||
nativeCallAudioProgress(callId, epoch, position, atUs, nowUs, measured)
|
||||
/** out: epoch, source_epoch, played, written, pts_us, valid, measured, held_back_us. */
|
||||
fun callAudioClock(callId: Long, nowUs: Long, out: LongArray): Boolean = nativeCallAudioClock(callId, nowUs, out)
|
||||
/** call=debug: encoded_ms, processing_ms, reserve_ms, spread_ms, pcm_samples, tempo_x100, underruns, waiting. */
|
||||
@@ -404,7 +405,7 @@ object NativeLib {
|
||||
private external fun nativeCallAudioStart(callId: Long): Boolean
|
||||
private external fun nativeCallAudioFeed(callId: Long, samples: ShortArray, capturePtsUs: Long): Boolean
|
||||
private external fun nativeCallAudioPull(callId: Long, out: ShortArray): Int
|
||||
private external fun nativeCallAudioProgress(callId: Long, epoch: Long, position: Long, nowUs: Long)
|
||||
private external fun nativeCallAudioProgress(callId: Long, epoch: Long, position: Long, atUs: Long, nowUs: Long, measured: Boolean)
|
||||
private external fun nativeCallAudioClock(callId: Long, nowUs: Long, out: LongArray): Boolean
|
||||
private external fun nativeCallAudioDiagnostics(callId: Long, out: LongArray): Boolean
|
||||
private external fun nativeCallAudioStop()
|
||||
|
||||
@@ -257,6 +257,11 @@ fun CallScreen(
|
||||
stats.minMs, stats.maxMs, "%.2f".format(stats.tempoX100 / 100.0), stats.rttMs, stats.dropped, stats.underruns),
|
||||
modifier = Modifier.padding(horizontal = 16.dp), color = Color(0xFFAEBCC8),
|
||||
fontSize = 12.sp, textAlign = TextAlign.Center)
|
||||
Text(stringResource(R.string.call_output_stats,
|
||||
stats.output.queueMs.takeIf { it >= 0 }?.toString() ?: "—",
|
||||
stats.output.outputMs.takeIf { it >= 0 }?.toString() ?: "—"),
|
||||
modifier = Modifier.padding(horizontal = 16.dp), color = Color(0xFFAEBCC8),
|
||||
fontSize = 12.sp, textAlign = TextAlign.Center)
|
||||
}
|
||||
Spacer(Modifier.height(40.dp))
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user