From bb380441a35f733bd41207f93e19fccc206baf73 Mon Sep 17 00:00:00 2001 From: evgeny Date: Thu, 22 Oct 2026 16:53:25 +0200 Subject: [PATCH] Fix shared microphone crackle from timestamp jitter --- .../com/utun/chat/data/MediaAudioProbe.kt | 12 +- .../utun/chat/data/MediaAudioController.kt | 16 ++- .../utun/chat/data/MediaCaptureTimeline.kt | 126 ++++++++++++++---- .../chat/data/MediaCaptureTimelineTest.kt | 93 ++++++++++++- 4 files changed, 216 insertions(+), 31 deletions(-) diff --git a/tools/chatgui-android/app/src/debug/java/com/utun/chat/data/MediaAudioProbe.kt b/tools/chatgui-android/app/src/debug/java/com/utun/chat/data/MediaAudioProbe.kt index c7c0264d..5e9591a6 100644 --- a/tools/chatgui-android/app/src/debug/java/com/utun/chat/data/MediaAudioProbe.kt +++ b/tools/chatgui-android/app/src/debug/java/com/utun/chat/data/MediaAudioProbe.kt @@ -74,12 +74,22 @@ class MediaAudioProbe : Instrumentation() { check(b.pcm.contentEquals(unchanged)) { "PCM copies are shared" } check(System.nanoTime() < until) { "Profile did not settle: ${MediaAudioController.state.value}" } count++ - } while (count < 30 || MediaAudioController.state.value.switching) + } while (count < 300 || MediaAudioController.state.value.switching) check(MediaAudioController.state.value.notice == null && MediaAudioController.state.value.inputName != null) { "Microphone was not confirmed: ${MediaAudioController.state.value}" } android.util.Log.i("MediaAudioProbe", "profile verified=$profile frames=$count state=${MediaAudioController.state.value}") } + // Непрерывный аппаратный участок длиннее окна диагностической сводки PCM clock. + repeat(300) { + val a = frame(second) + check(a.timeNs == previous + 20_000_000L) { "Steady capture PTS discontinuity" } + previous = a.timeNs + var b = frame(first) + while (b.sequence < a.sequence) b = frame(first) + check(a.sequence == b.sequence && a.pcm.contentEquals(b.pcm)) { "Steady consumers differ" } + } + android.util.Log.i("MediaAudioProbe", "steady hardware capture verified frames=300") first.close(); radio = null val next = frame(second) check(next.timeNs == previous + 20_000_000L) { "Closing radio interrupted stream capture" } diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/MediaAudioController.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/MediaAudioController.kt index 5a01d730..744c049c 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/MediaAudioController.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/MediaAudioController.kt @@ -281,9 +281,11 @@ object MediaAudioController { /** Постоянный stereo-вход с запасом времени на аппаратный reader и смену маршрута. */ private class InputBridge : MicrophoneCapture.Device { private val lock = Object() - private val timeline = MediaCaptureTimeline(System.nanoTime()) + private val timeline = MediaCaptureTimeline(System.nanoTime()) { level, message -> LogManager.addLog(level, "media_audio", message) } private var running = true private var reportedLate = 0L + private var reportedMissing = 0L + private var reportNs = 0L override val channels = 2 override val inputId: Int? get() = null fun reset(epoch: Long) = synchronized(lock) { timeline.reset(epoch) } @@ -299,7 +301,17 @@ object MediaAudioController { while (running) { val now = System.nanoTime() val wait = timeline.deadlineNs - now - if (wait <= 0) return@synchronized timeline.pull(now) + if (wait <= 0) { + val frame = timeline.pull(now) + if (now - reportNs >= 5_000_000_000L || (reportedMissing == 0L && timeline.missingFrames > 0)) { + val level = if (timeline.missingFrames > reportedMissing) "WARN" else "DEBUG" + LogManager.addLog(level, "media_audio", "PCM clock generation=${timeline.generation} " + + "jitterMaxUs=${timeline.timestampJitterMaxNs / 1000} jumps=${timeline.timestampJumps} ratePpm=${timeline.ratePpm} " + + "missingFrames=${timeline.missingFrames} silenceFrames=${timeline.silentFrames} lateBlocks=${timeline.lateFrames}") + reportedMissing = timeline.missingFrames; reportNs = now + } + return@synchronized frame + } lock.wait(wait / 1_000_000, (wait % 1_000_000).toInt()) } error("Shared microphone stopped") diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/MediaCaptureTimeline.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/MediaCaptureTimeline.kt index a8011be0..62853dab 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/MediaCaptureTimeline.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/MediaCaptureTimeline.kt @@ -1,39 +1,100 @@ package com.utun.chat.data import java.util.ArrayDeque +import kotlin.math.abs +import kotlin.math.floor import kotlin.math.roundToInt +import kotlin.math.roundToLong -/** Общая сетка PCM 48k stereo. Аппаратные timestamps определяют положение отсчётов; - * отсутствующие участки — тишина. Никакого переноса старого звука на новое время. - * Все методы вызываются под lock владельца. */ -internal class MediaCaptureTimeline(val originNs: Long) { +/** Общая сетка PCM 48k stereo. Внутри поколения отсчёты идут по sequence, без перекладки + * отдельных блоков по шумным timestamps. Медленный дрейф часов компенсируется непрерывной + * интерполяцией; только реальные пропуски дают тишину. Все методы — под lock владельца. */ +internal class MediaCaptureTimeline(val originNs: Long, private val log: (String, String) -> Unit = { _, _ -> }) { companion object { const val FRAMES = 960 const val PERIOD_NS = 20_000_000L const val RESERVE_NS = 200_000_000L + private const val NS_PER_FRAME = 1_000_000_000.0 / 48000 } private val pending = ArrayDeque() private var position = 0L private var lastInputNs = -1L + private var lastSequence = -1L + private var lastCreatedNs = 0L + private var anchorNs = 0L + private var anchorSequence = 0L + private var startPosition = 0L + private var anchored = false + private var cursor = Double.NaN + private var clockOffset = 0.0 + private var timestampBiasNs = 0L var generation = 0L private set var lateFrames = 0L private set var silentFrames = 0L private set + var missingFrames = 0L + private set + var timestampJitterMaxNs = 0L + private set + var timestampJumps = 0L + private set + var ratePpm = 0 + private set val nextTimeNs: Long get() = originNs + position * 1_000_000_000 / 48000 val deadlineNs: Long get() = nextTimeNs + PERIOD_NS + RESERVE_NS fun reset(epoch: Long) { - generation = epoch; pending.clear(); lastInputNs = -1 + generation = epoch; pending.clear(); lastInputNs = -1; lastSequence = -1; lastCreatedNs = 0 + anchored = false; cursor = Double.NaN; clockOffset = 0.0; ratePpm = 0; timestampJitterMaxNs = 0; timestampBiasNs = 0; timestampJumps = 0 } fun offer(frame: MicrophoneCapture.Frame): Boolean { - require(frame.pcm.size == FRAMES * 2) - if (frame.generation != generation || frame.timeNs <= lastInputNs) return false - lastInputNs = frame.timeNs - if (frame.timeNs + PERIOD_NS <= nextTimeNs) { lateFrames++; return false } - if (pending.size >= 24) { lateFrames++; return false } + require(frame.pcm.size == FRAMES * 2 && frame.sequence >= 0 && frame.timeNs >= 0) + if (frame.generation != generation) return false + if (lastSequence >= 0) { + if (frame.sequence <= lastSequence) { + log("WARN", "microphone stale PCM generation=$generation sequence=${frame.sequence}/$lastSequence time=${frame.timeNs}/$lastInputNs") + return false + } + val elapsed = (frame.sequence - lastSequence) * 1_000_000_000 / 48000 + val jitter = frame.timeNs - lastInputNs - elapsed + timestampJitterMaxNs = maxOf(timestampJitterMaxNs, abs(jitter)) + // Отличаем перестановку timestamp от остановки аппаратного reader при запуске Bluetooth. + // Свежий звук после реальной остановки привязываем заново, не воспроизводя старую очередь. + if (abs(jitter) > PERIOD_NS * 2) { + val deliveryGap = frame.createdNs - lastCreatedNs - elapsed + val latePcm = !cursor.isNaN() && frame.sequence + FRAMES <= cursor + val reanchor = frame.timeNs + PERIOD_NS > nextTimeNs && (deliveryGap > PERIOD_NS * 2 || latePcm) + if (reanchor) { pending.clear(); anchored = false; cursor = Double.NaN; clockOffset = 0.0; timestampBiasNs = 0 } + else timestampBiasNs += jitter + timestampJumps++ + if (timestampJumps == 1L || timestampJumps % 50L == 0L) + log("WARN", "microphone timestamp jump generation=$generation deltaNs=$jitter deliveryGapNs=$deliveryGap " + + "late=$latePcm reanchor=$reanchor count=$timestampJumps") + } + if (frame.sequence != lastSequence + FRAMES) + log("WARN", "microphone sequence gap generation=$generation expected=${lastSequence + FRAMES} actual=${frame.sequence}") + } + lastInputNs = frame.timeNs; lastSequence = frame.sequence; lastCreatedNs = frame.createdNs + val late = if (anchored && !cursor.isNaN()) frame.sequence + FRAMES <= cursor else frame.timeNs + PERIOD_NS <= nextTimeNs + if (late || pending.size >= 24) { + lateFrames++ + if (lateFrames == 1L || lateFrames % 50L == 0L) + log("WARN", "microphone PCM rejected generation=$generation late=$late queue=${pending.size} " + + "sequence=${frame.sequence} cursor=$cursor ageUs=${(frame.createdNs - frame.timeNs) / 1000} count=$lateFrames") + return false + } + if (!anchored) { + anchorNs = frame.timeNs; anchorSequence = frame.sequence + startPosition = ((anchorNs - originNs) / NS_PER_FRAME).roundToLong() + anchored = true + log("INFO", "microphone clock anchored generation=$generation sequence=$anchorSequence position=$startPosition") + } + // Фильтр около 1с подавляет джиттер измерений; фазовая ошибка не двигает уже прочитанный PCM. + val measured = (frame.timeNs - anchorNs - timestampBiasNs) / NS_PER_FRAME - (frame.sequence - anchorSequence) + clockOffset += (measured - clockOffset) * 0.02 pending.addLast(frame) return true } @@ -42,24 +103,37 @@ internal class MediaCaptureTimeline(val originNs: Long) { check(nowNs >= deadlineNs) val start = nextTimeNs val pcm = ShortArray(FRAMES * 2) - var filled = 0 - while (pending.isNotEmpty()) { - val frame = pending.first - val offset = ((frame.timeNs - start).toDouble() * 48000 / 1_000_000_000).roundToInt() - if (offset >= FRAMES) break - val source = maxOf(0, -offset) - val target = maxOf(0, offset, filled) - val skip = target - maxOf(0, offset) - val count = minOf(FRAMES - source - skip, FRAMES - target) - if (count > 0) { - frame.pcm.copyInto(pcm, target * 2, (source + skip) * 2, (source + skip + count) * 2) - silentFrames += target - filled - filled = target + count + val leading = if (anchored) (startPosition - position).coerceIn(0, FRAMES.toLong()).toInt() else FRAMES + if (anchored && leading < FRAMES && cursor.isNaN()) cursor = anchorSequence + maxOf(0, position - startPosition).toDouble() + var step = 1.0 + if (!cursor.isNaN()) { + val error = anchorSequence + (position + leading - startPosition).toDouble() - clockOffset - cursor + // 1мс мёртвой зоны: обычный джиттер не меняет PCM. Коррекция дрейфа ограничена ±0.2%. + val correction = if (abs(error) <= 48) 0.0 else (error - if (error > 0) 48 else -48) / 96000 + step += correction.coerceIn(-0.002, 0.002) + } + ratePpm = ((step - 1) * 1_000_000).roundToInt() + for (i in 0 until FRAMES) { + if (i < leading) { silentFrames++; continue } + val sample = floor(cursor).toLong() + while (pending.isNotEmpty() && pending.first.sequence + FRAMES <= sample) pending.removeFirst() + val frame = pending.peekFirst() + val offset = if (frame == null) -1 else sample - frame.sequence + if (frame == null || offset < 0 || offset >= FRAMES) { + silentFrames++; missingFrames++ + } else { + val index = offset.toInt() + val fraction = cursor - sample + val next = if (index + 1 < FRAMES) frame else pending.elementAtOrNull(1)?.takeIf { it.sequence == frame.sequence + FRAMES } + val nextIndex = if (index + 1 < FRAMES) index + 1 else 0 + for (channel in 0..1) { + val a = frame.pcm[index * 2 + channel].toInt() + val b = next?.pcm?.get(nextIndex * 2 + channel)?.toInt() ?: a + pcm[i * 2 + channel] = (a + (b - a) * fraction).roundToInt().toShort() + } } - if (offset + FRAMES > FRAMES) break - pending.removeFirst() + cursor += step } - silentFrames += FRAMES - filled return MicrophoneCapture.Frame(pcm, start, position, nowNs, generation).also { position += FRAMES } } } diff --git a/tools/chatgui-android/app/src/test/java/com/utun/chat/data/MediaCaptureTimelineTest.kt b/tools/chatgui-android/app/src/test/java/com/utun/chat/data/MediaCaptureTimelineTest.kt index 461d6fab..95768340 100644 --- a/tools/chatgui-android/app/src/test/java/com/utun/chat/data/MediaCaptureTimelineTest.kt +++ b/tools/chatgui-android/app/src/test/java/com/utun/chat/data/MediaCaptureTimelineTest.kt @@ -5,8 +5,8 @@ import org.junit.Test class MediaCaptureTimelineTest { private val origin = 1_000_000_000L - private fun frame(time: Long, value: Int, epoch: Long = 1) = - MicrophoneCapture.Frame(ShortArray(1920) { value.toShort() }, time, 0, generation = epoch) + private fun frame(time: Long, value: Int, epoch: Long = 1, sequence: Long = (time - origin) * 48000 / 1_000_000_000) = + MicrophoneCapture.Frame(ShortArray(1920) { value.toShort() }, time, sequence, generation = epoch) private fun pull(timeline: MediaCaptureTimeline) = timeline.pull(timeline.deadlineNs) @Test fun switchingKeepsTimeAndSequenceAndRejectsOldDevice() { @@ -68,4 +68,93 @@ class MediaCaptureTimelineTest { assertFalse(timeline.offer(frame(origin + 480_000_000L, 2))) assertEquals(1L, timeline.lateFrames) } + + @Test fun timestampJitterDoesNotInsertSilenceOrChangeAnySample() { + for (jitter in listOf(20_000L, 100_000L, 350_000L)) { + val timeline = MediaCaptureTimeline(origin) + timeline.reset(1) + repeat(510) { index -> + val pcm = ShortArray(1920) { ((index * 1920 + it) % 30000 + 1).toShort() } + val offset = if (index == 0) 0 else if (index % 2 == 0) jitter else -jitter + assertTrue(timeline.offer(MicrophoneCapture.Frame(pcm, origin + index * 20_000_000L + offset, + index * 960L, generation = 1))) + if (index >= 10) { + val output = pull(timeline) + val expected = ShortArray(1920) { (((index - 10) * 1920 + it) % 30000 + 1).toShort() } + assertArrayEquals("jitter=$jitter block=$index", expected, output.pcm) + assertEquals(0L, timeline.missingFrames) + assertEquals(0, timeline.ratePpm) + } + } + } + } + + @Test fun actualSequenceGapIsSilenceAndLaterSoundKeepsItsPosition() { + val timeline = MediaCaptureTimeline(origin) + timeline.reset(1) + assertTrue(timeline.offer(frame(origin, 11))) + assertTrue(timeline.offer(frame(origin + 40_000_000, 22))) + assertTrue(pull(timeline).pcm.all { it == 11.toShort() }) + assertTrue(pull(timeline).pcm.all { it == 0.toShort() }) + assertTrue(pull(timeline).pcm.all { it == 22.toShort() }) + assertEquals(960L, timeline.missingFrames) + } + + @Test fun slowClockDriftIsResampledWithoutHolesOrSharpEdges() { + for (ppm in listOf(-300.0, 300.0)) { + val timeline = MediaCaptureTimeline(origin) + timeline.reset(1) + var sent = 0 + var previous = 0 + repeat(15000) { output -> + val until = output * 20_000_000L + MediaCaptureTimeline.RESERVE_NS + while (sent * 20_000_000.0 * (1 + ppm / 1_000_000) <= until) { + val pcm = ShortArray(1920) { channel -> + (10000 + 4000 * kotlin.math.sin(2 * Math.PI * 200 * (sent * 960L + channel / 2) / 48000)).toInt().toShort() + } + assertTrue(timeline.offer(MicrophoneCapture.Frame(pcm, + origin + (sent * 20_000_000.0 * (1 + ppm / 1_000_000)).toLong(), sent * 960L, generation = 1))) + sent++ + } + val frame = pull(timeline) + for (i in 0 until 960) { + val sample = frame.pcm[i * 2].toInt() + assertTrue("zero/hole ppm=$ppm block=$output", sample >= 5999) + if (previous != 0) assertTrue("click ppm=$ppm block=$output", kotlin.math.abs(sample - previous) <= 110) + assertEquals(frame.pcm[i * 2], frame.pcm[i * 2 + 1]) + previous = sample + } + assertEquals(0L, timeline.missingFrames) + } + assertTrue("clock correction ppm=$ppm actual=${timeline.ratePpm}", kotlin.math.abs(timeline.ratePpm + ppm) < 20) + } + } + + @Test fun bluetoothTimestampJumpsDoNotInterruptContinuousPcm() { + val timeline = MediaCaptureTimeline(origin) + timeline.reset(1) + assertTrue(timeline.offer(frame(origin, 11))) + assertTrue(timeline.offer(frame(origin + 247_000_000, 22, sequence = 960))) + assertTrue(timeline.offer(frame(origin + 40_000_000, 33, sequence = 1920))) + assertTrue(pull(timeline).pcm.all { it == 11.toShort() }) + assertTrue(pull(timeline).pcm.all { it == 22.toShort() }) + assertTrue(pull(timeline).pcm.all { it == 33.toShort() }) + assertEquals(2L, timeline.timestampJumps) + assertEquals(0L, timeline.missingFrames) + assertEquals(0, timeline.ratePpm) + } + + @Test fun realCaptureStallReanchorsFreshPcmInsteadOfRejectingEveryFollowingBlock() { + val timeline = MediaCaptureTimeline(origin) + timeline.reset(1) + assertTrue(timeline.offer(MicrophoneCapture.Frame(ShortArray(1920) { 11 }, origin, 0, origin + 20_000_000, 1))) + assertTrue(pull(timeline).pcm.all { it == 11.toShort() }) + repeat(10) { pull(timeline) } + assertTrue(timeline.offer(MicrophoneCapture.Frame(ShortArray(1920) { 22 }, origin + 240_000_000, + 960, origin + 260_000_000, 1))) + assertTrue(pull(timeline).pcm.all { it == 0.toShort() }) + assertTrue(pull(timeline).pcm.all { it == 22.toShort() }) + assertEquals(1L, timeline.timestampJumps) + assertEquals(0L, timeline.lateFrames) + } }