diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt index a2473c23..98681689 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt @@ -345,11 +345,12 @@ class StreamCaptureEngine( } @SuppressLint("MissingPermission") private fun startAudio() { - val min = AudioRecord.getMinBufferSize(48000, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT) + val min = AudioRecord.getMinBufferSize(48000, AudioFormat.CHANNEL_IN_STEREO, AudioFormat.ENCODING_PCM_16BIT) check(min > 0) - val rec = AudioRecord(MediaRecorder.AudioSource.CAMCORDER, 48000, AudioFormat.CHANNEL_IN_MONO, - AudioFormat.ENCODING_PCM_16BIT, (min * 2).coerceAtLeast(960 * 8)) + val rec = AudioRecord(MediaRecorder.AudioSource.CAMCORDER, 48000, AudioFormat.CHANNEL_IN_STEREO, + AudioFormat.ENCODING_PCM_16BIT, (min * 2).coerceAtLeast(1920 * 8)) record = rec; check(rec.state == AudioRecord.STATE_INITIALIZED) + check(rec.channelCount == 2) { "CAMCORDER stereo capture unavailable: channels=${rec.channelCount}" } rec.startRecording(); check(rec.recordingState == AudioRecord.RECORDSTATE_RECORDING) { "CAMCORDER capture did not start" } audioRunning = true LogManager.addLog("INFO", "stream", "audio capture started id=$id source=CAMCORDER(${rec.audioSource}) " + @@ -358,7 +359,7 @@ class StreamCaptureEngine( var opus = 0L try { opus = NativeLib.streamOpusCreate(true); check(opus != 0L) - val pcm = ShortArray(960); val stamp = AudioTimestamp() + val pcm = ShortArray(1920); val stamp = AudioTimestamp() var total = 0L; var waitingStamp = true; var last = -1L while (audioRunning) { var filled = 0 @@ -368,7 +369,7 @@ class StreamCaptureEngine( if (n > 0) filled += n } if (!audioRunning || filled != pcm.size) break - val first = total; total += filled + val first = total; total += filled / 2 // AudioTimestamp считает кадры на канал, а read — interleaved сэмплы. if (rec.getTimestamp(stamp, AudioTimestamp.TIMEBASE_MONOTONIC) != AudioRecord.SUCCESS) { if (waitingStamp) LogManager.addLog("DEBUG", "stream", "audio timestamp pending id=$id samples=$total") continue diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt index e19ea012..56062a07 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamPlayerEngine.kt @@ -117,7 +117,10 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv while (running && offset < pcm.size) { val count = t.write(pcm, offset, pcm.size - offset, AudioTrack.WRITE_NON_BLOCKING) check(count >= 0) { "AudioTrack write=$count" } - if (count > 0) { offset += count; written += count } + if (count > 0) { + check(count % 2 == 0) { "AudioTrack returned an incomplete stereo frame: samples=$count" } + offset += count; written += count / 2 + } else { check(System.nanoTime() < deadline) { "AudioTrack stalled for 100ms" } Thread.sleep(2) @@ -207,13 +210,13 @@ class StreamPlayerEngine(private val group: Long, private val source: Long, priv opus = NativeLib.streamOpusCreate(false); check(opus != 0L) track = AudioTrack.Builder().setAudioAttributes(AudioAttributes.Builder().setUsage(AudioAttributes.USAGE_MEDIA) .setContentType(AudioAttributes.CONTENT_TYPE_MOVIE).build()).setAudioFormat(AudioFormat.Builder() - .setSampleRate(48000).setChannelMask(AudioFormat.CHANNEL_OUT_MONO).setEncoding(AudioFormat.ENCODING_PCM_16BIT).build()) - .setBufferSizeInBytes(AudioTrack.getMinBufferSize(48000, AudioFormat.CHANNEL_OUT_MONO, AudioFormat.ENCODING_PCM_16BIT) - .coerceAtLeast(9600)).setTransferMode(AudioTrack.MODE_STREAM).build() + .setSampleRate(48000).setChannelMask(AudioFormat.CHANNEL_OUT_STEREO).setEncoding(AudioFormat.ENCODING_PCM_16BIT).build()) + .setBufferSizeInBytes(AudioTrack.getMinBufferSize(48000, AudioFormat.CHANNEL_OUT_STEREO, AudioFormat.ENCODING_PCM_16BIT) + .coerceAtLeast(19200)).setTransferMode(AudioTrack.MODE_STREAM).build() check(track!!.state == AudioTrack.STATE_INITIALIZED) audioBase = audios.first.pts track!!.play() - LogManager.addLog("INFO", "stream", "audio playback starts id=$id PTS=$audioBase") + LogManager.addLog("INFO", "stream", "audio playback starts id=$id PTS=$audioBase rate=48000 channels=${track!!.channelCount}") } val t = track if (t != null && audios.isNotEmpty()) { diff --git a/tools/chatgui-android/jni_bridge/android_stream_jni.c b/tools/chatgui-android/jni_bridge/android_stream_jni.c index f85b8726..589124ca 100644 --- a/tools/chatgui-android/jni_bridge/android_stream_jni.c +++ b/tools/chatgui-android/jni_bridge/android_stream_jni.c @@ -11,6 +11,8 @@ #include "../libutun_lite/stream_bridge.h" #include "../../../src/stream/stream_media.h" +enum { STREAM_AUDIO_CHANNELS = 2, STREAM_AUDIO_FRAMES = 960, STREAM_AUDIO_SAMPLES = 1920 }; + struct android_stream_opus { opus_codec_encoder_t *encoder; opus_codec_decoder_t *decoder; @@ -41,7 +43,7 @@ JNIEXPORT jint JNICALL Java_com_utun_chat_data_NativeLib_nativeStreamFeed( jbyte *bytes = (*env)->GetByteArrayElements(env, data, NULL); if (!bytes) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream feed array access failed"); return -1; } struct stream_media_format format = {(uint16_t)width, (uint16_t)height, (uint16_t)rotation, - STREAM_MEDIA_H264, audio ? STREAM_MEDIA_OPUS : 0, (uint32_t)revision}; + STREAM_MEDIA_H264, audio ? STREAM_MEDIA_OPUS_STEREO : 0, (uint32_t)revision}; int result = android_stream_feed((uint64_t)id, video, (uint64_t)pts, key, (const uint8_t *)bytes, (size_t)length, video ? &format : NULL); (*env)->ReleaseByteArrayElements(env, data, bytes, JNI_ABORT); return result; @@ -67,16 +69,17 @@ JNIEXPORT jlong JNICALL Java_com_utun_chat_data_NativeLib_nativeStreamOpusCreate struct android_stream_opus *codec = u_calloc(1, sizeof(*codec)); if (!codec) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream Opus allocation failed"); return 0; } if (encode) { - codec->encoder = opus_codec_encoder_create(48000, 1); + codec->encoder = opus_codec_encoder_create(48000, STREAM_AUDIO_CHANNELS); if (!codec->encoder || opus_codec_encoder_application_set(codec->encoder, OPUS_CODEC_APP_AUDIO) < 0 - || opus_codec_encoder_bitrate_set(codec->encoder, 64000) < 0) goto fail; + || opus_codec_encoder_bitrate_set(codec->encoder, 128000) < 0) goto fail; codec->compressor = audio_compressor_create(); if (!codec->compressor) goto fail; } else { - codec->decoder = opus_codec_decoder_create(48000, 1); + codec->decoder = opus_codec_decoder_create(48000, STREAM_AUDIO_CHANNELS); if (!codec->decoder) goto fail; } - DEBUG_INFO(DEBUG_CATEGORY_STREAM, "JNI stream Opus created codec=%p encode=%d rate=48000 channels=1 frame=960", (void *)codec, encode); + DEBUG_INFO(DEBUG_CATEGORY_STREAM, "JNI stream Opus created codec=%p encode=%d rate=48000 channels=2 frame=960 bitrate=128000", + (void *)codec, encode); return (jlong)(intptr_t)codec; fail: DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream Opus initialization failed encode=%d", encode); @@ -88,7 +91,7 @@ JNIEXPORT jbyteArray JNICALL Java_com_utun_chat_data_NativeLib_nativeStreamOpusE JNIEnv *env, jobject obj, jlong handle, jshortArray samples, jboolean enabled, jint max_gain_db, jint rise_rate) { (void)obj; struct android_stream_opus *codec = (struct android_stream_opus *)(intptr_t)handle; - if (!codec || !codec->encoder || !codec->compressor || !samples || (*env)->GetArrayLength(env, samples) != 960) { + if (!codec || !codec->encoder || !codec->compressor || !samples || (*env)->GetArrayLength(env, samples) != STREAM_AUDIO_SAMPLES) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream Opus encode invalid handle/frame"); return NULL; } if (max_gain_db < 1 || max_gain_db > 60 || rise_rate < 1 || rise_rate > 100) { @@ -96,7 +99,7 @@ JNIEXPORT jbyteArray JNICALL Java_com_utun_chat_data_NativeLib_nativeStreamOpusE } if (codec->compressor_enabled != !!enabled || codec->max_gain_db != max_gain_db || codec->rise_rate != rise_rate) { audio_compressor_config_t config = {0}; - config.sample_rate = 48000; config.channels = 1; config.block_duration_ms = 20; + config.sample_rate = 48000; config.channels = STREAM_AUDIO_CHANNELS; config.block_duration_ms = 20; config.lookback_ms = 200; config.lookahead_ms = 0; config.target_level = 0.25f; config.max_gain_db = max_gain_db; config.rise_rate_per_sec = rise_rate; config.release_rate_per_sec = rise_rate * 5.0f; if (audio_compressor_configure(codec->compressor, &config) != 0) { @@ -108,23 +111,27 @@ JNIEXPORT jbyteArray JNICALL Java_com_utun_chat_data_NativeLib_nativeStreamOpusE "JNI stream compressor applied codec=%p enabled=%d max_gain=%ddB rise=%d release=%d target=0.25 history=200ms delay=0ms", (void *)codec, !!enabled, max_gain_db, rise_rate, rise_rate * 5); } - int16_t pcm[960]; uint8_t packet[1275]; - (*env)->GetShortArrayRegion(env, samples, 0, 960, pcm); + int16_t pcm[STREAM_AUDIO_SAMPLES]; uint8_t packet[1275]; + (*env)->GetShortArrayRegion(env, samples, 0, STREAM_AUDIO_SAMPLES, pcm); if ((*env)->ExceptionCheck(env)) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream PCM read failed"); return NULL; } - int report = codec->frames % 50 == 0, input_peak = 0, output_peak = 0; - if (report) for (unsigned i = 0; i < 960; i++) if (abs((int)pcm[i]) > input_peak) input_peak = abs((int)pcm[i]); - if (audio_compressor_process_frame(codec->compressor, pcm, pcm, 960) != 0) { + int report = codec->frames % 50 == 0, input_peak = 0, output_peak = 0, stereo_difference = 0; + if (report) { + for (unsigned i = 0; i < STREAM_AUDIO_SAMPLES; i++) if (abs((int)pcm[i]) > input_peak) input_peak = abs((int)pcm[i]); + for (unsigned i = 0; i < STREAM_AUDIO_FRAMES; i++) stereo_difference += abs((int)pcm[2 * i] - pcm[2 * i + 1]); + } + if (audio_compressor_process_frame(codec->compressor, pcm, pcm, STREAM_AUDIO_SAMPLES) != 0) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream compressor processing failed codec=%p", (void *)codec); return NULL; } - int length = opus_codec_encode(codec->encoder, pcm, 960, packet, sizeof(packet)); + int length = opus_codec_encode(codec->encoder, pcm, STREAM_AUDIO_FRAMES, packet, sizeof(packet)); if (length < 0) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream Opus encode failed result=%d", length); return NULL; } codec->frames++; if (report) { - for (unsigned i = 0; i < 960; i++) if (abs((int)pcm[i]) > output_peak) output_peak = abs((int)pcm[i]); + for (unsigned i = 0; i < STREAM_AUDIO_SAMPLES; i++) if (abs((int)pcm[i]) > output_peak) output_peak = abs((int)pcm[i]); uint32_t clips = audio_compressor_clip_count(codec->compressor); - DEBUG_DEBUG(DEBUG_CATEGORY_STREAM, "JNI stream audio codec=%p frames=%llu enabled=%d gain=%.1fdB peaks=%d->%d clips=%u packet=%d", + DEBUG_DEBUG(DEBUG_CATEGORY_STREAM, + "JNI stream audio codec=%p frames=%llu channels=2 enabled=%d gain=%.1fdB peaks=%d->%d lrDifference=%d clips=%u packet=%d", (void *)codec, (unsigned long long)codec->frames, codec->compressor_enabled, - audio_compressor_gain_db(codec->compressor), input_peak, output_peak, clips, length); + audio_compressor_gain_db(codec->compressor), input_peak, output_peak, stereo_difference / STREAM_AUDIO_FRAMES, clips, length); if (clips != codec->reported_clips) DEBUG_WARN(DEBUG_CATEGORY_STREAM, "JNI stream audio clipped codec=%p samples=%u gain=%.1fdB", (void *)codec, clips - codec->reported_clips, audio_compressor_gain_db(codec->compressor)); codec->reported_clips = clips; @@ -142,13 +149,13 @@ JNIEXPORT jshortArray JNICALL Java_com_utun_chat_data_NativeLib_nativeStreamOpus if (!codec || !codec->decoder || !packet || (*env)->GetArrayLength(env, packet) > 1275) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream Opus decode invalid handle/packet"); return NULL; } - jsize length = (*env)->GetArrayLength(env, packet); uint8_t bytes[1275]; int16_t pcm[960]; + jsize length = (*env)->GetArrayLength(env, packet); uint8_t bytes[1275]; int16_t pcm[STREAM_AUDIO_SAMPLES]; (*env)->GetByteArrayRegion(env, packet, 0, length, (jbyte *)bytes); if ((*env)->ExceptionCheck(env)) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream Opus packet read failed"); return NULL; } - int count = opus_codec_decode(codec->decoder, length ? bytes : NULL, length, pcm, 960); - if (count != 960) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream Opus decode failed count=%d", count); return NULL; } - jshortArray output = (*env)->NewShortArray(env, count); - if (output) (*env)->SetShortArrayRegion(env, output, 0, count, pcm); + int count = opus_codec_decode(codec->decoder, length ? bytes : NULL, length, pcm, STREAM_AUDIO_FRAMES); + if (count != STREAM_AUDIO_FRAMES) { DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream Opus decode failed count=%d", count); return NULL; } + jshortArray output = (*env)->NewShortArray(env, STREAM_AUDIO_SAMPLES); + if (output) (*env)->SetShortArrayRegion(env, output, 0, STREAM_AUDIO_SAMPLES, pcm); else DEBUG_ERROR(DEBUG_CATEGORY_STREAM, "JNI stream Opus PCM allocation failed"); return output; }