From 94a537ebddce040c2830430bfec5e6fa1cf73e06 Mon Sep 17 00:00:00 2001 From: evgeny Date: Wed, 29 Jul 2026 01:15:57 +0300 Subject: [PATCH] voice: waveform recording + display, playback wiring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - voice_recorder: waveform_levels[100] accumulated during feed, normalized to 100 values in stop() stored as 'wf=...;dur=...;' in chat_msg_submit data - bridge: filePath in get_messages_json from local_attrs.fp, g_db_path moved global - Message model: waveform: List, voiceDurationMs: Int, filePath: String - VoiceBubble: Canvas waveform (100 bars, log scale), play button, duration - ChatViewModel.playVoiceMessage → AudioRecorderManager.playVoiceFile --- .../java/com/utun/chat/data/ChatRepository.kt | 30 ++++++- .../utun/chat/ui/components/MessageBubble.kt | 78 ++++++++++++++++--- .../com/utun/chat/ui/screens/ChatScreen.kt | 4 +- .../com/utun/chat/viewmodel/ChatViewModel.kt | 5 ++ .../jni_bridge/android_jni_bridge.c | 31 ++++++-- .../libutun_lite/voice_recorder.c | 49 ++++++++++-- 6 files changed, 167 insertions(+), 30 deletions(-) diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/ChatRepository.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/ChatRepository.kt index c9130a8c..15351201 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/data/ChatRepository.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/data/ChatRepository.kt @@ -7,7 +7,9 @@ import org.json.JSONObject data class Channel(val id: String, val name: String, val lastMsgAt: Long = 0, val lastMsg: String = "", val peersOnline: Int = 0) data class Message(val id: Long, val author: String, val text: String, val ts: Long, - val isOutgoing: Boolean = false, val contentType: String = "text/plain") + val isOutgoing: Boolean = false, val contentType: String = "text/plain", + val filePath: String = "", val waveform: List = emptyList(), + val voiceDurationMs: Int = 0) class ChatRepository { private val myNodeId: Long = NativeLib.getMyNodeId() @@ -47,13 +49,35 @@ class ChatRepository { val arr = JSONArray(NativeLib.getMessages(channelId, limit)) for (i in 0 until arr.length()) { val obj = arr.getJSONObject(i) + val text = obj.optString("text", "") + val contentType = obj.optString("contentType", "text/plain") + val filePath = obj.optString("filePath", "") + + /* Parse waveform from voice message text (format: "wf=0.1,0.2,...;dur=1234;") */ + var waveform = emptyList() + var voiceDurationMs = 0 + if (contentType == "audio/opus" && text.startsWith("wf=")) { + try { + val wfEnd = text.indexOf(";dur=") + if (wfEnd > 0) { + val wfPart = text.substring(3, wfEnd) + waveform = wfPart.split(",").map { it.toFloat() } + val durPart = text.substring(wfEnd + 5).takeWhile { it != ';' } + voiceDurationMs = durPart.toIntOrNull() ?: 0 + } + } catch (_: Exception) {} + } + list.add(Message( id = obj.getLong("id"), author = obj.getString("author"), - text = obj.getString("text"), + text = text, ts = obj.getLong("ts"), isOutgoing = obj.optBoolean("isOutgoing", false), - contentType = obj.optString("contentType", "text/plain") + contentType = contentType, + filePath = filePath, + waveform = waveform, + voiceDurationMs = voiceDurationMs )) } } catch (e: Exception) { Log.w("utun-gui", "getMessages error", e) } diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/components/MessageBubble.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/components/MessageBubble.kt index 8518c023..7525e83d 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/components/MessageBubble.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/components/MessageBubble.kt @@ -4,6 +4,7 @@ import android.Manifest import android.content.pm.PackageManager import androidx.activity.compose.rememberLauncherForActivityResult import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.Canvas import androidx.compose.foundation.background import androidx.compose.foundation.clickable import androidx.compose.foundation.layout.* @@ -14,6 +15,8 @@ import androidx.compose.runtime.* import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size import androidx.compose.ui.graphics.Color import androidx.compose.ui.input.pointer.pointerInput import androidx.compose.ui.platform.LocalConfiguration @@ -22,13 +25,14 @@ import androidx.compose.ui.platform.LocalDensity import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp +import kotlin.math.log10 import androidx.core.content.ContextCompat import com.utun.chat.data.Message import com.utun.chat.data.NativeLib import kotlinx.coroutines.delay @Composable -fun MessageBubble(message: Message) { +fun MessageBubble(message: Message, onPlayVoice: (String) -> Unit = {}) { val bgColor = if (message.isOutgoing) Color(0xFF4A90D9) else Color(0xFFE8E8E8) val textColor = if (message.isOutgoing) Color.White else Color.Black val alignment = if (message.isOutgoing) Arrangement.End else Arrangement.Start @@ -48,21 +52,15 @@ fun MessageBubble(message: Message) { .background(bgColor) .padding(12.dp) ) { - Text(message.author, fontSize = 13.sp, fontWeight = FontWeight.Bold, color = textColor.copy(alpha = 0.7f)) + if (!isVoice) Text(message.author, fontSize = 13.sp, fontWeight = FontWeight.Bold, color = textColor.copy(alpha = 0.7f)) Spacer(Modifier.height(2.dp)) when { - isVoice -> { - Row(verticalAlignment = Alignment.CenterVertically) { - Text("\u25B6", fontSize = 20.sp, color = textColor) - Spacer(Modifier.width(8.dp)) - Text("Voice message", fontSize = 15.sp, color = textColor) - } - } - isMedia -> { - Text("\uD83D\uDCC4 ${message.text}", fontSize = 15.sp, color = textColor) - } + isVoice -> VoiceBubble(message, textColor, onPlayVoice) + isMedia -> Text("\uD83D\uDCC4 ${message.text}", fontSize = 15.sp, color = textColor) else -> { + Text(message.author, fontSize = 13.sp, fontWeight = FontWeight.Bold, color = textColor.copy(alpha = 0.7f)) + Spacer(Modifier.height(2.dp)) Text(message.text, fontSize = 17.sp, color = textColor) } } @@ -78,6 +76,62 @@ fun MessageBubble(message: Message) { } } +@Composable +private fun VoiceBubble(message: Message, textColor: Color, onPlay: (String) -> Unit) { + val durationSec = message.voiceDurationMs / 1000f + val wf = message.waveform + val wfCount = wf.size + + Row(verticalAlignment = Alignment.CenterVertically) { + /* Play button */ + Box( + modifier = Modifier.size(32.dp).background(Color(0xFF3390EC), CircleShape) + .clickable { if (message.filePath.isNotEmpty()) onPlay(message.filePath) }, + contentAlignment = Alignment.Center + ) { + Text("\u25B6", fontSize = 14.sp, color = Color.White) + } + Spacer(Modifier.width(8.dp)) + + /* Waveform */ + if (wfCount >= 2) { + Box(modifier = Modifier.weight(1f).height(32.dp)) { + Canvas(modifier = Modifier.fillMaxSize()) { + val barW = size.width / wfCount + val ph = size.height.toFloat() + for (i in 0 until wfCount) { + val v = wf[i] + val db = if (v > 0.00001f) 20f * kotlin.math.log10(v.toDouble()).toFloat() else -100f + val logV = ((db + 60f) / 60f).coerceIn(0f, 1f) + val barH = (2f + logV * (ph - 2f)).coerceAtMost(ph) + val x = i.toFloat() * barW + val y = ph - barH + drawRect( + color = Color(0xFF3390EC).copy(alpha = 0.5f), + topLeft = Offset(x, y), + size = Size(barW * 0.8f, barH) + ) + } + } + } + Spacer(Modifier.width(4.dp)) + } + + /* Duration */ + Text( + formatDurationMs(message.voiceDurationMs), + fontSize = 13.sp, + color = textColor.copy(alpha = 0.7f), + modifier = Modifier.padding(start = 4.dp) + ) + } +} + +private fun formatDurationMs(ms: Int): String { + val secs = ms / 1000 + return "%d:%02d".format(secs / 60, secs % 60) +} + private val LOCK_SWIPE_THRESHOLD_DP = 80f @Composable diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/ChatScreen.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/ChatScreen.kt index 2bbcc458..6ba73670 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/ChatScreen.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/ui/screens/ChatScreen.kt @@ -63,7 +63,9 @@ fun ChatScreen( state = listState ) { items(messages.size) { idx -> - MessageBubble(messages[idx]) + MessageBubble(messages[idx], onPlayVoice = { path -> + viewModel.playVoiceMessage(path) + }) } item { Spacer(Modifier.height(8.dp)) } } diff --git a/tools/chatgui-android/app/src/main/java/com/utun/chat/viewmodel/ChatViewModel.kt b/tools/chatgui-android/app/src/main/java/com/utun/chat/viewmodel/ChatViewModel.kt index d06d6bc2..f774461c 100644 --- a/tools/chatgui-android/app/src/main/java/com/utun/chat/viewmodel/ChatViewModel.kt +++ b/tools/chatgui-android/app/src/main/java/com/utun/chat/viewmodel/ChatViewModel.kt @@ -264,6 +264,11 @@ class ChatViewModel : ViewModel() { NativeLib.voiceSetCompressorConfig(maxGainDb, lookbackMs, lookaheadMs, riseRate, targetLevelDb) } + fun playVoiceMessage(filePath: String) { + LogManager.addLog("INFO", "VM", "play voice: $filePath") + audioRecorder.playVoiceFile(filePath) + } + fun clearState() { repo?.close() repo = null diff --git a/tools/chatgui-android/jni_bridge/android_jni_bridge.c b/tools/chatgui-android/jni_bridge/android_jni_bridge.c index b7a203b0..e25d80cf 100644 --- a/tools/chatgui-android/jni_bridge/android_jni_bridge.c +++ b/tools/chatgui-android/jni_bridge/android_jni_bridge.c @@ -52,6 +52,8 @@ static int blevel_to_android(int blevel) { static utun_log_fn g_on_log = NULL; static utun_event_fn g_on_event = NULL; +static char g_db_path[512]; + static void bridge_log(int level, const char* fmt, ...) { va_list ap; va_start(ap, fmt); @@ -356,7 +358,7 @@ char* utun_bridge_get_messages_json(const char* channel_id, int limit) { msg_table_name(channel_id, tbl_msg, sizeof(tbl_msg)); char sql[256]; - snprintf(sql, sizeof(sql), "SELECT id, node_id, data, timestamp FROM \"%s\" ORDER BY timestamp ASC LIMIT %d", tbl_msg, limit); + snprintf(sql, sizeof(sql), "SELECT id, node_id, data, timestamp, local_attrs FROM \"%s\" ORDER BY timestamp ASC LIMIT %d", tbl_msg, limit); sqlite3_stmt* st = NULL; if (sqlite3_prepare_v2(db, sql, -1, &st, NULL) != SQLITE_OK) return strdup("[]"); @@ -373,6 +375,7 @@ char* utun_bridge_get_messages_json(const char* channel_id, int limit) { int64_t ts = sqlite3_column_int64(st, 3); const uint8_t* dptr = sqlite3_column_blob(st, 2); int dlen = sqlite3_column_bytes(st, 2); + const char* lattr = (const char*)sqlite3_column_text(st, 4); char txt[256] = "", ct[64] = "text/plain"; if (dptr && dlen > 0) { @@ -392,12 +395,29 @@ char* utun_bridge_get_messages_json(const char* channel_id, int limit) { if (!author_name[0]) snprintf(author_name, sizeof(author_name), "0x%016llx", (unsigned long long)node_id); - size_t needed = snprintf(NULL, 0, "%s{\"id\":%lld,\"author\":\"%s\",\"text\":\"%s\",\"ts\":%lld,\"isOutgoing\":%s,\"contentType\":\"%s\"}", - sep, (long long)msg_id, author_name, esc_txt, (long long)ts, is_out ? "true" : "false", esc_ct); + /* build filePath for voice/attachment messages */ + char file_path[1024] = ""; + if (lattr) { + const char* fps = strstr(lattr, "\"fp\":\""); + if (fps) { fps += 6; + char* fpe = strchr((char*)fps, '"'); + if (fpe) { size_t fpl = (size_t)(fpe - fps); + snprintf(file_path, sizeof(file_path), "%s/media/%s/%.*s", + g_db_path, channel_id, (int)fpl, fps); + } + } + } + char* esc_fp = file_path[0] ? json_escape_alloc(file_path) : NULL; + + size_t needed = snprintf(NULL, 0, "%s{\"id\":%lld,\"author\":\"%s\",\"text\":\"%s\",\"ts\":%lld,\"isOutgoing\":%s,\"contentType\":\"%s\"%s%s%s}", + sep, (long long)msg_id, author_name, esc_txt, (long long)ts, is_out ? "true" : "false", esc_ct, + esc_fp ? ",\"filePath\":\"" : "", esc_fp ? esc_fp : "", esc_fp ? "\"" : ""); while (pos + needed + 2 > cap) { cap *= 2; char* tmp = u_realloc(json, cap); if (!tmp) break; json = tmp; } if (pos + needed + 2 <= cap) - pos += (size_t)snprintf(json + pos, cap - pos, "%s{\"id\":%lld,\"author\":\"%s\",\"text\":\"%s\",\"ts\":%lld,\"isOutgoing\":%s,\"contentType\":\"%s\"}", - sep, (long long)msg_id, author_name, esc_txt, (long long)ts, is_out ? "true" : "false", esc_ct); + pos += (size_t)snprintf(json + pos, cap - pos, "%s{\"id\":%lld,\"author\":\"%s\",\"text\":\"%s\",\"ts\":%lld,\"isOutgoing\":%s,\"contentType\":\"%s\"%s%s%s}", + sep, (long long)msg_id, author_name, esc_txt, (long long)ts, is_out ? "true" : "false", esc_ct, + esc_fp ? ",\"filePath\":\"" : "", esc_fp ? esc_fp : "", esc_fp ? "\"" : ""); + if (esc_fp) u_free(esc_fp); u_free(esc_ct); u_free(esc_txt); } @@ -584,7 +604,6 @@ static jobject g_config_callback = NULL; static jobject g_log_callback = NULL; static jobject g_event_callback = NULL; static jmethodID g_event_mid = NULL; -static char g_db_path[512]; JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved) { (void)reserved; diff --git a/tools/chatgui-android/libutun_lite/voice_recorder.c b/tools/chatgui-android/libutun_lite/voice_recorder.c index 355c6104..0bdebc6b 100644 --- a/tools/chatgui-android/libutun_lite/voice_recorder.c +++ b/tools/chatgui-android/libutun_lite/voice_recorder.c @@ -47,6 +47,9 @@ struct voice_recorder { int preset; float peak_level; + float waveform_levels[100]; + int waveform_count; + int64_t next_wf_time_ms; int64_t start_time_ms; }; @@ -54,6 +57,11 @@ struct voice_recorder { static struct voice_recorder* g_rec = NULL; static pthread_mutex_t g_init_mtx = PTHREAD_MUTEX_INITIALIZER; +static int64_t now_ms(void) { + struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1000LL + ts.tv_nsec / 1000000LL; +} + int voice_recorder_init(const char* db_path) { pthread_mutex_lock(&g_init_mtx); if (g_rec) { pthread_mutex_unlock(&g_init_mtx); return 0; } @@ -128,6 +136,8 @@ int voice_recorder_start(const char* channel_id, int sample_rate, int channels) g_rec->pcm_count = 0; g_rec->active = 1; + g_rec->waveform_count = 0; + g_rec->next_wf_time_ms = 0; struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); g_rec->start_time_ms = ts.tv_sec * 1000LL + ts.tv_nsec / 1000000LL; @@ -158,6 +168,14 @@ int voice_recorder_feed(const int16_t* samples, int count) { sum_sq += v * v; float av = fabsf(v); if (av > peak) peak = av; } g_rec->peak_level = (sqrtf(sum_sq / (float)count) * 2.0f + peak) / 2.0f; + + /* accumulate waveform level every ~100ms */ + { int64_t now = now_ms(); + if (g_rec->waveform_count < 100 && now >= g_rec->next_wf_time_ms) { + g_rec->waveform_levels[g_rec->waveform_count++] = g_rec->peak_level; + g_rec->next_wf_time_ms = now + 100; + } + } } if (g_rec->compressor && g_rec->compressor_enabled) { @@ -189,11 +207,6 @@ static uint32_t media_calc_block_size(uint64_t file_size) { return (uint32_t)target; } -static int64_t now_ms(void) { - struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); - return ts.tv_sec * 1000LL + ts.tv_nsec / 1000000LL; -} - static void voice_cleanup_locked(struct voice_recorder* rec) { u_free(rec->pcm_buffer); rec->pcm_buffer = NULL; rec->pcm_count = 0; rec->pcm_cap = 0; @@ -383,8 +396,27 @@ int voice_recorder_stop(int* out_duration_ms) { fclose(src); unlink(temp_path); - /* build and post chat_msg_submit */ - struct chat_msg_submit* req = u_calloc(1, sizeof(struct chat_msg_submit) + 1); + /* build and post chat_msg_submit with waveform data */ + char wf_buf[1024]; + int wf_off = 0; + { int wfN = g_rec->waveform_count > 0 ? g_rec->waveform_count : 1; + int step100 = wfN > 100 ? wfN : 100; + for (int i = 0; i < 100; i++) { + int s = i * wfN / 100, e = (i + 1) * wfN / 100; + if (s >= wfN) s = wfN - 1; if (e <= s) e = s + 1; if (e > wfN) e = wfN; + float sum = 0; int cnt = 0; + for (int j = s; j < e; j++) { sum += g_rec->waveform_levels[j]; cnt++; } + float v = cnt > 0 ? sum / cnt : 0; + if (i > 0) wf_off += snprintf(wf_buf + wf_off, sizeof(wf_buf) - wf_off, ","); + wf_off += snprintf(wf_buf + wf_off, sizeof(wf_buf) - wf_off, "%.3f", v); + } + } + + char wf_data[1280]; + int wf_data_len = snprintf(wf_data, sizeof(wf_data), "wf=%s;dur=%lld;", wf_buf, (long long)elapsed_ms); + + size_t total_data_len = (size_t)wf_data_len; + struct chat_msg_submit* req = u_calloc(1, sizeof(struct chat_msg_submit) + total_data_len + 1); if (!req) { voice_cleanup_locked(g_rec); pthread_mutex_unlock(&g_rec->mtx); pthread_mutex_unlock(&g_init_mtx); return -1; } snprintf(req->channel_id, sizeof(req->channel_id), "%s", g_rec->channel_id); @@ -395,7 +427,8 @@ int voice_recorder_stop(int* out_duration_ms) { snprintf(req->media_ext, sizeof(req->media_ext), "opus"); req->media_num_blocks = (uint32_t)num_blocks; req->data = (uint8_t*)(req + 1); - req->data_len = 0; + req->data_len = (uint32_t)total_data_len; + memcpy(req->data, wf_data, total_data_len); req->timestamp = 0; DEBUG_INFO(DEBUG_CATEGORY_GENERAL, "voice_recorder_stop: posting media ch=%s dt=%s stem=%s blocks=%d",