Share Android file codecs and request IDs with avatars

tools/chatgui-android/app/src/main/java/com/utun/chat/data/AvatarProtocol.kt

@@ -38,48 +38,18 @@ object AvatarProtocol {
                      val cursor: Long = 0, val version: ByteArray = byteArrayOf(), val more: Boolean = false,
                      val path: String = "", val payload: ByteArray = byteArrayOf())
     fun response(data: ByteArray): Reply {
-        require(data.size >= 141) { "file response length=${data.size}" }
+        val r = MemberFilesProtocol.response(data)
-        val b = buffer(data)
+        return Reply(r.id, r.group, r.server, r.owner, r.command, r.status, r.size, hash(r.hash), r.type,
-        val group = b.long; val server = b.long
+            r.cursor, r.version, r.more, r.path, r.payload)
-        require(b.get().toInt() == 4 && b.get().toInt() == 2) { "invalid file response protocol/kind" }
-        val command = b.get().toInt() and 255; val status = b.short.toInt() and 65535
-        val id = b.long; val owner = b.long
-        b.long; val size = b.long
-        val cursor = b.long; val version = ByteArray(16).also { b.get(it) }
-        b.position(16 + 77); val hash = ByteArray(32); b.get(hash)
-        b.position(16 + 117); val fileType = b.get().toInt() and 255
-        val more = b.get().toInt() and 255; require(more in 0..1) { "invalid list more flag" }
-        b.position(16 + 119)
-        val path = b.short.toInt() and 65535; val target = b.short.toInt() and 65535; val payload = b.short.toInt() and 65535
-        require(path <= 1024 && target <= 1024 && payload <= 8192 && b.remaining() == path + target + payload) {
-            "invalid file response body lengths"
-        }
-        val remote = ByteArray(path).also { b.get(it) }; b.position(b.position() + target)
-        val body = ByteArray(payload).also { b.get(it) }
-        return Reply(id, group, server, owner, command, status, size, hash(hash), fileType, cursor, version, more!=0,
-            String(remote, Charsets.UTF_8), body)
     }
     data class Entry(val file: String, val mtime: Long)
     fun entries(reply: Reply): List<Entry> {
-        val b = buffer(reply.payload); val result = mutableListOf<Entry>()
+        return MemberFilesProtocol.entries(reply.payload).filter { it.type == 1 && validFile(it.name) && it.size in 1..MAX_MANIFEST }
-        while (b.hasRemaining()) {
+            .map { Entry(it.name, it.mtime) }
-            require(b.remaining() >= 35) { "truncated list entry" }
-            val length = b.short.toInt() and 65535; val type = b.get().toInt() and 255
-            val size = b.long; val mtime = b.long; b.position(b.position() + 16)
-            require(length in 1..255 && length <= b.remaining() && type in 1..2) { "invalid list entry" }
-            val name = String(ByteArray(length).also { b.get(it) }, Charsets.UTF_8)
-            if (type == 1 && validFile(name) && size in 1..MAX_MANIFEST) result += Entry(name, mtime)
-        }
-        return result
     }
     fun transfer(data: ByteArray): Reply? {
-        require(data.size == 51) { "transfer length=${data.size}" }
+        val r = MemberFilesProtocol.progress(data)
-        val b = buffer(data)
+        return if (r.finished) Reply(r.id, r.group, r.server, r.owner, status = r.status, size = r.total) else null
-        val id = b.long; val group = b.long; val server = b.long; val owner = b.long
-        val done = b.long; val total = b.long; val status = b.short.toInt() and 65535
-        val state = b.get().toInt()
-        require(state in 1..2 && done >= 0 && total >= 0 && done <= total) { "invalid transfer progress" }
-        return if (state == 2) Reply(id, group, server, owner, status = status, size = total) else null
     }
     fun published(data: ByteArray): Reply {
         require(data.size == 17 && data[16].toInt() in 0..1) { "invalid publication result" }

tools/chatgui-android/app/src/main/java/com/utun/chat/data/AvatarRepository.kt

@@ -14,7 +14,6 @@ import kotlinx.coroutines.sync.withPermit
 import java.io.File
 import java.io.IOException
 import java.security.MessageDigest
-import java.util.concurrent.atomic.AtomicLong
 
 data class AvatarMemberKey(val group: String, val owner: Long)
 data class AvatarStorage(val group: String, val nodeId: Long, val name: String)
@@ -28,7 +27,6 @@ class AvatarRepository(context: Context) {
     private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate)
     private val transfers = Semaphore(2)
     private val preparations = Semaphore(2)
-    private val ids = AtomicLong(System.nanoTime())
     private var generation = 0L
     private var account = 0L
     private val prefs = app.getSharedPreferences("avatars", Context.MODE_PRIVATE)
@@ -197,7 +195,7 @@ class AvatarRepository(context: Context) {
 
     private suspend fun request(type: Int, group: String, server: Long = 0, owner: Long = 0, command: Int = 0,
                                 enqueue: (Long, Long) -> Boolean): ByteArray {
-        val id = ids.incrementAndGet(); val gid = AvatarProtocol.groupId(group)
+        val id = MemberFilesProtocol.nextId(); val gid = AvatarProtocol.groupId(group)
         require(gid != 0L) { "invalid decimal channel_id=$group" }
         val result = CompletableDeferred<ByteArray>()
         val operation = Pending(type, gid, server, owner, command, result)
@@ -313,7 +311,7 @@ class AvatarRepository(context: Context) {
                 }
                 if (!hit) {
                     withContext(Dispatchers.IO) { if (!root.isDirectory && !root.mkdirs()) throw IOException("cannot create avatar cache") }
-                    val temp=File(root,".download-${ids.incrementAndGet()}")
+                    val temp=File(root,".download-${MemberFilesProtocol.nextId()}")
                     try {
                         val reply=AvatarProtocol.transfer(request(50,group,m.avatarServer,m.nodeId) { id,gid ->
                             NativeLib.transferAvatar(id,gid,m.avatarServer,m.nodeId,"avatars/$file",temp.path,maxBytes=limit)

tools/chatgui-android/app/src/main/java/com/utun/chat/data/MemberFilesProtocol.kt

@@ -0,0 +1,112 @@
+package com.utun.chat.data
+
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import java.nio.charset.CodingErrorAction
+import java.util.concurrent.atomic.AtomicLong
+
+/** Общий wire-кодек файлов и аватарок. UID хранится как исходные 64 бита. */
+object MemberFilesProtocol {
+    const val LIST = 1
+    const val STAT = 2
+    const val MKDIR = 8
+    const val RENAME = 9
+    const val REMOVE = 10
+    const val QUOTA = 16
+    const val FILE = 1
+    const val DIRECTORY = 2
+    const val STALE = 7
+    const val BUSY = 6
+    const val NOT_FOUND = 2
+    const val CANCELLED = 12
+    private val ids = AtomicLong(System.nanoTime())
+    fun nextId(): Long { var id = ids.incrementAndGet(); if (id == 0L) id = ids.incrementAndGet(); return id }
+    private fun buffer(data: ByteArray) = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN)
+    private fun utf8(data: ByteArray): String = Charsets.UTF_8.newDecoder()
+        .onMalformedInput(CodingErrorAction.REPORT).onUnmappableCharacter(CodingErrorAction.REPORT).decode(ByteBuffer.wrap(data)).toString()
+
+    fun validPath(path: String, root: Boolean = true): Boolean {
+        if (path.isEmpty()) return root
+        if (path.toByteArray().size > 1024 || path.any { it.isSurrogate() } &&
+            runCatching { utf8(path.toByteArray()) != path }.getOrDefault(true)) return false
+        val components = path.split('/')
+        return components.size <= 60 && components.all { name ->
+            name.isNotEmpty() && name.toByteArray().size <= 255 && !name.startsWith('.') &&
+                !name.endsWith('.') && !name.endsWith(' ') && name.none { it.code < 32 || it.code == 127 || it in "\\:*?\"<>|" } &&
+                !Regex("(?i)(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])").matches(name.substringBefore('.'))
+        }
+    }
+    fun servicePath(path: String) = path.substringBefore('/').equals("avatars", true)
+    fun child(path: String, name: String): String {
+        require(!name.contains('/') && validPath(name, false)) { "invalid file name" }
+        return (if (path.isEmpty()) name else "$path/$name").also { require(validPath(it, false)) { "invalid path" } }
+    }
+
+    data class Reply(val id: Long, val group: Long, val server: Long, val owner: Long, val command: Int, val status: Int,
+                     val size: Long, val mtime: Long, val hash: ByteArray, val type: Int, val cursor: Long,
+                     val version: ByteArray, val more: Boolean, val path: String, val payload: ByteArray)
+    data class Entry(val name: String, val type: Int, val size: Long, val mtime: Long, val version: ByteArray)
+    data class Progress(val id: Long, val group: Long, val server: Long, val owner: Long, val done: Long,
+                        val total: Long, val status: Int, val finished: Boolean)
+    data class Quota(val used: Long, val limit: Long, val reserved: Long, val files: Long, val directories: Long)
+
+    fun request(id: Long, owner: Long, command: Int, path: String, target: String = "", cursor: Long = 0,
+                version: ByteArray = ByteArray(16)): ByteArray {
+        require(id != 0L && command in listOf(LIST, STAT, MKDIR, RENAME, REMOVE, QUOTA))
+        require(validPath(path, command in listOf(LIST, STAT, QUOTA)) && version.size == 16 && cursor >= 0)
+        require(if (command == RENAME) validPath(target, false) else target.isEmpty())
+        val a = path.toByteArray(); val b = target.toByteArray()
+        return ByteBuffer.allocate(125 + a.size + b.size).order(ByteOrder.LITTLE_ENDIAN).apply {
+            put(4); put(1); put(command.toByte()); position(5); putLong(id); putLong(owner)
+            position(37); putLong(cursor); put(version); position(119)
+            putShort(a.size.toShort()); putShort(b.size.toShort()); putShort(0); put(a); put(b)
+        }.array()
+    }
+
+    fun response(data: ByteArray): Reply {
+        require(data.size >= 141) { "file response length=${data.size}" }
+        val b = buffer(data); val group = b.long; val server = b.long
+        require(b.get().toInt() == 4 && b.get().toInt() == 2) { "invalid file response protocol/kind" }
+        val command = b.get().toInt() and 255; val status = b.short.toInt() and 65535
+        val id = b.long; val owner = b.long; b.long; val size = b.long; val cursor = b.long
+        val version = ByteArray(16).also { b.get(it) }; b.position(93)
+        val hash = ByteArray(32).also { b.get(it) }; val mtime = b.long
+        val type = b.get().toInt() and 255; val more = b.get().toInt() and 255
+        val a = b.short.toInt() and 65535; val target = b.short.toInt() and 65535; val length = b.short.toInt() and 65535
+        require(id != 0L && command in 1..16 && status in 0..14 && more in 0..1 && a <= 1024 && target <= 1024 &&
+            length <= 8192 && b.remaining() == a + target + length) { "invalid file response header" }
+        val pathBytes = ByteArray(a).also { b.get(it) }; val targetBytes = ByteArray(target).also { b.get(it) }
+        require(0.toByte() !in pathBytes && 0.toByte() !in targetBytes) { "NUL in file path" }
+        val path = utf8(pathBytes); utf8(targetBytes)
+        return Reply(id, group, server, owner, command, status, size, mtime, hash, type, cursor, version, more != 0,
+            path, ByteArray(length).also { b.get(it) })
+    }
+
+    fun entries(data: ByteArray): List<Entry> {
+        val b = buffer(data); val entries = mutableListOf<Entry>(); val names = mutableSetOf<String>()
+        while (b.hasRemaining()) {
+            require(b.remaining() >= 35) { "truncated directory entry" }
+            val length = b.short.toInt() and 65535; val type = b.get().toInt() and 255
+            val size = b.long; val mtime = b.long; val version = ByteArray(16).also { b.get(it) }
+            require(length in 1..255 && length <= b.remaining() && type in FILE..DIRECTORY && size >= 0 && mtime >= 0)
+            val name = utf8(ByteArray(length).also { b.get(it) })
+            require(!name.contains('/') && validPath(name, false) && names.add(name)) { "invalid/duplicate directory name" }
+            entries += Entry(name, type, size, mtime, version)
+        }
+        return entries
+    }
+
+    fun progress(data: ByteArray): Progress {
+        require(data.size == 51) { "transfer length=${data.size}" }
+        val b = buffer(data); val id = b.long; val group = b.long; val server = b.long; val owner = b.long
+        val done = b.long; val total = b.long; val status = b.short.toInt() and 65535; val state = b.get().toInt()
+        require(id != 0L && state in 1..2 && status in 0..14 && done >= 0 && total >= 0 && done <= total)
+        return Progress(id, group, server, owner, done, total, status, state == 2)
+    }
+    fun quota(data: ByteArray): Quota {
+        require(data.size == 40) { "quota length=${data.size}" }
+        val b = buffer(data); val q = Quota(b.long, b.long, b.long, b.long, b.long)
+        require(q.used >= 0 && q.limit > 0 && q.reserved >= 0 && q.files >= 0 && q.directories >= 0)
+        return q
+    }
+}