Decode all supported video formats from uAV2 on desktop

tools/chatgui-android/app/src/main/java/com/utun/chat/data/StreamCaptureEngine.kt

@@ -16,6 +16,7 @@ import android.media.AudioTimestamp
 import android.media.MediaCodec
 import android.media.MediaCodecInfo
 import android.media.MediaFormat
+import android.os.Build
 import android.media.MediaRecorder
 import android.os.Bundle
 import android.os.Handler

tools/chatgui/src/streamplayer_window.cpp

@@ -293,16 +293,39 @@ void StreamPlayerWindow::decode() {
             deadline = nowUs() + 10000000;
         }
         if (waitKey && !(record.flags & STREAM_MEDIA_KEY)) continue;
+        const uint8_t* data = reinterpret_cast<const uint8_t*>(record.data.constData());
+        const uint8_t* configuration = nullptr; size_t size = (size_t)record.data.size(), configurationSize = 0;
+        if (record.flags & STREAM_MEDIA_KEY) {
+            if (stream_media_key_parse(data, size, &configuration, &configurationSize, &data, &size) != STREAM_OK) {
+                fail(text("Stream video decoder failed")); break;
+            }
+        }
         if (!decoder) {
-            const AVCodec* codec = avcodec_find_decoder(AV_CODEC_ID_H264);
+            AVCodecID codecId = record.format.video_codec == STREAM_MEDIA_H264 ? AV_CODEC_ID_H264
+                : record.format.video_codec == STREAM_MEDIA_H265 ? AV_CODEC_ID_HEVC
+                : record.format.video_codec == STREAM_MEDIA_VP9 ? AV_CODEC_ID_VP9
+                : record.format.video_codec == STREAM_MEDIA_AV1 ? AV_CODEC_ID_AV1 : AV_CODEC_ID_NONE;
+            if (record.format.video_codec == STREAM_MEDIA_H266) {
+                const AVCodecDescriptor* descriptor = avcodec_descriptor_get_by_name("vvc");
+                if (descriptor) codecId = descriptor->id;
+            }
+            const AVCodec* codec = avcodec_find_decoder(codecId);
             decoder = codec ? avcodec_alloc_context3(codec) : nullptr;
-            if (decoder) { decoder->thread_count = 1; decoder->flags |= AV_CODEC_FLAG_LOW_DELAY; }
+            if (decoder) {
+                decoder->thread_count = 1; decoder->flags |= AV_CODEC_FLAG_LOW_DELAY;
+                if (configurationSize) {
+                    decoder->extradata = static_cast<uint8_t*>(av_mallocz(configurationSize + AV_INPUT_BUFFER_PADDING_SIZE));
+                    if (!decoder->extradata) { fail(text("Stream decoder allocation failed")); break; }
+                    std::memcpy(decoder->extradata, configuration, configurationSize); decoder->extradata_size = (int)configurationSize;
+                }
+            }
             if (!decoder || avcodec_open2(decoder, codec, nullptr) < 0) { fail(text("Stream video decoder failed")); break; }
+            DEBUG_INFO(DEBUG_CATEGORY_STREAM, "desktop decoder codec=%u name=%s config=%zu", record.format.video_codec, codec->name, configurationSize);
             decoder->pkt_timebase = AVRational{1, 1000000}; waitKey = false;
         }
         av_packet_unref(packet);
-        if (av_new_packet(packet, record.data.size()) < 0) { fail(text("Stream decoder allocation failed")); break; }
-        std::memcpy(packet->data, record.data.constData(), (size_t)record.data.size());
+        if (av_new_packet(packet, (int)size) < 0) { fail(text("Stream decoder allocation failed")); break; }
+        std::memcpy(packet->data, data, size);
         packet->pts = packet->dts = (int64_t)record.pts;
         int sent = avcodec_send_packet(decoder, packet);
         if (sent < 0) { DEBUG_WARN(DEBUG_CATEGORY_STREAM, "desktop H264 packet rejected result=%d", sent); reset(true); continue; }