Separate hardware and software live encoder selection

tools/chatgui/src/live_video_encoder.cpp

@@ -53,6 +53,10 @@ QList<QByteArray> encoders(int codec) {
     default: return {};
     }
 }
+bool matchesBackend(const QByteArray& name, LiveVideoEncoder::Backend backend) {
+    bool hardware = name.endsWith("_nvenc") || name.endsWith("_qsv") || name.endsWith("_amf") || name.endsWith("_vaapi");
+    return backend == LiveVideoEncoder::Backend::Automatic || hardware == (backend == LiveVideoEncoder::Backend::HardwareOnly);
+}
 AVPixelFormat softwareFormat(const AVCodec* codec) {
     const AVPixelFormat* formats = nullptr;
 #if LIBAVCODEC_VERSION_MAJOR >= 61
@@ -106,7 +110,7 @@ bool verifiedProfile(const LiveVideoConfig& config, const QList<LiveEncodedFrame
 }
 #endif
 
-bool LiveVideoEncoder::open(const LiveVideoConfig& config) {
+bool LiveVideoEncoder::open(const LiveVideoConfig& config, Backend backend) {
     close();
     bool constrained = config.profile != LiveVideoProfile::Automatic;
     if (config.width < 16 || config.height < 16 || config.width > 4096 || config.height > 4096
@@ -120,6 +124,7 @@ bool LiveVideoEncoder::open(const LiveVideoConfig& config) {
     }
 #ifdef HAVE_FFMPEG
     for (const auto& name : encoders(config.codec)) {
+        if (!matchesBackend(name, backend)) continue;
         if (!config.backend.isEmpty() && name != config.backend.toLatin1()) continue;
         if (constrained && name != "libx264" && name != "libx265") continue;
         const AVCodec* encoder = avcodec_find_encoder_by_name(name.constData());
@@ -212,11 +217,13 @@ bool LiveVideoEncoder::open(const LiveVideoConfig& config) {
                 QImage probe(config.width, config.height, QImage::Format_RGB32); probe.fill(Qt::black);
                 QList<LiveEncodedFrame> output;
                 if (!encode(probe, 0, true, output) || output.isEmpty() || !output.front().key) {
+                    if (m->error.isEmpty()) m->error = "Encoder did not produce an immediate key frame";
                     DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "desktop encoder probe lacks immediate KEY name=%s error=%s", name.constData(), qPrintable(m->error));
                     break;
                 }
                 output.clear();
                 if (!encode(probe, 1000000 / config.fps, true, output) || output.isEmpty() || !output.front().key) {
+                    if (m->error.isEmpty()) m->error = "Encoder cannot produce an independent key frame on request";
                     DEBUG_WARN(DEBUG_CATEGORY_VIDEO, "desktop encoder cannot force an independent KEY name=%s error=%s", name.constData(), qPrintable(m->error));
                     break;
                 }
@@ -236,7 +243,8 @@ bool LiveVideoEncoder::open(const LiveVideoConfig& config) {
 #else
     m->error = "FFmpeg is unavailable";
 #endif
-    DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop encoder open failed: %s", qPrintable(m->error)); return false;
+    DEBUG_ERROR(DEBUG_CATEGORY_VIDEO, "desktop encoder open failed codec=%d backend=%d: %s",
+        config.codec, int(backend), qPrintable(m->error)); return false;
 }
 
 bool LiveVideoEncoder::encode(const QImage& image, uint64_t pts, bool key, QList<LiveEncodedFrame>& output) {
@@ -309,17 +317,18 @@ bool LiveVideoEncoder::encode(const QImage& image, uint64_t pts, bool key, QList
 #endif
 }
 
-QList<int> LiveVideoEncoder::available() {
+QList<int> LiveVideoEncoder::available(Backend backend) {
     QList<int> result;
-    for (int codec = 1; codec <= 5; codec++) {
 #ifdef HAVE_FFMPEG
-        bool installed = false;
-        for (const auto& name : encoders(codec)) if (avcodec_find_encoder_by_name(name.constData())) { installed = true; break; }
-        if (!installed) { DEBUG_DEBUG(DEBUG_CATEGORY_VIDEO, "desktop codec has no installed encoder codec=%d", codec); continue; }
-#endif
-        LiveVideoEncoder encoder; LiveVideoConfig config; config.width = 320; config.height = 180; config.codec = codec;
-        if (encoder.open(config)) result.push_back(codec);
+    for (int codec = 1; codec <= 5; codec++) {
+        for (const auto& name : encoders(codec)) {
+            if (!matchesBackend(name, backend) || !avcodec_find_encoder_by_name(name.constData())) continue;
+            result.push_back(codec); break;
+        }
     }
+#else
+    Q_UNUSED(backend);
+#endif
     return result;
 }