From 032b084a1570dab01e068935f60479d228e80112 Mon Sep 17 00:00:00 2001 From: evgeny Date: Mon, 2 Nov 2026 23:00:45 +0100 Subject: [PATCH] Apply language detection to unlabelled code and add corpus runner --- tools/chatgui/CMakeLists.txt | 7 +- tools/chatgui/src/message_text.cpp | 12 ++-- tools/chatgui/tests/test_code_language.cpp | 77 ++++++++++++++++++++++ 3 files changed, 91 insertions(+), 5 deletions(-) create mode 100644 tools/chatgui/tests/test_code_language.cpp diff --git a/tools/chatgui/CMakeLists.txt b/tools/chatgui/CMakeLists.txt index 3fa167b9a..7db008bd0 100644 --- a/tools/chatgui/CMakeLists.txt +++ b/tools/chatgui/CMakeLists.txt @@ -41,7 +41,7 @@ else() set(CHAT_SYNTAX_LIBRARY KF5::SyntaxHighlighting) endif() -add_library(chatgui_message_text STATIC src/message_text.cpp src/message_links.cpp) +add_library(chatgui_message_text STATIC src/message_text.cpp src/message_links.cpp src/code_language.cpp) target_link_libraries(chatgui_message_text PUBLIC ${QT_LIBS} PRIVATE ${CHAT_SYNTAX_LIBRARY} utun) # Каталоги в ресурсах: перевод работает и в переносимой сборке без файлов рядом с exe. @@ -790,6 +790,11 @@ target_link_libraries(test_message_text PRIVATE chatgui_message_text ${QT_TEST_L add_test(NAME test_message_text COMMAND test_message_text) set_tests_properties(test_message_text PROPERTIES ENVIRONMENT "QT_QPA_PLATFORM=offscreen" TIMEOUT 30) +add_executable(test_code_language tests/test_code_language.cpp) +target_link_libraries(test_code_language PRIVATE chatgui_message_text ${QT_TEST_LIB}) +add_test(NAME test_code_language COMMAND test_code_language) +set_tests_properties(test_code_language PROPERTIES TIMEOUT 30) + add_executable(test_join_dialogs tests/test_join_dialogs.cpp src/joindialog.cpp src/inviteby.cpp src/invite_link.cpp "${CMAKE_CURRENT_BINARY_DIR}/translations.qrc") add_dependencies(test_join_dialogs chatgui_translations) diff --git a/tools/chatgui/src/message_text.cpp b/tools/chatgui/src/message_text.cpp index 7dccd0f33..8c8c0be62 100644 --- a/tools/chatgui/src/message_text.cpp +++ b/tools/chatgui/src/message_text.cpp @@ -1,4 +1,5 @@ #include "message_text.h" +#include "code_language.h" #include "messagedelegate.h" #include "debug_ui.h" #include @@ -165,15 +166,18 @@ QSharedPointer formattedMessageText(const QModelIndex& index) {"py", "Python"}, {"js", "JavaScript"}, {"ts", "TypeScript"}, {"sh", "Bash"}, {"shell", "Bash"}, {"cs", "C#"}, {"csharp", "C#"}, {"kt", "Kotlin"}, {"rs", "Rust"}, {"yml", "YAML"}}; for (const auto& block : message->code) { - if (block.language.isEmpty() || block.language == "text" || block.language == "plain") continue; - auto definition = repository.definitionForName(aliases.value(block.language, block.language)); + if (block.language == "text" || block.language == "plain") continue; + const QString language = block.language.isEmpty() + ? detectCodeLanguage(QStringView(message->text).mid(block.start, block.length)) : block.language; + if (language.isEmpty()) continue; + auto definition = repository.definitionForName(aliases.value(language, language)); if (!definition.isValid()) { for (const auto& candidate : repository.definitions()) { - if (candidate.name().compare(block.language, Qt::CaseInsensitive) == 0) { definition = candidate; break; } + if (candidate.name().compare(language, Qt::CaseInsensitive) == 0) { definition = candidate; break; } } } if (definition.isValid()) highlighter.highlight(block, definition); - else GUI_DEBUG("message code: unknown language=%s; using plain code", qPrintable(block.language)); + else GUI_DEBUG("message code: unknown language=%s; using plain code", qPrintable(language)); } } QSharedPointer prepared = message->changed ? message : QSharedPointer(); diff --git a/tools/chatgui/tests/test_code_language.cpp b/tools/chatgui/tests/test_code_language.cpp new file mode 100644 index 000000000..030ab8f91 --- /dev/null +++ b/tools/chatgui/tests/test_code_language.cpp @@ -0,0 +1,77 @@ +/* Язык определяется только по тексту. Необязательная внешняя выборка: CODE_LANGUAGE_CORPUS=/path/manifest.json. */ +#include "../src/code_language.h" +#include "../src/debug_ui.h" +#include +#include +#include +#include +#include +#include +#include + +class CodeLanguageTest : public QObject { + Q_OBJECT +private slots: + void initTestCase() { debug_enable_console(true); debug_set_category_level(DEBUG_CATEGORY_GENERAL, DEBUG_LEVEL_DEBUG); } + void samples_data() { + QTest::addColumn("text"); QTest::addColumn("language"); + QTest::newRow("c") << "#include \nint main(void) { puts(\"hello\"); }" << "C"; + QTest::newRow("cpp") << "#include \nint main() { std::cout << 42; }" << "C++"; + QTest::newRow("java") << "package example;\npublic class Hello { public static void main(String[] args) {} }" << "Java"; + QTest::newRow("kotlin") << "fun main() { println(\"hello\") }" << "Kotlin"; + QTest::newRow("rust") << "fn main() { println!(\"hello\"); }" << "Rust"; + QTest::newRow("php") << " console.log(name);" << "JavaScript"; + QTest::newRow("css") << ".hello { color: red; margin: 1px; }" << "CSS"; + QTest::newRow("html") << "\n" << "HTML"; + QTest::newRow("html-fragment") << "
Hello
" << "HTML"; + QTest::newRow("empty") << "" << ""; + QTest::newRow("ambiguous") << "int x = 42;" << ""; + QTest::newRow("prose") << "Java and Kotlin have class and function declarations; use CSS for color." << ""; + QTest::newRow("comments") << "/* fn main() { println!(\"hello\"); } */\n// std::vector x;" << ""; + QTest::newRow("strings") << "\"std::vector\"\n'fun main() {}'" << ""; + QTest::newRow("json") << "{\"code\": \"fn main() {}\", \"color\": \"red\"}" << ""; + QTest::newRow("conflicting") << "fn test() {}\nfun main() {}" << ""; + QTest::newRow("repeated-weak") << QString("const value = 1;\n").repeated(200) << ""; + QTest::newRow("bounded") << (QString(16384, ' ') + "std::cout << 42;") << ""; + QTest::newRow("crlf") << "#include \r\nint main(void) { return 0; }" << "C"; + } + void samples() { QFETCH(QString, text); QFETCH(QString, language); QCOMPARE(detectCodeLanguage(text), language); } + + void corpus() { + const QString manifest = qEnvironmentVariable("CODE_LANGUAGE_CORPUS"); + if (manifest.isEmpty()) QSKIP("Set CODE_LANGUAGE_CORPUS to run the downloaded GitHub corpus"); + QFile file(manifest); QVERIFY2(file.open(QIODevice::ReadOnly), qPrintable(file.errorString())); + QJsonParseError error; + const auto document = QJsonDocument::fromJson(file.readAll(), &error); + QCOMPARE(error.error, QJsonParseError::NoError); QVERIFY(document.isArray()); + const QDir directory = QFileInfo(manifest).absoluteDir(); + struct Counts { int correct = 0, unknown = 0, wrong = 0; } full, snippets; + for (const auto& item : document.array()) { + const auto entry = item.toObject(); const QString path = entry["file"].toString(); + QFile source(directory.filePath(path)); QVERIFY2(source.open(QIODevice::ReadOnly), qPrintable(path)); + const auto bytes = source.readAll(); + QCOMPARE(QString::fromLatin1(QCryptographicHash::hash(bytes, QCryptographicHash::Sha256).toHex()), entry["sha256"].toString()); + const QString text = QString::fromUtf8(bytes), expected = entry["language"].toString(); + const auto check = [&](const QString& code, const char* kind, Counts& counts) { + const QString actual = detectCodeLanguage(code); + if (actual == expected) ++counts.correct; + else if (actual.isEmpty()) ++counts.unknown; + else ++counts.wrong; + GUI_INFO("corpus: file=%s kind=%s expected=%s actual=%s", qPrintable(path), kind, + qPrintable(expected), actual.isEmpty() ? "unknown" : qPrintable(actual)); + }; + check(text, "full", full); + const auto lines = text.split('\n'); + // Фиксированные окна, не выбираемые по результату: начало, середина, конец по 20 строк. + for (int start : {0, qMax(0, int(lines.size()) / 2 - 10), qMax(0, int(lines.size()) - 20)}) + check(lines.mid(start, 20).join('\n'), "snippet", snippets); + } + GUI_INFO("corpus totals: full correct=%d unknown=%d wrong=%d; snippets correct=%d unknown=%d wrong=%d", + full.correct, full.unknown, full.wrong, snippets.correct, snippets.unknown, snippets.wrong); + QVERIFY(full.correct > 0); QCOMPARE(full.wrong, 0); QCOMPARE(snippets.wrong, 0); + } +}; +QTEST_GUILESS_MAIN(CodeLanguageTest) +#include "test_code_language.moc"