diff --git a/tools/chatgui/src/code_language.cpp b/tools/chatgui/src/code_language.cpp index 914d6ff09..ef516d2e1 100644 --- a/tools/chatgui/src/code_language.cpp +++ b/tools/chatgui/src/code_language.cpp @@ -14,7 +14,9 @@ struct Rule { // Каждый признак учитывается один раз: повторение общего слова не повышает уверенность. const Rule rules[] = { - {C, 3, QRegularExpression(R"(^\s*#\s*include\s*[<][\w/]+\.h[>])", QRegularExpression::MultilineOption)}, + {C, 3, QRegularExpression(R"(^\s*#\s*include[ \t]+)", QRegularExpression::MultilineOption)}, + {C, 2, QRegularExpression(R"(^\s*#\s*(?:define|ifdef|ifndef)\s+\w+)", QRegularExpression::MultilineOption)}, + {C, 2, QRegularExpression(R"(\bunsigned\s+(?:(?:long|int|char)\s+)?\w+\s*[;=])")}, {C, 4, QRegularExpression(R"(\b(?:void|int|char|double|float|size_t)\s+\**\s*\w+\s*\([^;{}]*\)\s*\{)")}, {C, 2, QRegularExpression(R"(\b(?:typedef\s+(?:struct|enum)|malloc\s*\(|calloc\s*\(|sizeof\s*\())")}, {C, 2, QRegularExpression(R"(\b(?:const\s+)?(?:char|void|int)\s*\*+\s*\w+)")}, @@ -23,30 +25,32 @@ const Rule rules[] = { QRegularExpression::MultilineOption)}, {Cpp, 6, QRegularExpression(R"(\btemplate\s*<|\bnamespace\s+[\w:]+\s*\{)")}, {Cpp, 4, QRegularExpression(R"(\b(?:class|struct)\s+\w+(?:\s*:\s*[^\n{]+)?\s*\{)")}, - {Cpp, 3, QRegularExpression(R"(\b(?:public|private|protected)\s*:|\b\w+::~?\w+\s*\()")} , + {Cpp, 3, QRegularExpression(R"(\b(?:public|private|protected)\s*:|\b\w+::~?\w+\s*\()")}, {Java, 3, QRegularExpression(R"(^\s*(?:package|import)\s+(?:static\s+)?[\w.*]+\s*;)", QRegularExpression::MultilineOption)}, {Java, 4, QRegularExpression(R"(\b(?:public|protected|private)\s+(?:(?:static|final|abstract)\s+)*(?:class|interface|enum)\s+\w+)")}, - {Java, 5, QRegularExpression(R"(\b(?:public|protected|private)\s+(?:(?:static|final|synchronized)\s+)*(?:void|boolean|String|int|long)\s+\w+\s*\()")} , + {Java, 5, QRegularExpression(R"(\b(?:public|protected|private)\s+(?:(?:static|final|synchronized)\s+)*)" + R"((?:void|boolean|String|int|long)\s+\w+\s*\()")}, {Java, 4, QRegularExpression(R"(\b(?:implements|extends)\s+[A-Z]\w*|\bSystem\.(?:out|err)\.)")}, - {Kotlin, 6, QRegularExpression(R"(\bfun\s+(?:<[^>\n]+>\s*)?[\w.]+\s*\()")} , + {Kotlin, 6, QRegularExpression(R"(\bfun\s+(?:<[^>\n]+>\s*)?[\w.]+\s*\()")}, {Kotlin, 3, QRegularExpression(R"(^\s*(?:(?:private|public|internal|override|const)\s+)*(?:val|var)\s+\w+\s*[:=])", QRegularExpression::MultilineOption)}, {Kotlin, 5, QRegularExpression(R"(\b(?:data\s+class|sealed\s+class|companion\s+object)\b|\bwhen\s*\([^\n]*\)\s*\{)")}, - {Rust, 7, QRegularExpression(R"(\bfn\s+\w+\s*(?:<[^>\n]+>)?\s*\()")} , - {Rust, 5, QRegularExpression(R"(^\s*(?:pub(?:\([^)]*\))?\s+)?(?:use\s+[\w:]+|impl(?:\s*<[^>\n]+>)?\s+\w+))", + {Rust, 7, QRegularExpression(R"(\bfn\s+\w+\s*(?:<[^>\n]+>)?\s*\()")}, + {Rust, 7, QRegularExpression(R"(^\s*(?:pub(?:\([^)]*\))?\s+)?(?:use\s+[\w:]+\s*(?:;|::|\{)|impl(?:\s*<[^>\n]+>)?\s+\w+))", QRegularExpression::MultilineOption)}, - {Rust, 3, QRegularExpression(R"(\blet\s+mut\s+|\b(?:println|vec|assert_eq|format)!\s*[(\[]|#\[derive\s*\()")} , + {Rust, 3, QRegularExpression(R"(\blet\s+mut\s+|\b(?:println|vec|assert_eq|format)!\s*[(\[]|#\[derive\s*\()")}, {Php, 4, QRegularExpression(R"(\$[a-zA-Z_]\w*\s*(?:->|\[|=))")}, {Php, 4, QRegularExpression(R"(\bfunction\s+\w+\s*\([^\n)]*\$|\b(?:public|private|protected)\s+(?:static\s+)?function\b)")}, {Php, 3, QRegularExpression(R"(\b(?:echo|require_once|include_once)\b|\barray\s*\(|\$this\b)")}, {JavaScript, 4, QRegularExpression(R"(\b(?:const|let|var)\s+[a-zA-Z_$][\w$]*\s*=)")}, - {JavaScript, 3, QRegularExpression(R"(=>|\bfunction\s*[\w$]*\s*\()")} , + {JavaScript, 3, QRegularExpression(R"(=>|\bfunction\s*[\w$]*\s*\()")}, {JavaScript, 4, QRegularExpression(R"(\b(?:module\.exports|exports\.\w+|console\.\w+|document\.\w+)\b)")}, {JavaScript, 4, QRegularExpression(R"(^\s*(?:export\s+(?:default|function|const|class)\b|import\s+[^;\n]+\s+from\b))", QRegularExpression::MultilineOption)}, {Css, 3, QRegularExpression(R"((?:^|[}\n])\s*[.#]?[\w*-][\w\s.#:>+~,*\[\]="'-]*\s*\{)")}, {Css, 4, QRegularExpression(R"((?:^|[;{\n])\s*(?:color|background(?:-color)?|font(?:-size|-family|-weight)?|display|position)\s*:[^;{}\n]+;)")}, - {Css, 4, QRegularExpression(R"((?:^|[;{\n])\s*(?:margin(?:-\w+)?|padding(?:-\w+)?|border(?:-\w+)?|width|height|opacity|align-items)\s*:[^;{}\n]+;)")}, + {Css, 4, QRegularExpression(R"((?:^|[;{\n])\s*(?:margin(?:-\w+)?|padding(?:-\w+)?|border(?:-\w+)?|width|height|opacity|align-items))" + R"(\s*:[^;{}\n]+;)")}, }; } @@ -73,7 +77,9 @@ QString detectCodeLanguage(QStringView source) { // Фрагменты HTML без doctype: нужен парный известный тег; JSX с объявлениями JS не выбираем как HTML. static const QRegularExpression tags(R"(<(div|span|p|a|ul|li|table|section|form|button|h[1-6])\b[^<>]*>[\s\S]*?\1\s*>)", QRegularExpression::CaseInsensitiveOption); - if (scores[JavaScript] == 0 && tags.match(code).hasMatch()) scores[Html] = 7; + if (scores[JavaScript] == 0 && tags.match(code).hasMatch()) { scores[Html] = 7; scores[Css] = 0; } + // std:: и Type::method встречаются также в Rust; use/fn/impl отличают его от C++. + if (scores[Rust] >= 7) scores[Cpp] = 0; // C является подмножеством C++: специфичные конструкции C++ имеют приоритет. if (scores[Cpp] >= 6) scores[C] = 0; int best = 0, second = 0; diff --git a/tools/chatgui/tests/test_code_language.cpp b/tools/chatgui/tests/test_code_language.cpp index 030ab8f91..df8601ad5 100644 --- a/tools/chatgui/tests/test_code_language.cpp +++ b/tools/chatgui/tests/test_code_language.cpp @@ -20,12 +20,15 @@ private slots: QTest::newRow("java") << "package example;\npublic class Hello { public static void main(String[] args) {} }" << "Java"; QTest::newRow("kotlin") << "fun main() { println(\"hello\") }" << "Kotlin"; QTest::newRow("rust") << "fn main() { println!(\"hello\"); }" << "Rust"; + QTest::newRow("rust-std") << "use std::{borrow::Cow, collections::BTreeSet};" << "Rust"; + QTest::newRow("c-macros") << "#include \"local.h\"\n#define COUNT 4\nunsigned long value;" << "C"; QTest::newRow("php") << " console.log(name);" << "JavaScript"; QTest::newRow("css") << ".hello { color: red; margin: 1px; }" << "CSS"; QTest::newRow("html") << "\n" << "HTML"; QTest::newRow("html-fragment") << "
Hello
" << "HTML"; QTest::newRow("empty") << "" << ""; QTest::newRow("ambiguous") << "int x = 42;" << ""; QTest::newRow("prose") << "Java and Kotlin have class and function declarations; use CSS for color." << ""; @@ -54,7 +57,7 @@ private slots: const auto bytes = source.readAll(); QCOMPARE(QString::fromLatin1(QCryptographicHash::hash(bytes, QCryptographicHash::Sha256).toHex()), entry["sha256"].toString()); const QString text = QString::fromUtf8(bytes), expected = entry["language"].toString(); - const auto check = [&](const QString& code, const char* kind, Counts& counts) { + const auto check = [&](const QString& code, const QString& expected, const char* kind, Counts& counts) { const QString actual = detectCodeLanguage(code); if (actual == expected) ++counts.correct; else if (actual.isEmpty()) ++counts.unknown; @@ -62,11 +65,15 @@ private slots: GUI_INFO("corpus: file=%s kind=%s expected=%s actual=%s", qPrintable(path), kind, qPrintable(expected), actual.isEmpty() ? "unknown" : qPrintable(actual)); }; - check(text, "full", full); + check(text, expected, "full", full); const auto lines = text.split('\n'); // Фиксированные окна, не выбираемые по результату: начало, середина, конец по 20 строк. - for (int start : {0, qMax(0, int(lines.size()) / 2 - 10), qMax(0, int(lines.size()) - 20)}) - check(lines.mid(start, 20).join('\n'), "snippet", snippets); + int window = 0; + const auto labels = entry["snippets"].toArray(); + for (int start : {0, qMax(0, int(lines.size()) / 2 - 10), qMax(0, int(lines.size()) - 20)}) { + const QString label = labels.isEmpty() ? expected : labels[window].toString(); + check(lines.mid(start, 20).join('\n'), label, "snippet", snippets); ++window; + } } GUI_INFO("corpus totals: full correct=%d unknown=%d wrong=%d; snippets correct=%d unknown=%d wrong=%d", full.correct, full.unknown, full.wrong, snippets.correct, snippets.unknown, snippets.wrong);