Distinguish Rust paths and embedded markup in language detection
This commit is contained in:
evgeny committed
1 parent
032b084a15
commit
99b4bea3f7
2 files changed
+27
-14
No files matched your search
@@ -14,7 +14,9 @@ struct Rule {
|
||||
|
||||
// Каждый признак учитывается один раз: повторение общего слова не повышает уверенность.
|
||||
const Rule rules[] = {
|
||||
{C, 3, QRegularExpression(R"(^\s*#\s*include\s*[<][\w/]+\.h[>])", QRegularExpression::MultilineOption)},
|
||||
{C, 3, QRegularExpression(R"(^\s*#\s*include[ \t]+)", QRegularExpression::MultilineOption)},
|
||||
{C, 2, QRegularExpression(R"(^\s*#\s*(?:define|ifdef|ifndef)\s+\w+)", QRegularExpression::MultilineOption)},
|
||||
{C, 2, QRegularExpression(R"(\bunsigned\s+(?:(?:long|int|char)\s+)?\w+\s*[;=])")},
|
||||
{C, 4, QRegularExpression(R"(\b(?:void|int|char|double|float|size_t)\s+\**\s*\w+\s*\([^;{}]*\)\s*\{)")},
|
||||
{C, 2, QRegularExpression(R"(\b(?:typedef\s+(?:struct|enum)|malloc\s*\(|calloc\s*\(|sizeof\s*\())")},
|
||||
{C, 2, QRegularExpression(R"(\b(?:const\s+)?(?:char|void|int)\s*\*+\s*\w+)")},
|
||||
@@ -23,30 +25,32 @@ const Rule rules[] = {
|
||||
QRegularExpression::MultilineOption)},
|
||||
{Cpp, 6, QRegularExpression(R"(\btemplate\s*<|\bnamespace\s+[\w:]+\s*\{)")},
|
||||
{Cpp, 4, QRegularExpression(R"(\b(?:class|struct)\s+\w+(?:\s*:\s*[^\n{]+)?\s*\{)")},
|
||||
{Cpp, 3, QRegularExpression(R"(\b(?:public|private|protected)\s*:|\b\w+::~?\w+\s*\()")} ,
|
||||
{Cpp, 3, QRegularExpression(R"(\b(?:public|private|protected)\s*:|\b\w+::~?\w+\s*\()")},
|
||||
{Java, 3, QRegularExpression(R"(^\s*(?:package|import)\s+(?:static\s+)?[\w.*]+\s*;)", QRegularExpression::MultilineOption)},
|
||||
{Java, 4, QRegularExpression(R"(\b(?:public|protected|private)\s+(?:(?:static|final|abstract)\s+)*(?:class|interface|enum)\s+\w+)")},
|
||||
{Java, 5, QRegularExpression(R"(\b(?:public|protected|private)\s+(?:(?:static|final|synchronized)\s+)*(?:void|boolean|String|int|long)\s+\w+\s*\()")} ,
|
||||
{Java, 5, QRegularExpression(R"(\b(?:public|protected|private)\s+(?:(?:static|final|synchronized)\s+)*)"
|
||||
R"((?:void|boolean|String|int|long)\s+\w+\s*\()")},
|
||||
{Java, 4, QRegularExpression(R"(\b(?:implements|extends)\s+[A-Z]\w*|\bSystem\.(?:out|err)\.)")},
|
||||
{Kotlin, 6, QRegularExpression(R"(\bfun\s+(?:<[^>\n]+>\s*)?[\w.]+\s*\()")} ,
|
||||
{Kotlin, 6, QRegularExpression(R"(\bfun\s+(?:<[^>\n]+>\s*)?[\w.]+\s*\()")},
|
||||
{Kotlin, 3, QRegularExpression(R"(^\s*(?:(?:private|public|internal|override|const)\s+)*(?:val|var)\s+\w+\s*[:=])",
|
||||
QRegularExpression::MultilineOption)},
|
||||
{Kotlin, 5, QRegularExpression(R"(\b(?:data\s+class|sealed\s+class|companion\s+object)\b|\bwhen\s*\([^\n]*\)\s*\{)")},
|
||||
{Rust, 7, QRegularExpression(R"(\bfn\s+\w+\s*(?:<[^>\n]+>)?\s*\()")} ,
|
||||
{Rust, 5, QRegularExpression(R"(^\s*(?:pub(?:\([^)]*\))?\s+)?(?:use\s+[\w:]+|impl(?:\s*<[^>\n]+>)?\s+\w+))",
|
||||
{Rust, 7, QRegularExpression(R"(\bfn\s+\w+\s*(?:<[^>\n]+>)?\s*\()")},
|
||||
{Rust, 7, QRegularExpression(R"(^\s*(?:pub(?:\([^)]*\))?\s+)?(?:use\s+[\w:]+\s*(?:;|::|\{)|impl(?:\s*<[^>\n]+>)?\s+\w+))",
|
||||
QRegularExpression::MultilineOption)},
|
||||
{Rust, 3, QRegularExpression(R"(\blet\s+mut\s+|\b(?:println|vec|assert_eq|format)!\s*[(\[]|#\[derive\s*\()")} ,
|
||||
{Rust, 3, QRegularExpression(R"(\blet\s+mut\s+|\b(?:println|vec|assert_eq|format)!\s*[(\[]|#\[derive\s*\()")},
|
||||
{Php, 4, QRegularExpression(R"(\$[a-zA-Z_]\w*\s*(?:->|\[|=))")},
|
||||
{Php, 4, QRegularExpression(R"(\bfunction\s+\w+\s*\([^\n)]*\$|\b(?:public|private|protected)\s+(?:static\s+)?function\b)")},
|
||||
{Php, 3, QRegularExpression(R"(\b(?:echo|require_once|include_once)\b|\barray\s*\(|\$this\b)")},
|
||||
{JavaScript, 4, QRegularExpression(R"(\b(?:const|let|var)\s+[a-zA-Z_$][\w$]*\s*=)")},
|
||||
{JavaScript, 3, QRegularExpression(R"(=>|\bfunction\s*[\w$]*\s*\()")} ,
|
||||
{JavaScript, 3, QRegularExpression(R"(=>|\bfunction\s*[\w$]*\s*\()")},
|
||||
{JavaScript, 4, QRegularExpression(R"(\b(?:module\.exports|exports\.\w+|console\.\w+|document\.\w+)\b)")},
|
||||
{JavaScript, 4, QRegularExpression(R"(^\s*(?:export\s+(?:default|function|const|class)\b|import\s+[^;\n]+\s+from\b))",
|
||||
QRegularExpression::MultilineOption)},
|
||||
{Css, 3, QRegularExpression(R"((?:^|[}\n])\s*[.#]?[\w*-][\w\s.#:>+~,*\[\]="'-]*\s*\{)")},
|
||||
{Css, 4, QRegularExpression(R"((?:^|[;{\n])\s*(?:color|background(?:-color)?|font(?:-size|-family|-weight)?|display|position)\s*:[^;{}\n]+;)")},
|
||||
{Css, 4, QRegularExpression(R"((?:^|[;{\n])\s*(?:margin(?:-\w+)?|padding(?:-\w+)?|border(?:-\w+)?|width|height|opacity|align-items)\s*:[^;{}\n]+;)")},
|
||||
{Css, 4, QRegularExpression(R"((?:^|[;{\n])\s*(?:margin(?:-\w+)?|padding(?:-\w+)?|border(?:-\w+)?|width|height|opacity|align-items))"
|
||||
R"(\s*:[^;{}\n]+;)")},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -73,7 +77,9 @@ QString detectCodeLanguage(QStringView source) {
|
||||
// Фрагменты HTML без doctype: нужен парный известный тег; JSX с объявлениями JS не выбираем как HTML.
|
||||
static const QRegularExpression tags(R"(<(div|span|p|a|ul|li|table|section|form|button|h[1-6])\b[^<>]*>[\s\S]*?</\1\s*>)",
|
||||
QRegularExpression::CaseInsensitiveOption);
|
||||
if (scores[JavaScript] == 0 && tags.match(code).hasMatch()) scores[Html] = 7;
|
||||
if (scores[JavaScript] == 0 && tags.match(code).hasMatch()) { scores[Html] = 7; scores[Css] = 0; }
|
||||
// std:: и Type::method встречаются также в Rust; use/fn/impl отличают его от C++.
|
||||
if (scores[Rust] >= 7) scores[Cpp] = 0;
|
||||
// C является подмножеством C++: специфичные конструкции C++ имеют приоритет.
|
||||
if (scores[Cpp] >= 6) scores[C] = 0;
|
||||
int best = 0, second = 0;
|
||||
|
||||
@@ -20,12 +20,15 @@ private slots:
|
||||
QTest::newRow("java") << "package example;\npublic class Hello { public static void main(String[] args) {} }" << "Java";
|
||||
QTest::newRow("kotlin") << "fun main() { println(\"hello\") }" << "Kotlin";
|
||||
QTest::newRow("rust") << "fn main() { println!(\"hello\"); }" << "Rust";
|
||||
QTest::newRow("rust-std") << "use std::{borrow::Cow, collections::BTreeSet};" << "Rust";
|
||||
QTest::newRow("c-macros") << "#include \"local.h\"\n#define COUNT 4\nunsigned long value;" << "C";
|
||||
QTest::newRow("php") << "<?php\necho 'hello';" << "PHP/PHP";
|
||||
QTest::newRow("php-fragment") << "function hello($name) { $message = $name; return $message; }" << "PHP/PHP";
|
||||
QTest::newRow("javascript") << "const greet = (name) => console.log(name);" << "JavaScript";
|
||||
QTest::newRow("css") << ".hello { color: red; margin: 1px; }" << "CSS";
|
||||
QTest::newRow("html") << "<!doctype html>\n<html><script>const f = () => 1;</script></html>" << "HTML";
|
||||
QTest::newRow("html-fragment") << "<div class=\"hello\"><span>Hello</span></div>" << "HTML";
|
||||
QTest::newRow("html-css") << "<style>.x { color: red; margin: 1px; }</style><p>Hello</p>" << "HTML";
|
||||
QTest::newRow("empty") << "" << "";
|
||||
QTest::newRow("ambiguous") << "int x = 42;" << "";
|
||||
QTest::newRow("prose") << "Java and Kotlin have class and function declarations; use CSS for color." << "";
|
||||
@@ -54,7 +57,7 @@ private slots:
|
||||
const auto bytes = source.readAll();
|
||||
QCOMPARE(QString::fromLatin1(QCryptographicHash::hash(bytes, QCryptographicHash::Sha256).toHex()), entry["sha256"].toString());
|
||||
const QString text = QString::fromUtf8(bytes), expected = entry["language"].toString();
|
||||
const auto check = [&](const QString& code, const char* kind, Counts& counts) {
|
||||
const auto check = [&](const QString& code, const QString& expected, const char* kind, Counts& counts) {
|
||||
const QString actual = detectCodeLanguage(code);
|
||||
if (actual == expected) ++counts.correct;
|
||||
else if (actual.isEmpty()) ++counts.unknown;
|
||||
@@ -62,11 +65,15 @@ private slots:
|
||||
GUI_INFO("corpus: file=%s kind=%s expected=%s actual=%s", qPrintable(path), kind,
|
||||
qPrintable(expected), actual.isEmpty() ? "unknown" : qPrintable(actual));
|
||||
};
|
||||
check(text, "full", full);
|
||||
check(text, expected, "full", full);
|
||||
const auto lines = text.split('\n');
|
||||
// Фиксированные окна, не выбираемые по результату: начало, середина, конец по 20 строк.
|
||||
for (int start : {0, qMax(0, int(lines.size()) / 2 - 10), qMax(0, int(lines.size()) - 20)})
|
||||
check(lines.mid(start, 20).join('\n'), "snippet", snippets);
|
||||
int window = 0;
|
||||
const auto labels = entry["snippets"].toArray();
|
||||
for (int start : {0, qMax(0, int(lines.size()) / 2 - 10), qMax(0, int(lines.size()) - 20)}) {
|
||||
const QString label = labels.isEmpty() ? expected : labels[window].toString();
|
||||
check(lines.mid(start, 20).join('\n'), label, "snippet", snippets); ++window;
|
||||
}
|
||||
}
|
||||
GUI_INFO("corpus totals: full correct=%d unknown=%d wrong=%d; snippets correct=%d unknown=%d wrong=%d",
|
||||
full.correct, full.unknown, full.wrong, snippets.correct, snippets.unknown, snippets.wrong);
|
||||
|
||||
Reference in new issue
Block a user