feat: tighten search grammar and use unicode case folding

YKanBan.Tests/Search/SearchQueryParserTests.cs

@@ -147,9 +147,10 @@ public class SearchQueryParserTests
     }
 
     [TestMethod]
-    public void QualifierValueMayContainColons()
+    public void QuotedQualifierValueMayContainColonsAndOperators()
     {
-        AssertParsesTo(new SearchNode.TagName("a:b"), "tag:a:b");
+        AssertParsesTo(new SearchNode.TagName("a:b"), "tag:\"a:b\"");
+        AssertParsesTo(new SearchNode.TagName("AND"), "tag:\"AND\"");
     }
 
     [TestMethod]
@@ -169,13 +170,20 @@ public class SearchQueryParserTests
     public void IdQualifierParsesNumber()
     {
         AssertParsesTo(new SearchNode.CardId(42), "id:42");
-        AssertParsesTo(new SearchNode.CardId(12), "id:\"12\"");
+        AssertParsesTo(new SearchNode.CardId(61), "id:#61");
     }
 
     [TestMethod]
-    public void ColonWithoutKeyPrefixIsFreeText()
+    public void QuotedTextWithColonIsFreeText()
     {
-        AssertParsesTo(new SearchNode.FreeText(":foo"), ":foo");
+        AssertParsesTo(new SearchNode.FreeText("http://x"), "\"http://x\"");
+    }
+
+    [TestMethod]
+    public void WordMayContainHashAndPunctuation()
+    {
+        AssertParsesTo(new SearchNode.FreeText("#61"), "#61");
+        AssertParsesTo(new SearchNode.FreeText("c#/.net"), "c#/.net");
     }
 
     #endregion
@@ -262,9 +270,58 @@ public class SearchQueryParserTests
     }
 
     [TestMethod]
-    public void NonNumericIdIsAnError()
+    [DataRow("id:12x")]
+    [DataRow("id:abc")]
+    [DataRow("id:0")]
+    [DataRow("id:-1")]
+    [DataRow("id:+1")]
+    [DataRow("id:#")]
+    [DataRow("id:##1")]
+    [DataRow("id:1.5")]
+    [DataRow("id:٣")]
+    [DataRow("id:99999999999999999999")]
+    public void InvalidIdValueIsAnError(string text)
     {
-        AssertSyntaxError("id:12x");
+        AssertSyntaxError(text);
+    }
+
+    [TestMethod]
+    public void QuotedIdValueIsAnError()
+    {
+        AssertSyntaxError("id:\"61\"");
+    }
+
+    [TestMethod]
+    [DataRow("tag:AND")]
+    [DataRow("tag:OR")]
+    [DataRow("title:AND")]
+    public void BareOperatorAsQualifierValueIsAnError(string text)
+    {
+        AssertSyntaxError(text);
+    }
+
+    [TestMethod]
+    public void UnquotedQualifierValueWithColonIsAnError()
+    {
+        AssertSyntaxError("tag:a:b");
+    }
+
+    [TestMethod]
+    [DataRow(":foo")]
+    [DataRow("http://x")]
+    [DataRow("a:")]
+    public void ColonAfterNonKeyIsAnError(string text)
+    {
+        AssertSyntaxError(text);
+    }
+
+    [TestMethod]
+    [DataRow("a\\b")]
+    [DataRow("\\")]
+    [DataRow("tag:a\\b")]
+    public void BackslashInBareWordIsAnError(string text)
+    {
+        AssertSyntaxError(text);
     }
 
     [TestMethod]

YKanBan.Tests/Search/SearchSqlCompilerTests.cs

@@ -169,10 +169,25 @@ public class SearchSqlCompilerTests
     {
         CompiledSearch compiled = SearchSqlCompiler.Compile(new SearchNode.FreeText("x"));
 
+        // One folded parameter shared by the title and content tests.
         StringAssert.Contains(compiled.Predicate, "$p0");
-        StringAssert.Contains(compiled.Predicate, "$p1");
+        Assert.AreEqual(1, compiled.Parameters.Count);
-        Assert.AreEqual(2, compiled.Parameters.Count);
+        Assert.AreEqual("x", compiled.Parameters[0].Value);
-        Assert.AreEqual("%x%", compiled.Parameters[0].Value);
+    }
-        Assert.AreEqual("%x%", compiled.Parameters[1].Value);
+
+    [TestMethod]
+    public void MatchingFoldsNonAsciiCase()
+    {
+        using var fixture = new SearchFixture();
+        SqliteTestHelper.Exec(fixture.Connection, "INSERT INTO columns (title, description, created_at, updated_at) VALUES ('Érable', '', 1, 1);");
+        SqliteTestHelper.Exec(fixture.Connection, "INSERT INTO cards (column_id, title, content, created_at, updated_at) VALUES (3, 'Ärger', 'Straße ΣΟΦΙΑ', 1, 1);");
+        SqliteTestHelper.Exec(fixture.Connection, "INSERT INTO tags (name, color, description) VALUES ('Äpfel', '#123456', '');");
+        SqliteTestHelper.Exec(fixture.Connection, "INSERT INTO card_tags (card_id, tag_id) VALUES (5, 4);");
+
+        CollectionAssert.AreEqual(new[] { 5L }, fixture.Search("ärger"));
+        CollectionAssert.AreEqual(new[] { 5L }, fixture.Search("title:ÄRGER"));
+        CollectionAssert.AreEqual(new[] { 5L }, fixture.Search("content:σοφια"));
+        CollectionAssert.AreEqual(new[] { 5L }, fixture.Search("tag:äpfel"));
+        CollectionAssert.AreEqual(new[] { 5L }, fixture.Search("column:érable"));
     }
 }

YKanBan/Search/SearchLexer.cs

@@ -35,8 +35,8 @@ public abstract record SearchToken
 /// quoted phrases and parentheses; whitespace only separates tokens.
 ///
 /// The lexer is strict: inside a phrase only <c>\"</c> and <c>\\</c> are valid
-/// escapes, a bare backslash or an unknown escape is rejected, and an
+/// escapes, a bare backslash or an unknown escape is rejected, a backslash in a
-/// unterminated phrase is rejected. Any input not covered by the token rules
+/// bare word is rejected, and an unterminated phrase is rejected. Any input not covered by the token rules
 /// is rejected rather than silently ignored.
 /// </summary>
 public static class SearchLexer
@@ -62,9 +62,11 @@ public static class SearchLexer
     private static readonly Parser<char, SearchToken> CloseParenParser =
         Parser.Char(')').Select(_ => (SearchToken)new SearchToken.CloseParen());
 
+    // ':' stays inside the word token (the parser splits qualifiers); a backslash
+    // is not a word character, so a bare one is left over and rejected by End.
     private static readonly Parser<char, SearchToken> WordParser =
         Parser<char>.Token(character =>
-                !char.IsWhiteSpace(character) && character != '(' && character != ')' && character != '"')
+                !char.IsWhiteSpace(character) && character != '(' && character != ')' && character != '"' && character != '\\')
             .AtLeastOnce()
             .Select(chars => (SearchToken)new SearchToken.Word(new string(chars.ToArray())));
 

YKanBan/Search/SearchQueryParser.cs

@@ -6,8 +6,10 @@ namespace YKanBan.Search;
 /// <summary>
 /// Parses the search grammar with Pidgin, strictly: any malformed input
 /// (unbalanced parentheses, dangling or consecutive operators, unterminated
-/// phrases, invalid escapes, qualifier keys without a value or with an
+/// phrases, invalid escapes, a backslash in a bare word, unknown qualifier
-/// unknown/non-numeric id value) raises <see cref="SearchSyntaxException"/>.
+/// keys, qualifiers without a value, unquoted values containing ':' or equal
+/// to AND/OR, and id values that are not an unquoted positive integer with an
+/// optional '#') raises <see cref="SearchSyntaxException"/>.
 ///
 /// Only uppercase AND/OR are operators (lowercase ones are plain words); AND —
 /// explicit or by adjacency — binds tighter than OR; parentheses group. Blank
@@ -131,55 +133,86 @@ public static class SearchQueryParser
     private static Parser<SearchToken, SearchNode> QualifierFrom(string word)
     {
         int colonIndex = word.IndexOf(':');
-
+        if (colonIndex < 0)
-        // Only a leading run of letters/digits/underscores before ':' can be a qualifier key.
-        if (colonIndex <= 0 || !IsQualifierKeyCandidate(word.AsSpan(0, colonIndex)))
         {
             return Parser<SearchToken>.Return<SearchNode>(new SearchNode.FreeText(word));
         }
 
+        // Any ':' in a bare word makes it a qualifier, so the text before it must be a known key.
         string key = word[..colonIndex];
         string value = word[(colonIndex + 1)..];
 
         if (!TryLookupQualifier(key, out QualifierKind kind))
         {
-            return Parser<SearchToken>.Fail<SearchNode>($"Unknown qualifier key '{key}'.");
+            return Parser<SearchToken>.Fail<SearchNode>($"Unknown qualifier key '{key}'; quote the text to search for it literally.");
         }
 
         // "key:" without an inline value takes the value from the following phrase, e.g. tag:"a b".
-        return value.Length == 0
+        if (value.Length == 0)
-            ? PhraseText.Bind(text => AtomParser(kind, text))
+        {
-            : AtomParser(kind, value);
+            return kind == QualifierKind.Id
+                ? Parser<SearchToken>.Fail<SearchNode>("The id qualifier expects an unquoted positive integer.")
+                : PhraseText.Select(text => TextAtom(kind, text));
+        }
+
+        if (value.Contains(':'))
+        {
+            return Parser<SearchToken>.Fail<SearchNode>($"Unquoted value '{value}' of qualifier '{key}' contains ':'; quote the value.");
+        }
+
+        if (value is "AND" or "OR")
+        {
+            return Parser<SearchToken>.Fail<SearchNode>($"Operator {value} cannot be a qualifier value; quote it to search for it literally.");
+        }
+
+        if (kind == QualifierKind.Id)
+        {
+            return TryParseCardId(value, out long id)
+                ? Parser<SearchToken>.Return<SearchNode>(new SearchNode.CardId(id))
+                : Parser<SearchToken>.Fail<SearchNode>($"The id qualifier expects a positive integer, got '{value}'.");
+        }
+
+        return Parser<SearchToken>.Return(TextAtom(kind, value));
     }
 
     /// <summary>
-    /// Builds the atom parser for a qualifier value, failing strictly on a non-numeric id.
+    /// Builds the atom for a textual qualifier value.
     /// </summary>
-    /// <param name="kind">The qualifier kind.</param>
+    /// <param name="kind">The qualifier kind; never <see cref="QualifierKind.Id"/>.</param>
     /// <param name="value">The qualifier value.</param>
-    /// <returns>The parser producing the atom node.</returns>
+    /// <returns>The atom node.</returns>
-    private static Parser<SearchToken, SearchNode> AtomParser(QualifierKind kind, string value) =>
+    private static SearchNode TextAtom(QualifierKind kind, string value) => kind switch
-        AtomFrom(kind, value) is { } atom
-            ? Parser<SearchToken>.Return(atom)
-            : Parser<SearchToken>.Fail<SearchNode>($"The id qualifier expects an integer, got '{value}'.");
-
-    /// <summary>
-    /// Builds the atom for a qualifier value; null when an id value is not an integer.
-    /// </summary>
-    /// <param name="kind">The qualifier kind.</param>
-    /// <param name="value">The qualifier value.</param>
-    /// <returns>The atom node, or <see langword="null"/> for an invalid id value.</returns>
-    private static SearchNode? AtomFrom(QualifierKind kind, string value) => kind switch
     {
         QualifierKind.Tag => new SearchNode.TagName(value),
         QualifierKind.Title => new SearchNode.FieldText(SearchField.Title, value),
         QualifierKind.Content => new SearchNode.FieldText(SearchField.Content, value),
         QualifierKind.Column => new SearchNode.ColumnTitle(value),
-        QualifierKind.Id when long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out long id) =>
+        _ => throw new InvalidOperationException($"Qualifier kind {kind} has no textual value."),
-            new SearchNode.CardId(id),
-        _ => null,
     };
 
+    /// <summary>
+    /// Parses an id qualifier value: decimal digits with an optional leading
+    /// <c>#</c>, denoting a positive integer.
+    /// </summary>
+    /// <param name="value">The inline qualifier value.</param>
+    /// <param name="id">The parsed card id.</param>
+    /// <returns><see langword="true"/> when the value is a valid positive card id.</returns>
+    private static bool TryParseCardId(string value, out long id)
+    {
+        ReadOnlySpan<char> digits = value.AsSpan();
+        if (digits.StartsWith("#"))
+        {
+            digits = digits[1..];
+        }
+
+        // Digits only: the span check rules out signs, whitespace and other NumberStyles leniency.
+        id = 0;
+        return digits.Length > 0
+            && !digits.ContainsAnyExceptInRange('0', '9')
+            && long.TryParse(digits, NumberStyles.None, CultureInfo.InvariantCulture, out id)
+            && id > 0;
+    }
+
     /// <summary>
     /// Looks up a qualifier key in the extension table with ordinal (case-sensitive) matching.
     /// </summary>
@@ -201,25 +234,6 @@ public static class SearchQueryParser
         return false;
     }
 
-    /// <summary>
-    /// Returns whether the text looks like a qualifier key: a non-empty run of
-    /// letters, digits or underscores.
-    /// </summary>
-    /// <param name="text">The candidate key text.</param>
-    /// <returns><see langword="true"/> when every character is a word character.</returns>
-    private static bool IsQualifierKeyCandidate(ReadOnlySpan<char> text)
-    {
-        foreach (char character in text)
-        {
-            if (!(char.IsLetterOrDigit(character) || character == '_'))
-            {
-                return false;
-            }
-        }
-
-        return true;
-    }
-
     /// <summary>
     /// Collapses an AND chain into a single node.
     /// </summary>

YKanBan/Search/SearchSqlCompiler.cs

@@ -1,3 +1,5 @@
+using YKanBan.Storage;
+
 namespace YKanBan.Search;
 
 /// <summary>
@@ -10,10 +12,11 @@ public sealed record CompiledSearch(string Predicate, IReadOnlyList<(string Name
 
 /// <summary>
 /// Compiles a parsed search AST into a parameterized SQL predicate. Matching is
-/// case-insensitive with SQLite's built-in semantics: LIKE's default case
+/// Unicode case-insensitive: both sides are folded with
-/// folding for substring matches plus COLLATE NOCASE for exact matches. LIKE
+/// <see cref="SqliteDatabase.Fold"/> (the column side through the per-connection
-/// wildcards and the escape character in user text are escaped so they match
+/// <see cref="SqliteDatabase.FoldFunctionName"/> function) and then compared
-/// literally.
+/// ordinally — <c>instr</c> for substrings, <c>=</c> for exact matches. LIKE is
+/// deliberately not used, so <c>%</c> and <c>_</c> carry no wildcard meaning.
 /// </summary>
 public static class SearchSqlCompiler
 {
@@ -48,29 +51,28 @@ public static class SearchSqlCompiler
             case SearchNode.FreeText freeText:
             {
                 // A bare word matches a substring of the title OR the content.
-                string titleParam = AddParameter(parameters, "%" + EscapeLike(freeText.Text) + "%");
+                string param = AddParameter(parameters, SqliteDatabase.Fold(freeText.Text));
-                string contentParam = AddParameter(parameters, "%" + EscapeLike(freeText.Text) + "%");
+                return $"({Contains("c.title", param)} OR {Contains("c.content", param)})";
-                return $"(c.title LIKE {titleParam} ESCAPE '\\' OR c.content LIKE {contentParam} ESCAPE '\\')";
             }
 
             case SearchNode.FieldText field:
             {
                 string column = field.Field == SearchField.Title ? "c.title" : "c.content";
-                string param = AddParameter(parameters, "%" + EscapeLike(field.Text) + "%");
+                string param = AddParameter(parameters, SqliteDatabase.Fold(field.Text));
-                return $"{column} LIKE {param} ESCAPE '\\'";
+                return Contains(column, param);
             }
 
             case SearchNode.TagName tag:
             {
-                string param = AddParameter(parameters, tag.Name);
+                string param = AddParameter(parameters, SqliteDatabase.Fold(tag.Name));
                 return "EXISTS (SELECT 1 FROM card_tags ct JOIN tags t ON t.id = ct.tag_id " +
-                       $"WHERE ct.card_id = c.id AND t.name COLLATE NOCASE = {param})";
+                       $"WHERE ct.card_id = c.id AND {Folded("t.name")} = {param})";
             }
 
             case SearchNode.ColumnTitle column:
             {
-                string param = AddParameter(parameters, column.Title);
+                string param = AddParameter(parameters, SqliteDatabase.Fold(column.Title));
-                return $"c.column_id IN (SELECT id FROM columns WHERE title COLLATE NOCASE = {param})";
+                return $"c.column_id IN (SELECT id FROM columns WHERE {Folded("title")} = {param})";
             }
 
             case SearchNode.CardId id:
@@ -98,10 +100,17 @@ public static class SearchSqlCompiler
     }
 
     /// <summary>
-    /// Escapes LIKE wildcards and the escape character inside user text.
+    /// Wraps a column expression in the case-folding SQL function.
     /// </summary>
-    /// <param name="text">The raw user text.</param>
+    /// <param name="column">The column expression.</param>
-    /// <returns>The text with <c>\</c>, <c>%</c> and <c>_</c> escaped.</returns>
+    /// <returns>The folded column expression.</returns>
-    private static string EscapeLike(string text) =>
+    private static string Folded(string column) => $"{SqliteDatabase.FoldFunctionName}({column})";
-        text.Replace("\\", "\\\\").Replace("%", "\\%").Replace("_", "\\_");
+
+    /// <summary>
+    /// Builds an ordinal substring test of a folded column against an already folded parameter.
+    /// </summary>
+    /// <param name="column">The column expression.</param>
+    /// <param name="param">The placeholder of the folded search text.</param>
+    /// <returns>The SQL boolean expression.</returns>
+    private static string Contains(string column, string param) => $"instr({Folded(column)}, {param}) > 0";
 }

YKanBan/Storage/SqliteDatabase.cs

@@ -5,10 +5,25 @@ namespace YKanBan.Storage;
 /// <summary>
 /// Shared SQLite plumbing for workspace databases: opens a connection with
 /// the mandated PRAGMA configuration (WAL journal, NORMAL synchronous mode,
-/// foreign keys on) and applies incremental <c>user_version</c> migrations.
+/// foreign keys on), registers the custom SQL functions and applies
+/// incremental <c>user_version</c> migrations.
 /// </summary>
 public static class SqliteDatabase
 {
+    /// <summary>
+    /// Name of the per-connection SQL function that applies <see cref="Fold"/>;
+    /// search uses it for Unicode case-insensitive matching.
+    /// </summary>
+    public const string FoldFunctionName = "ykb_fold";
+
+    /// <summary>
+    /// Unicode case folding shared by the SQL function and the search
+    /// parameters it is compared against; comparisons after folding are ordinal.
+    /// </summary>
+    /// <param name="text">The text to fold.</param>
+    /// <returns>The invariant lowercase form.</returns>
+    public static string Fold(string text) => text.ToLowerInvariant();
+
     /// <summary>
     /// Opens (creating if needed) a database file, applies the mandatory
     /// PRAGMAs and runs any pending migrations.
@@ -31,6 +46,7 @@ public static class SqliteDatabase
         try
         {
             ApplyPragmas(connection);
+            RegisterFunctions(connection);
             ApplyMigrations(connection, migrations);
             return connection;
         }
@@ -67,6 +83,18 @@ public static class SqliteDatabase
         Execute(connection, "PRAGMA foreign_keys=ON;");
     }
 
+    /// <summary>
+    /// Registers the custom SQL functions; like the PRAGMAs they live on the
+    /// connection and must be registered on every one.
+    /// </summary>
+    /// <param name="connection">An open database connection.</param>
+    internal static void RegisterFunctions(SqliteConnection connection)
+    {
+        // NULL never reaches it from the NOT NULL text columns, but stays NULL-safe regardless.
+        connection.CreateFunction<string?, string?>(
+            FoldFunctionName, text => text is null ? null : Fold(text), isDeterministic: true);
+    }
+
     /// <summary>
     /// Runs every migration newer than the stored <c>user_version</c>, each in
     /// its own transaction together with its version bump.