using System.Text.RegularExpressions; using Blog.Models; namespace Blog.Services; /// A language's rules, resolved once from a path and reused for every line of it. public sealed class SyntaxGrammar { internal SyntaxGrammar(Regex pattern) => Pattern = pattern; internal Regex Pattern { get; } } /// /// Splits source text into s for the pages under /git. /// /// /// One regex per language, alternating over named groups, matched left to right: whatever the /// engine finds first wins, so the order of the alternatives *is* the precedence. Comments and /// strings come first in every grammar, which is what keeps a keyword inside a string, or a quote /// inside a comment, from being read as code. /// /// This highlights a diff for reading; it does not parse anything. There is no state between /// matches and no notion of scope, so it gets a few things wrong on purpose: a > in HTML /// prose reads as a tag bracket, and the curly-brace languages share one keyword union rather than /// a table each, because no file in these repositories holds both defer and yield /// and the alternative is eight tables saying almost the same thing. Being wrong costs a bold word. /// /// Not [GeneratedRegex], which the rest of this site uses: these patterns are assembled /// from shared fragments and a source generator needs a constant. They are built once, compiled, /// and given a timeout — a highlighter is not worth a hung request. /// public static class SyntaxHighlighter { /// /// Past this the text comes back as plain lines. Two reasons, and the second is the binding /// one: a pathological file should not be able to hold a request open, and a highlighted line /// is several times its own length in markup, because every token it holds becomes a span. A /// file of machine-generated HTML tokenises into almost nothing else. /// private const int MaxCharacters = 160_000; private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(1); // -- Shared fragments ---------------------------------------------------- /// Three quote marks. A C# raw string cannot hold its own delimiter. private const string Triple = "\"\"\""; /// /// Quotes that have to close on the line they opened on. An apostrophe in prose would /// otherwise open a string that runs to the end of the file. /// private const string Quotes = @"""(?:\\.|[^""\\\n])*""|'(?:\\.|[^'\\\n])*'"; /// Decimal, hex and binary, with a trailing unit or suffix: 10px, 1.5f, 0xFF, 50%. private const string Number = @"\b(?:0[xX][0-9a-fA-F_]+|0[bB][01_]+|\d[\d_]*(?:\.\d[\d_]*)?(?:[eE][+-]?\d+)?)[a-zA-Z%]*"; // -- Grammars ------------------------------------------------------------ /// /// C, C#, Java, JavaScript, Go, Rust and the rest of the curly-brace family. One keyword /// union: see the note on being wrong, above. /// private static readonly SyntaxGrammar CFamily = Grammar( @"(?//[^\n]*|/\*[\s\S]*?\*/)", // C# raw and verbatim strings and JS templates all cross lines, so they come first and // are the only ones allowed to. $@"(?{Triple}[\s\S]*?{Triple}|@""(?:""""|[^""])*""|`(?:\\.|[^`\\])*`|{Quotes})", // A preprocessor directive, or a C# attribute alone on its line. Anchored, because // [Something] in the middle of a line is an index, not an attribute. @"(?^[ \t]*\#[ \t]*\w+|^[ \t]*\[[A-Za-z][\w.]*(?:\([^\n]*\))?\][ \t]*$)", $"(?{Number})", Keywords( "abstract alias and as assert async await base bool break byte case catch char class", "comptime const constexpr continue decimal default defer delegate do double dynamic", "else enum event explicit export extends extern false final finally float fn for", "foreach from func function global goto if impl implements implicit import in init", "inline instanceof int interface internal is let lock long match namespace new nil not", "null nullptr object operator or out override package params partial private protected", "public readonly record ref required return sbyte sealed select short sizeof stackalloc", "static string struct super switch this throw throws trait true try typedef typeof", "uint ulong unsafe ushort using var virtual void volatile when where while yield")); /// Shell, Python, Ruby, Elixir, YAML — anything whose comments start with a hash. private static readonly SyntaxGrammar Hash = Grammar( @"(?#[^\n]*)", $@"(?{Triple}[\s\S]*?{Triple}|'''[\s\S]*?'''|{Quotes})", // $VAR, ${VAR}, $1, $@ — what a shell line is usually about. @"(?\$\{[^}\n]*\}|\$[A-Za-z_]\w*|\$[0-9@*#?!$-])", $"(?{Number})", Keywords( "after alias and as break case class cond continue def defmacro defmodule defp do done", "elif else elsif end esac eval exec exit export false fi finally fn for from function", "global if import in lambda local next nil not or pass raise receive require rescue", "return select set shift source then trap true try unset until use while yield")); /// HTML, XML, SVG, Razor, csproj — everything shaped like a tag. private static readonly SyntaxGrammar Markup = Grammar( @"(?)", @"(?<\?[\s\S]*?\?>||\n]*>|&\#?\w+;|@[A-Za-z_][\w.]*)", @"(?""[^""\n]*""|'[^'\n]*')", // The brackets and the tag name, not the attributes: a name in bold with its attributes // left plain is what makes the shape of a document readable at a glance. @"(?)"); private static readonly SyntaxGrammar Css = Grammar( @"(?/\*[\s\S]*?\*/)", @"(?""[^""\n]*""|'[^'\n]*')", // At-rules, custom properties and !important. In this stylesheet the custom properties // are the design system, so they are worth marking as something other than a property. @"(?@[\w-]+|--[\w-]+|![\w-]+)", $@"(?\#[0-9a-fA-F]{{3,8}}\b|{Number})", // A property name is whatever is about to be followed by a colon. That also catches the // pseudo-class in a:hover, which is no great loss: both are the language, not the value. @"(?\b[-a-zA-Z]+(?=\s*:))"); private static readonly SyntaxGrammar Json = Grammar( @"(?""(?:\\.|[^""\\\n])*""(?=\s*:))", @"(?""(?:\\.|[^""\\\n])*"")", @"(?-?\b\d[\d]*(?:\.\d+)?(?:[eE][+-]?\d+)?)", @"(?\b(?:true|false|null)\b)"); private static readonly SyntaxGrammar Markdown = Grammar( @"(?^[ \t]{0,3}>[^\n]*)", @"(?```[\s\S]*?```|~~~[\s\S]*?~~~|`[^`\n]+`)", @"(?^\#{1,6}[ \t][^\n]*|^[ \t]{0,3}(?:[-*+]|\d{1,9}[.)])[ \t])", @"(?\[[^\]\n]*\]\([^)\n]*\)|^[ \t]{0,3}(?:[-*_][ \t]*){3,}$)"); // -- Language by filename ------------------------------------------------ /// /// The grammar for a path, or null for a file this knows nothing about — which renders as /// plain text rather than as a guess. /// public static SyntaxGrammar? GrammarFor(string path) { var name = Path.GetFileName(path).ToLowerInvariant(); // Files whose name is their type. GetExtension calls the whole of ".gitignore" an // extension, so these have to be settled before it is asked. switch (name) { case "dockerfile" or "makefile" or "justfile" or "procfile": case ".gitignore" or ".gitattributes" or ".editorconfig" or ".gitmodules": return Hash; case "license" or "licence" or "authors" or "notice": return null; } return Path.GetExtension(name) switch { ".cs" or ".csx" or ".c" or ".h" or ".cpp" or ".cc" or ".hpp" or ".java" or ".js" or ".mjs" or ".cjs" or ".ts" or ".jsx" or ".tsx" or ".go" or ".rs" or ".swift" or ".kt" or ".kts" or ".zig" or ".scala" or ".dart" or ".php" or ".m" or ".gradle" => CFamily, ".sh" or ".bash" or ".zsh" or ".fish" or ".py" or ".rb" or ".ex" or ".exs" or ".erl" or ".pl" or ".yml" or ".yaml" or ".toml" or ".ini" or ".conf" or ".cfg" or ".env" or ".service" or ".nix" => Hash, ".html" or ".htm" or ".xhtml" or ".xml" or ".svg" or ".razor" or ".cshtml" or ".xaml" or ".csproj" or ".props" or ".targets" or ".config" or ".plist" or ".resx" or ".xsd" or ".xsl" or ".vue" => Markup, ".css" or ".scss" or ".sass" or ".less" => Css, ".json" or ".jsonc" or ".webmanifest" => Json, ".md" or ".markdown" or ".mdx" => Markdown, _ => null }; } // -- Tokenising ---------------------------------------------------------- /// /// Splits into one list of tokens per line. A null /// , text too long to be worth scanning, and a regex that runs out /// of time all give the same answer: the lines, untouched. /// public static IReadOnlyList> Highlight(string text, SyntaxGrammar? grammar) { if (text.Length == 0) return []; if (grammar is null || text.Length > MaxCharacters) return Plain(text); try { return Scan(text, grammar.Pattern); } catch (RegexMatchTimeoutException) { return Plain(text); } } private static IReadOnlyList> Scan(string text, Regex pattern) { var lines = new List>(); var line = new List(); var written = 0; // A token can cross a newline — a block comment, a verbatim string, a fenced code block — // so nothing reaches a line until it has been cut at every newline inside it. void Emit(SyntaxKind kind, ReadOnlySpan value) { while (value.IndexOf('\n') is var cut and >= 0) { if (cut > 0) line.Add(new SyntaxToken(kind, value[..cut].ToString())); lines.Add(line); line = []; value = value[(cut + 1)..]; } if (value.Length > 0) line.Add(new SyntaxToken(kind, value.ToString())); } foreach (Match match in pattern.Matches(text)) { if (match.Index > written) Emit(SyntaxKind.Text, text.AsSpan(written..match.Index)); Emit(KindOf(match), match.ValueSpan); written = match.Index + match.Length; } if (written < text.Length) Emit(SyntaxKind.Text, text.AsSpan(written)); lines.Add(line); return lines; } /// /// Which alternative matched. Every grammar names its groups from the same set, and a match /// can only have come from one of them. /// private static SyntaxKind KindOf(Match match) { if (match.Groups["comment"].Success) return SyntaxKind.Comment; if (match.Groups["string"].Success) return SyntaxKind.String; if (match.Groups["keyword"].Success) return SyntaxKind.Keyword; if (match.Groups["number"].Success) return SyntaxKind.Number; if (match.Groups["meta"].Success) return SyntaxKind.Meta; return SyntaxKind.Text; } private static IReadOnlyList> Plain(string text) => text.Split('\n') .Select(IReadOnlyList (value) => value.Length == 0 ? [] : [new SyntaxToken(SyntaxKind.Text, value)]) .ToArray(); // -- Building ------------------------------------------------------------ private static SyntaxGrammar Grammar(params string[] alternatives) => new(new Regex( string.Join('|', alternatives), RegexOptions.Compiled | RegexOptions.ExplicitCapture | RegexOptions.Multiline, Timeout)); /// A keyword alternative out of space-separated words, kept readable in the source. private static string Keywords(params string[] words) => $@"(?\b(?:{string.Join('|', words.SelectMany(line => line.Split(' ')))})\b)"; }