Blog/Services/SyntaxHighlighter.cs 12.6 K · 265 lines · raw · history

1 using System.Text.RegularExpressions;
2 using Blog.Models;
3
4 namespace Blog.Services;
5
6 /// <summary>A language's rules, resolved once from a path and reused for every line of it.</summary>
7 public sealed class SyntaxGrammar
8 {
9 internal SyntaxGrammar(Regex pattern) => Pattern = pattern;
10
11 internal Regex Pattern { get; }
12 }
13
14 /// <summary>
15 /// Splits source text into <see cref="SyntaxToken"/>s for the pages under <c>/git</c>.
16 /// </summary>
17 /// <remarks>
18 /// One regex per language, alternating over named groups, matched left to right: whatever the
19 /// engine finds first wins, so the order of the alternatives *is* the precedence. Comments and
20 /// strings come first in every grammar, which is what keeps a keyword inside a string, or a quote
21 /// inside a comment, from being read as code.
22 ///
23 /// This highlights a diff for reading; it does not parse anything. There is no state between
24 /// matches and no notion of scope, so it gets a few things wrong on purpose: a <c>&gt;</c> in HTML
25 /// prose reads as a tag bracket, and the curly-brace languages share one keyword union rather than
26 /// a table each, because no file in these repositories holds both <c>defer</c> and <c>yield</c>
27 /// and the alternative is eight tables saying almost the same thing. Being wrong costs a bold word.
28 ///
29 /// Not <c>[GeneratedRegex]</c>, which the rest of this site uses: these patterns are assembled
30 /// from shared fragments and a source generator needs a constant. They are built once, compiled,
31 /// and given a timeout — a highlighter is not worth a hung request.
32 /// </remarks>
33 public static class SyntaxHighlighter
34 {
35 /// <summary>
36 /// Past this the text comes back as plain lines. Two reasons, and the second is the binding
37 /// one: a pathological file should not be able to hold a request open, and a highlighted line
38 /// is several times its own length in markup, because every token it holds becomes a span. A
39 /// file of machine-generated HTML tokenises into almost nothing else.
40 /// </summary>
41 private const int MaxCharacters = 160_000;
42
43 private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(1);
44
45 // -- Shared fragments ----------------------------------------------------
46
47 /// <summary>Three quote marks. A C# raw string cannot hold its own delimiter.</summary>
48 private const string Triple = "\"\"\"";
49
50 /// <summary>
51 /// Quotes that have to close on the line they opened on. An apostrophe in prose would
52 /// otherwise open a string that runs to the end of the file.
53 /// </summary>
54 private const string Quotes = @"""(?:\\.|[^""\\\n])*""|'(?:\\.|[^'\\\n])*'";
55
56 /// <summary>Decimal, hex and binary, with a trailing unit or suffix: 10px, 1.5f, 0xFF, 50%.</summary>
57 private const string Number =
58 @"\b(?:0[xX][0-9a-fA-F_]+|0[bB][01_]+|\d[\d_]*(?:\.\d[\d_]*)?(?:[eE][+-]?\d+)?)[a-zA-Z%]*";
59
60 // -- Grammars ------------------------------------------------------------
61
62 /// <summary>
63 /// C, C#, Java, JavaScript, Go, Rust and the rest of the curly-brace family. One keyword
64 /// union: see the note on being wrong, above.
65 /// </summary>
66 private static readonly SyntaxGrammar CFamily = Grammar(
67 @"(?<comment>//[^\n]*|/\*[\s\S]*?\*/)",
68 // C# raw and verbatim strings and JS templates all cross lines, so they come first and
69 // are the only ones allowed to.
70 $@"(?<string>{Triple}[\s\S]*?{Triple}|@""(?:""""|[^""])*""|`(?:\\.|[^`\\])*`|{Quotes})",
71 // A preprocessor directive, or a C# attribute alone on its line. Anchored, because
72 // [Something] in the middle of a line is an index, not an attribute.
73 @"(?<meta>^[ \t]*\#[ \t]*\w+|^[ \t]*\[[A-Za-z][\w.]*(?:\([^\n]*\))?\][ \t]*$)",
74 $"(?<number>{Number})",
75 Keywords(
76 "abstract alias and as assert async await base bool break byte case catch char class",
77 "comptime const constexpr continue decimal default defer delegate do double dynamic",
78 "else enum event explicit export extends extern false final finally float fn for",
79 "foreach from func function global goto if impl implements implicit import in init",
80 "inline instanceof int interface internal is let lock long match namespace new nil not",
81 "null nullptr object operator or out override package params partial private protected",
82 "public readonly record ref required return sbyte sealed select short sizeof stackalloc",
83 "static string struct super switch this throw throws trait true try typedef typeof",
84 "uint ulong unsafe ushort using var virtual void volatile when where while yield"));
85
86 /// <summary>Shell, Python, Ruby, Elixir, YAML — anything whose comments start with a hash.</summary>
87 private static readonly SyntaxGrammar Hash = Grammar(
88 @"(?<comment>#[^\n]*)",
89 $@"(?<string>{Triple}[\s\S]*?{Triple}|'''[\s\S]*?'''|{Quotes})",
90 // $VAR, ${VAR}, $1, $@ — what a shell line is usually about.
91 @"(?<meta>\$\{[^}\n]*\}|\$[A-Za-z_]\w*|\$[0-9@*#?!$-])",
92 $"(?<number>{Number})",
93 Keywords(
94 "after alias and as break case class cond continue def defmacro defmodule defp do done",
95 "elif else elsif end esac eval exec exit export false fi finally fn for from function",
96 "global if import in lambda local next nil not or pass raise receive require rescue",
97 "return select set shift source then trap true try unset until use while yield"));
98
99 /// <summary>HTML, XML, SVG, Razor, csproj — everything shaped like a tag.</summary>
100 private static readonly SyntaxGrammar Markup = Grammar(
101 @"(?<comment><!--[\s\S]*?-->)",
102 @"(?<meta><\?[\s\S]*?\?>|<!\[CDATA\[[\s\S]*?\]\]>|<![A-Za-z][^>\n]*>|&\#?\w+;|@[A-Za-z_][\w.]*)",
103 @"(?<string>""[^""\n]*""|'[^'\n]*')",
104 // The brackets and the tag name, not the attributes: a name in bold with its attributes
105 // left plain is what makes the shape of a document readable at a glance.
106 @"(?<keyword></?[A-Za-z][\w:.-]*|/?>)");
107
108 private static readonly SyntaxGrammar Css = Grammar(
109 @"(?<comment>/\*[\s\S]*?\*/)",
110 @"(?<string>""[^""\n]*""|'[^'\n]*')",
111 // At-rules, custom properties and !important. In this stylesheet the custom properties
112 // are the design system, so they are worth marking as something other than a property.
113 @"(?<meta>@[\w-]+|--[\w-]+|![\w-]+)",
114 $@"(?<number>\#[0-9a-fA-F]{{3,8}}\b|{Number})",
115 // A property name is whatever is about to be followed by a colon. That also catches the
116 // pseudo-class in a:hover, which is no great loss: both are the language, not the value.
117 @"(?<keyword>\b[-a-zA-Z]+(?=\s*:))");
118
119 private static readonly SyntaxGrammar Json = Grammar(
120 @"(?<meta>""(?:\\.|[^""\\\n])*""(?=\s*:))",
121 @"(?<string>""(?:\\.|[^""\\\n])*"")",
122 @"(?<number>-?\b\d[\d]*(?:\.\d+)?(?:[eE][+-]?\d+)?)",
123 @"(?<keyword>\b(?:true|false|null)\b)");
124
125 private static readonly SyntaxGrammar Markdown = Grammar(
126 @"(?<comment>^[ \t]{0,3}>[^\n]*)",
127 @"(?<string>```[\s\S]*?```|~~~[\s\S]*?~~~|`[^`\n]+`)",
128 @"(?<keyword>^\#{1,6}[ \t][^\n]*|^[ \t]{0,3}(?:[-*+]|\d{1,9}[.)])[ \t])",
129 @"(?<meta>\[[^\]\n]*\]\([^)\n]*\)|^[ \t]{0,3}(?:[-*_][ \t]*){3,}$)");
130
131 // -- Language by filename ------------------------------------------------
132
133 /// <summary>
134 /// The grammar for a path, or null for a file this knows nothing about — which renders as
135 /// plain text rather than as a guess.
136 /// </summary>
137 public static SyntaxGrammar? GrammarFor(string path)
138 {
139 var name = Path.GetFileName(path).ToLowerInvariant();
140
141 // Files whose name is their type. GetExtension calls the whole of ".gitignore" an
142 // extension, so these have to be settled before it is asked.
143 switch (name)
144 {
145 case "dockerfile" or "makefile" or "justfile" or "procfile":
146 case ".gitignore" or ".gitattributes" or ".editorconfig" or ".gitmodules":
147 return Hash;
148 case "license" or "licence" or "authors" or "notice":
149 return null;
150 }
151
152 return Path.GetExtension(name) switch
153 {
154 ".cs" or ".csx" or ".c" or ".h" or ".cpp" or ".cc" or ".hpp" or ".java" or ".js"
155 or ".mjs" or ".cjs" or ".ts" or ".jsx" or ".tsx" or ".go" or ".rs" or ".swift"
156 or ".kt" or ".kts" or ".zig" or ".scala" or ".dart" or ".php" or ".m" or ".gradle"
157 => CFamily,
158
159 ".sh" or ".bash" or ".zsh" or ".fish" or ".py" or ".rb" or ".ex" or ".exs" or ".erl"
160 or ".pl" or ".yml" or ".yaml" or ".toml" or ".ini" or ".conf" or ".cfg" or ".env"
161 or ".service" or ".nix"
162 => Hash,
163
164 ".html" or ".htm" or ".xhtml" or ".xml" or ".svg" or ".razor" or ".cshtml" or ".xaml"
165 or ".csproj" or ".props" or ".targets" or ".config" or ".plist" or ".resx"
166 or ".xsd" or ".xsl" or ".vue"
167 => Markup,
168
169 ".css" or ".scss" or ".sass" or ".less" => Css,
170 ".json" or ".jsonc" or ".webmanifest" => Json,
171 ".md" or ".markdown" or ".mdx" => Markdown,
172
173 _ => null
174 };
175 }
176
177 // -- Tokenising ----------------------------------------------------------
178
179 /// <summary>
180 /// Splits <paramref name="text"/> into one list of tokens per line. A null
181 /// <paramref name="grammar"/>, text too long to be worth scanning, and a regex that runs out
182 /// of time all give the same answer: the lines, untouched.
183 /// </summary>
184 public static IReadOnlyList<IReadOnlyList<SyntaxToken>> Highlight(string text, SyntaxGrammar? grammar)
185 {
186 if (text.Length == 0) return [];
187 if (grammar is null || text.Length > MaxCharacters) return Plain(text);
188
189 try
190 {
191 return Scan(text, grammar.Pattern);
192 }
193 catch (RegexMatchTimeoutException)
194 {
195 return Plain(text);
196 }
197 }
198
199 private static IReadOnlyList<IReadOnlyList<SyntaxToken>> Scan(string text, Regex pattern)
200 {
201 var lines = new List<IReadOnlyList<SyntaxToken>>();
202 var line = new List<SyntaxToken>();
203 var written = 0;
204
205 // A token can cross a newline — a block comment, a verbatim string, a fenced code block —
206 // so nothing reaches a line until it has been cut at every newline inside it.
207 void Emit(SyntaxKind kind, ReadOnlySpan<char> value)
208 {
209 while (value.IndexOf('\n') is var cut and >= 0)
210 {
211 if (cut > 0) line.Add(new SyntaxToken(kind, value[..cut].ToString()));
212 lines.Add(line);
213 line = [];
214 value = value[(cut + 1)..];
215 }
216
217 if (value.Length > 0) line.Add(new SyntaxToken(kind, value.ToString()));
218 }
219
220 foreach (Match match in pattern.Matches(text))
221 {
222 if (match.Index > written) Emit(SyntaxKind.Text, text.AsSpan(written..match.Index));
223
224 Emit(KindOf(match), match.ValueSpan);
225 written = match.Index + match.Length;
226 }
227
228 if (written < text.Length) Emit(SyntaxKind.Text, text.AsSpan(written));
229
230 lines.Add(line);
231 return lines;
232 }
233
234 /// <summary>
235 /// Which alternative matched. Every grammar names its groups from the same set, and a match
236 /// can only have come from one of them.
237 /// </summary>
238 private static SyntaxKind KindOf(Match match)
239 {
240 if (match.Groups["comment"].Success) return SyntaxKind.Comment;
241 if (match.Groups["string"].Success) return SyntaxKind.String;
242 if (match.Groups["keyword"].Success) return SyntaxKind.Keyword;
243 if (match.Groups["number"].Success) return SyntaxKind.Number;
244 if (match.Groups["meta"].Success) return SyntaxKind.Meta;
245 return SyntaxKind.Text;
246 }
247
248 private static IReadOnlyList<IReadOnlyList<SyntaxToken>> Plain(string text) =>
249 text.Split('\n')
250 .Select(IReadOnlyList<SyntaxToken> (value) =>
251 value.Length == 0 ? [] : [new SyntaxToken(SyntaxKind.Text, value)])
252 .ToArray();
253
254 // -- Building ------------------------------------------------------------
255
256 private static SyntaxGrammar Grammar(params string[] alternatives) =>
257 new(new Regex(
258 string.Join('|', alternatives),
259 RegexOptions.Compiled | RegexOptions.ExplicitCapture | RegexOptions.Multiline,
260 Timeout));
261
262 /// <summary>A keyword alternative out of space-separated words, kept readable in the source.</summary>
263 private static string Keywords(params string[] words) =>
264 $@"(?<keyword>\b(?:{string.Join('|', words.SelectMany(line => line.Split(' ')))})\b)";
265 }