Blog/Services/DropTableParser.cs 14.3 K · 360 lines · raw · history

1 using System.Globalization;
2 using System.Text.RegularExpressions;
3 using Blog.Models;
4
5 namespace Blog.Services;
6
7 /// <summary>
8 /// Turns warframe.com/droptables into <see cref="DropTables"/>.
9 /// </summary>
10 /// <remarks>
11 /// The page is one 4 MB machine-generated document: twenty <c>&lt;h3 id&gt;</c> sections, each a
12 /// single flat table whose rows carry no classes or ids. Three row grammars cover all twenty:
13 ///
14 /// <list type="bullet">
15 /// <item>Two-column reward tables (missions, keys, dynamic locations, sorties, relics): a
16 /// full-width header names the place, an optional <c>Rotation X</c> header splits it, and every
17 /// other row is <c>item | rarity (chance)</c>.</item>
18 /// <item>Three-column bounty tables: as above, plus an indented header per stage and a leading
19 /// spacer cell on every row.</item>
20 /// <item>Three-column "by source" tables (mods, blueprints, resources, relics by enemy): a header
21 /// naming the enemy and its overall drop chance, then that table's contents.</item>
22 /// </list>
23 ///
24 /// Rows that match no grammar are skipped rather than thrown over, so a section DE add later costs
25 /// coverage instead of the whole page.
26 /// </remarks>
27 public static partial class DropTableParser
28 {
29 [GeneratedRegex("""<h3 id="(?<id>[^"]+)"[^>]*>""")]
30 private static partial Regex SectionHeading { get; }
31
32 [GeneratedRegex("""<tr(?<attrs>[^>]*)>(?<body>.*?)</tr>""", RegexOptions.Singleline)]
33 private static partial Regex RowPattern { get; }
34
35 [GeneratedRegex("""<(?<tag>th|td)(?<attrs>[^>]*)>(?<text>.*?)</\k<tag>>""", RegexOptions.Singleline)]
36 private static partial Regex CellPattern { get; }
37
38 /// <summary>DE write every chance as <c>Rare (7.69%)</c>.</summary>
39 [GeneratedRegex(@"^(?<rarity>[A-Za-z ]+?)\s*\((?<pct>[0-9.]+)%\)$")]
40 private static partial Regex ChancePattern { get; }
41
42 /// <summary>The "by source" headers all end in <c>… Drop Chance: 3.00%</c>.</summary>
43 [GeneratedRegex(@"Drop Chance:\s*(?<pct>[0-9.]+)%")]
44 private static partial Regex SourceChancePattern { get; }
45
46 [GeneratedRegex(@"<b>Last Update:</b>\s*(?<date>[^<\r\n]+)")]
47 private static partial Regex LastUpdatePattern { get; }
48
49 /// <summary>A relic appears four times, once per refinement: <c>Axi A1 Relic (Intact)</c>.</summary>
50 [GeneratedRegex(@"^(?<relic>.+? Relic) \((?<refinement>Intact|Exceptional|Flawless|Radiant)\)$")]
51 private static partial Regex RelicHeaderPattern { get; }
52
53 /// <summary>Which grammar each section is written in, and what its sources should be called.</summary>
54 private static readonly Dictionary<string, DropCategory> RewardSections = new(StringComparer.Ordinal)
55 {
56 ["missionRewards"] = DropCategory.Mission,
57 ["keyRewards"] = DropCategory.Key,
58 ["transientRewards"] = DropCategory.Dynamic,
59 ["sortieRewards"] = DropCategory.Sortie
60 };
61
62 private static readonly string[] BountySections =
63 [
64 "cetusRewards", "solarisRewards", "deimosRewards",
65 "zarimanRewards", "entratiLabRewards", "hexRewards"
66 ];
67
68 private static readonly string[] SourceSections =
69 [
70 "modByAvatar", "blueprintByAvatar", "resourceByAvatar",
71 "sigilByAvatar", "additionalItemByAvatar", "relicByAvatar"
72 ];
73
74 public static DropTables Parse(string html, DateTimeOffset fetchedAt)
75 {
76 var builder = new Builder();
77
78 foreach (var (id, section) in Sections(html))
79 {
80 if (RewardSections.TryGetValue(id, out var category))
81 {
82 ParseRewards(section, category, builder);
83 }
84 else if (BountySections.Contains(id, StringComparer.Ordinal))
85 {
86 ParseBounties(section, builder);
87 }
88 else if (SourceSections.Contains(id, StringComparer.Ordinal))
89 {
90 ParseBySource(section, builder);
91 }
92 else if (id == "relicRewards")
93 {
94 ParseRelics(section, builder);
95 }
96
97 // modByDrop, blueprintByDrop and resourceByDrop hold the *ByAvatar data indexed the
98 // other way round. Builder indexes both ways, so reading them would only duplicate.
99 }
100
101 return builder.Build(LastUpdate(html), fetchedAt);
102 }
103
104 private static string LastUpdate(string html)
105 {
106 var match = LastUpdatePattern.Match(html);
107 return match.Success ? match.Groups["date"].Value.Trim() : "unknown";
108 }
109
110 /// <summary>Each <c>&lt;h3 id&gt;</c> and everything up to the next one.</summary>
111 private static IEnumerable<(string Id, string Html)> Sections(string html)
112 {
113 var headings = SectionHeading.Matches(html);
114
115 for (var i = 0; i < headings.Count; i++)
116 {
117 var start = headings[i].Index;
118 var end = i + 1 < headings.Count ? headings[i + 1].Index : html.Length;
119 yield return (headings[i].Groups["id"].Value, html[start..end]);
120 }
121 }
122
123 /// <summary>Missions, keys, dynamic locations and sorties: a place, maybe a rotation, then drops.</summary>
124 private static void ParseRewards(string section, DropCategory category, Builder builder)
125 {
126 string? location = null;
127 string? rotation = null;
128
129 foreach (var row in Rows(section))
130 {
131 switch (row)
132 {
133 // `Rotation A` splits the place above it; anything else full-width is a new place.
134 case [{ Header: true, FullWidth: true, Text: var header }]:
135 if (IsRotation(header)) rotation = header;
136 else (location, rotation) = (header, null);
137 break;
138
139 // Dynamic locations write their rotations as a header and an empty cell instead.
140 case [{ Header: true, Text: var header }, { Text: "" }]:
141 rotation = IsRotation(header) ? header : rotation;
142 break;
143
144 case [{ Header: false, Text: var item }, { Header: false, Text: var chance }]
145 when location is not null && Chance(chance) is (var rarity, var pct):
146 builder.AddSource(item, new DropSource(category, location, rotation, rarity, pct));
147 break;
148 }
149 }
150 }
151
152 /// <summary>Open-world bounties: a bounty, a rotation and a stage, then drops behind a spacer cell.</summary>
153 private static void ParseBounties(string section, Builder builder)
154 {
155 string? bounty = null;
156 string? rotation = null;
157 string? stage = null;
158
159 foreach (var row in Rows(section))
160 {
161 switch (row)
162 {
163 case [{ Header: true, FullWidth: true, Text: var header }]:
164 if (IsRotation(header)) rotation = header;
165 else (bounty, rotation, stage) = (header, null, null);
166 break;
167
168 case [{ Spacer: true }, { Header: true, Text: var header }]:
169 stage = header;
170 break;
171
172 case [_, { Header: false, Text: var item }, { Header: false, Text: var chance }]
173 when bounty is not null && Chance(chance) is (var rarity, var pct):
174 builder.AddSource(item, new DropSource(DropCategory.Bounty, bounty, Join(rotation, stage), rarity, pct));
175 break;
176 }
177 }
178 }
179
180 /// <summary>
181 /// Drops by enemy. The header carries how often the enemy drops from its table at all, so a
182 /// Very Common (100%) line off a 3% table is a 3% kill.
183 /// </summary>
184 private static void ParseBySource(string section, Builder builder)
185 {
186 string? source = null;
187 var sourceChance = 100.0;
188
189 foreach (var row in Rows(section))
190 {
191 switch (row)
192 {
193 case [{ Header: true, Text: var name }, { Header: true, Text: var chance }]:
194 source = name;
195 var match = SourceChancePattern.Match(chance);
196 sourceChance = match.Success ? Percent(match.Groups["pct"].Value) : 100.0;
197 break;
198
199 case [_, { Header: false, Text: var item }, { Header: false, Text: var chance }]
200 when source is not null && Chance(chance) is (var rarity, var pct):
201 builder.AddSource(item,
202 new DropSource(DropCategory.Enemy, source, null, rarity, pct * sourceChance / 100.0));
203 break;
204 }
205 }
206 }
207
208 /// <summary>
209 /// Relic contents. Every relic is listed once per refinement; only Intact and Radiant are kept.
210 /// </summary>
211 private static void ParseRelics(string section, Builder builder)
212 {
213 string? relic = null;
214 string? refinement = null;
215
216 foreach (var row in Rows(section))
217 {
218 switch (row)
219 {
220 case [{ Header: true, FullWidth: true, Text: var header }]:
221 var match = RelicHeaderPattern.Match(header);
222 (relic, refinement) = match.Success
223 ? (match.Groups["relic"].Value, match.Groups["refinement"].Value)
224 : (null, null);
225 break;
226
227 case [{ Header: false, Text: var item }, { Header: false, Text: var chance }]
228 when relic is not null && Chance(chance) is (var rarity, var pct):
229 builder.AddRelicLine(relic, item, rarity, pct, refinement == "Radiant");
230 break;
231 }
232 }
233 }
234
235 private static bool IsRotation(string header) =>
236 header.StartsWith("Rotation", StringComparison.OrdinalIgnoreCase);
237
238 private static string? Join(string? rotation, string? stage) =>
239 (rotation, stage) switch
240 {
241 (null, null) => null,
242 (null, var s) => s,
243 (var r, null) => r,
244 var (r, s) => $"{r} · {s}"
245 };
246
247 private static (string Rarity, double Percent)? Chance(string text)
248 {
249 var match = ChancePattern.Match(text);
250 return match.Success
251 ? (match.Groups["rarity"].Value, Percent(match.Groups["pct"].Value))
252 : null;
253 }
254
255 private static double Percent(string text) =>
256 double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var value) ? value : 0;
257
258 /// <summary>One cell: what it says, and the two attributes the grammars branch on.</summary>
259 private readonly record struct Cell(string Text, bool Header, bool FullWidth, bool Spacer);
260
261 private static IEnumerable<Cell[]> Rows(string section)
262 {
263 foreach (Match row in RowPattern.Matches(section))
264 {
265 var cells = CellPattern.Matches(row.Groups["body"].Value);
266 if (cells.Count == 0) continue;
267
268 var parsed = new Cell[cells.Count];
269 for (var i = 0; i < cells.Count; i++)
270 {
271 var attrs = cells[i].Groups["attrs"].Value;
272 parsed[i] = new Cell(
273 Text: cells[i].Groups["text"].Value.Trim(),
274 Header: cells[i].Groups["tag"].Value == "th",
275 FullWidth: attrs.Contains("colspan", StringComparison.Ordinal),
276 Spacer: attrs.Contains("pad-cell", StringComparison.Ordinal));
277 }
278
279 yield return parsed;
280 }
281 }
282
283 /// <summary>Collects rows as they are read, then sorts and indexes them.</summary>
284 private sealed class Builder
285 {
286 private readonly Dictionary<string, List<DropSource>> _sources = new(StringComparer.OrdinalIgnoreCase);
287 private readonly Dictionary<(string Relic, string Item), RelicLine> _relicLines = new();
288
289 public void AddSource(string item, DropSource source)
290 {
291 if (item.Length == 0) return;
292
293 if (!_sources.TryGetValue(item, out var list))
294 {
295 _sources[item] = list = [];
296 }
297
298 list.Add(source);
299 }
300
301 public void AddRelicLine(string relic, string item, string rarity, double chance, bool radiant)
302 {
303 if (item.Length == 0) return;
304
305 var existing = _relicLines.GetValueOrDefault((relic, item))
306 ?? new RelicLine(relic, item, rarity, Intact: 0, Radiant: 0);
307
308 _relicLines[(relic, item)] = radiant
309 ? existing with { Radiant = chance }
310 : existing with { Rarity = rarity, Intact = chance };
311 }
312
313 public DropTables Build(string lastUpdate, DateTimeOffset fetchedAt)
314 {
315 var byRelic = _relicLines.Values
316 .GroupBy(line => line.Relic, StringComparer.OrdinalIgnoreCase)
317 .ToDictionary(
318 group => group.Key,
319 group => (IReadOnlyList<RelicLine>)group.OrderByDescending(line => line.Intact).ToArray(),
320 StringComparer.OrdinalIgnoreCase);
321
322 var byItem = _relicLines.Values
323 .GroupBy(line => line.Item, StringComparer.OrdinalIgnoreCase)
324 .ToDictionary(
325 group => group.Key,
326 group => (IReadOnlyList<RelicLine>)group.OrderByDescending(line => line.Intact).ToArray(),
327 StringComparer.OrdinalIgnoreCase);
328
329 // A relic is both an item you farm and a container of items, so both indexes and the
330 // drop sources meet under one name.
331 var names = _sources.Keys
332 .Concat(byRelic.Keys)
333 .Concat(byItem.Keys)
334 .Distinct(StringComparer.OrdinalIgnoreCase)
335 .OrderBy(name => name, StringComparer.OrdinalIgnoreCase)
336 .ToArray();
337
338 var items = names.ToDictionary(
339 name => name,
340 name => new ItemDrops
341 {
342 Name = name,
343 Sources = _sources.TryGetValue(name, out var sources)
344 ? sources.OrderByDescending(source => source.Chance).ToArray()
345 : [],
346 InRelics = byItem.GetValueOrDefault(name, []),
347 Contents = byRelic.GetValueOrDefault(name, [])
348 },
349 StringComparer.OrdinalIgnoreCase);
350
351 return new DropTables
352 {
353 LastUpdate = lastUpdate,
354 FetchedAt = fetchedAt,
355 Items = items,
356 Names = names
357 };
358 }
359 }
360 }