add query page

author
Marijn Besseling <njirambem@gmail.com> · 2026-01-26 21:15 UTC
commit
e06f8b12764f4a8c0ee24baf9ca234989cd32704
parent
0ea1895745
tree
browse at this commit

4 files changed +2093 -0

Blog/Components/App.razor +1 -0

@@ -54,6 +54,7 @@
54 54 <li><NavLink href="/qrcodeScan">QR-code scanner</NavLink></li>
55 55 <li><NavLink href="/qrcodeGenerator">QR-code generator</NavLink></li>
56 56 <li><NavLink href="/BRPTestData">BRP Test Data</NavLink></li>
57 + <li><NavLink href="/Query">Query</NavLink></li>
57 58 <li><NavLink href="/Storage">Storage</NavLink></li>
58 59 <li><NavLink href="https://git.bes.is">Git</NavLink></li>
59 60 <!-- <li><a href="/webrtc.html">WebRTC</a></li> -->

Blog/Components/Pages/Query.razor +21 -0

@@ -0,0 +1,21 @@
1 +@page "/Query"
2 +<PageTitle>Query</PageTitle>
3 +<PageScript Src="./Components/Pages/Query.razor.js"/>
4 +
5 +<main>
6 + <textarea id="input" style="height:20vh" autofocus>personen.gezag:type=EenhoofdigOuderlijkGezag</textarea>
7 + <div id="results">
8 + </div>
9 + <Log/>
10 +
11 + <fieldset class="docs">
12 + <legend>Credits</legend>
13 + <div>
14 + <a href="https://dactal.org">DACTAL</a> by <a href="https://furia.com">glenn mcdonald</a>
15 + </div>
16 + <div>
17 + personen dataset from <a href="https://github.com/BRP-API/Haal-Centraal-BRP-bevragen">Haal-Centraal-BRP-bevragen
18 + mock-data</a>
19 + </div>
20 + </fieldset>
21 +</main>
No newline at end of file

Blog/Components/Pages/Query.razor.js +63 -0

@@ -0,0 +1,63 @@
1 +import {getById, h, span, addClass, div} from '/common.module.js'
2 +import {DACTAL} from "/dactal.js"
3 +
4 +const enterEvent = new KeyboardEvent('keydown', {
5 + key: 'Enter',
6 + code: 'Enter',
7 + keyCode: 13,
8 + which: 13,
9 + bubbles: true,
10 + cancelable: true
11 +});
12 +
13 +let dactal
14 +
15 +export async function onLoad() {
16 + const input = getById("input")
17 + dactal = new DACTAL()
18 +
19 + const brp = await fetch("/brp.json")
20 + const cdata = await brp.json()
21 +
22 + dactal.data = {
23 + personen: cdata
24 + }
25 +
26 + input.onkeydown = async (e) => {
27 + if (e.key === "Enter" && !e.shiftKey) {
28 + e.preventDefault()
29 + let result = await dactal.query(input.value)
30 +
31 + const results = getById("results")
32 + results.replaceChildren(renderValue(result.slice(0, 100)))
33 +
34 + if (result.length > 100) {
35 + results.appendChild(div(span(`Not showing ${result.length - 100} more results.`)))
36 + }
37 + }
38 + }
39 +
40 + input.dispatchEvent(enterEvent)
41 +}
42 +
43 +function renderValue(obj) {
44 + if (Array.isArray(obj)) {
45 + return h("ol",
46 + ...obj.map((item) => h("li", renderValue(item)))
47 + )
48 + } else if (typeof obj === "object") {
49 + return h("ul",
50 + ...Object.entries(obj).map(([name, item]) =>
51 + addClass(h("li",
52 + addClass(span(name), "name"),
53 + span(" "),
54 + renderValue(item)),
55 + "dash")))
56 + } else if (typeof obj === "boolean") {
57 + return addClass(span(obj), obj ? "json-true" : "json-false")
58 + } else if (typeof obj === "number") {
59 + return addClass(span(obj), "json-number")
60 + } else if (typeof obj === "string") {
61 + return span(obj)
62 + }
63 +}
No newline at end of file

Blog/wwwroot/dactal.js +2008 -0

@@ -0,0 +1,2008 @@
1 +export class DACTAL {
2 + constructor(data={}) {
3 + this.data = data;
4 + this.index = {};
5 + this.index_modified = false;
6 + this.adapters = {};
7 + this.data['query history'] = [];
8 + this.savedquerynames = new Set();
9 + this.debug = false;
10 + this.recache = false;
11 + this.timelimit = 120000;
12 + this.statusf = (statusmsg) => console.log(statusmsg);
13 + this.aggregators = {
14 + group: (item) => item.of,
15 + count: (item) => this.vals(item).length,
16 + total: (item) => this.numvals(item).reduce((acc, val) => acc + (this.dtype(val, 'number') ? val : 0), 0),
17 + average: (item) => this.numvals(item).reduce((acc, val) => acc + (this.dtype(val, 'number') ? val : 0), 0) / this.numvals(item).length,
18 + min: (item) => {
19 + const nums = this.numvals(item);
20 + return nums.length === 0 ? [] : Math.min(...nums)
21 + },
22 + max: (item) => {
23 + const nums = this.numvals(item);
24 + return nums.length === 0 ? [] : Math.max(...nums)
25 + },
26 + product: (item) => this.numvals(item).reduce((acc, val) => acc * (this.dtype(val, 'number') ? val : 1), 1),
27 + difference: (item) => this.numvals(item).reduce((acc, val) => acc - (this.dtype(val, 'number') ? val : 0)),
28 + quotient: (item) => this.numvals(item).reduce((acc, val) => acc / (this.dtype(val, 'number') ? val : 1)),
29 + percent: (item) => Math.round(this.numvals(item).reduce((acc, val) => acc / (this.dtype(val, 'number') ? val : 1)) * 100),
30 + sqrt: (item) => Math.sqrt(this.numvals(item)[0]),
31 + log: (item) => Math.log(this.numvals(item)[0]),
32 + log10: (item) => Math.log10(this.numvals(item)[0]),
33 + abs: (item) => Math.abs(this.numvals(item)[0]),
34 + is: (item) => (item.of?.length > 0) ? 1 : 0,
35 + isnt: (item) => (item.of?.length === 0) ? 1 : 0,
36 + yesno: (item) => (item.of?.length > 0) ? 'yes' : 'no',
37 + missing: (item) => (item.of?.length === 0) ? true : null,
38 + concatenate: (item) => this.vals(item).join(' '),
39 + join: (item) => item.of.map(this.getname).join(this.vals(item)[0]),
40 + str: (item) => item.of.map(this.getname).join(''),
41 + 'to json': (item) => JSON.stringify(item.of),
42 + quote: (item) => `“${this.vals(item)[0]}”`,
43 + url: (item) => {
44 + let u = item.of.map(this.getname).join('');
45 + if (!u.startsWith('https://')) u = 'https://' + u;
46 + for (const prop of Object.keys(item).filter((key) => key !== 'of')) {
47 + const val = item[prop];
48 + (Array.isArray(val) ? val : [val]).forEach((vv) => u = u + (u.match(/\?/) ? '&' : '?') + encodeURIComponent(prop) + '=' + encodeURIComponent(vv));
49 + }
50 + return u;
51 + },
52 + remove: (item) => this.vals(item).reduce((acc, val) => acc.replaceAll(val, '')),
53 + bmk: (item) => this.vals(item).map((v) => v.toString().toLowerCase().replaceAll(/[$\xA2-\xA5\u058F\u060B\u09F2\u09F3\u09FB\u0AF1\u0BF9\u0E3F\u17DB\u20A0-\u20BD\uA838\uFDFC\uFE69\uFF04\uFFE0\uFFE1\uFFE5\uFFE6,+]/g, '').replaceAll('b', 'kkk').replaceAll('m', 'kk').replaceAll('k', '000')),
54 + sortform: (item) => this.vals(item).map((val) => val.toString().toLowerCase().replace(/^the /, '')),
55 + zip: (item) => {
56 + const itemkeys = Object.keys(item).filter((k) => k !== 'of');
57 + const zipped = [];
58 + for (let i=0; i<item[itemkeys[0]].length; i++) {
59 + const zipline = {};
60 + for (const key of itemkeys) {
61 + zipline[key] = item[key][i];
62 + }
63 + zipped.push(zipline);
64 + }
65 + return zipped;
66 + },
67 + pairs: (item) => {
68 + return item.of.slice(0, -1).map((val, vx) => ({pair: [val, item.of[vx+1]]}));
69 + },
70 + triples: (item) => {
71 + return item.of.slice(0, -2).map((val, vx) => ({triple: [val, item.of[vx+1], item.of[vx+2]]}));
72 + },
73 + quads: (item) => {
74 + return item.of.slice(0, -3).map((val, vx) => ({quad: [val, item.of[vx+1], item.of[vx+2], item.of[vx+3]]}));
75 + },
76 + sequences: (item) => {
77 + return item.of.map((val, valx, vallist) => ({sequence: vallist.slice(0, valx+1).map((val) => this.dcopy(val))}));
78 + },
79 + split: (item) => {
80 + const itemvals = this.vals(item);
81 + let splitter;
82 + let tobesplit;
83 + if (Object.keys(item).length === 1) itemvals.push(' ');
84 + if (itemvals.length === 1) {
85 + splitter = itemvals[0];
86 + tobesplit = [this.getname(item)];
87 + } else {
88 + splitter = itemvals.pop();
89 + tobesplit = itemvals.slice(0);
90 + }
91 + if (splitter.startsWith('~')) splitter = new RegExp(splitter.replace(/^~*/, ''), splitter.startsWith('~~') ? 'i' : '');
92 + return tobesplit.flatMap((v) => v.toString().split(splitter));
93 + },
94 + extract: (item) => {
95 + const itemvals = this.vals(item);
96 + const delimiters = itemvals.pop();
97 + const res = [];
98 + for (const itemval of itemvals) {
99 + for(let i=0; i<delimiters.length; i+=2) {
100 + const d1 = delimiters[i];
101 + const d2 = delimiters[i+1];
102 + const d1x = itemval.indexOf(d1);
103 + const d2x = itemval.indexOf(d2);
104 + if (d1x > -1 && d2x > d1x) {
105 + res.push(itemval.slice(d1x+1, d2x).trim());
106 + break;
107 + }
108 + }
109 + }
110 + return res;
111 + },
112 + unchain: (item) => {
113 + const chainprops = Object.keys(item).filter((key) => key !== 'of');
114 + const unchained = [];
115 + const queue = item.of.slice(0);
116 + while (queue.length > 0) {
117 + const thisitem = queue.shift();
118 + if (!unchained.includes(thisitem)) {
119 + unchained.push(thisitem);
120 + for (const chainprop of chainprops) {
121 + if (this.dtype(thisitem, 'object') && chainprop in thisitem) {
122 + if (this.dtype(thisitem[chainprop], 'array')) {
123 + for (const x of thisitem[chainprop].slice(0).reverse()) queue.unshift(x);
124 + } else if (thisitem[chainprop]) {
125 + queue.unshift(thisitem[chainprop])
126 + }
127 + }
128 + }
129 + }
130 + }
131 + return unchained;
132 + },
133 + itemize: (item) => {
134 + const propname = item?.property ?? 'property';
135 + const valname = item?.value ?? 'value';
136 + return item.of.flatMap((subitem) => Object.entries(subitem).map(([key, val]) => ({[propname]: key, [valname]: val})));
137 + },
138 + schematize: (item) => {
139 + const itemkeys = Object.keys(item).filter((k) => k !== 'of');
140 + const schematized = {};
141 + item[itemkeys[0]].forEach((subitem) => {
142 + let subkey;
143 + let subval;
144 + if (this.dtype(subitem, 'array')) {
145 + const [subkey, subval] = subitem;
146 + } else {
147 + [subkey, subval] = Object.values(subitem);
148 + }
149 + schematized[subkey] = subval;
150 + });
151 + return schematized;
152 + },
153 + index: (item) => {
154 + return Object.entries(item).filter(([k, v]) => k !== 'of').map(([k, v]) => ({id: k, name: !isNaN(v) ? Number(v) : v}));
155 + },
156 + unflatten: (item) => {
157 + const itemkeys = Object.keys(item).filter((k) => k !== 'of');
158 + if (itemkeys.length === 0) itemkeys.push('');
159 + const newindex = {};
160 + const neworder = [];
161 + item.of.forEach((subitem) => {
162 + Object.keys(subitem).forEach((field) => {
163 + itemkeys.forEach((key) => {
164 + if (field.startsWith(key)) {
165 + const subid = field.replace(key, '');
166 + if (subid.length > 0) {
167 + if (!(subid in newindex)) {
168 + newindex[subid] = {};
169 + neworder.push(subid);
170 + }
171 + newindex[subid][key] = subitem[field];
172 + }
173 + }
174 + });
175 + });
176 + });
177 + itemkeys.forEach((key) => delete item[key]);
178 + return neworder.map((k) => {
179 + const subitem = {};
180 + subitem.subid = k;
181 + Object.assign(subitem, newindex[k]);
182 + return subitem;
183 + });
184 + },
185 + detupled: (item) => {
186 + const newobj = {};
187 + item.of.forEach((subitem) => {
188 + if (Array.isArray(subitem) && subitem.length === 2) {
189 + newobj[subitem[0]] = subitem[1];
190 + }
191 + })
192 + return [newobj];
193 + },
194 + csv: (item) => {
195 + const keys = Object.entries(item).find(([k, v]) => k !== 'of')[1];
196 + const res = [];
197 + const vals = item.of.flatMap((val) => this.dtype(val, 'string') ? val.split('\n').map((val) => val.trim()) : val);
198 + for (let i=0; i<vals.length; i+=keys.length) {
199 + const newitem = {};
200 + for (let k=0; k<keys.length; k++) {
201 + newitem[keys[k]] = vals[i+k];
202 + }
203 + res.push(newitem);
204 + }
205 + return res;
206 + },
207 + tsv: (item) => {
208 + const text = this.getname(item);
209 + const lines = text.split('\n').filter((line) => line !== '').map((line) => line.split('\t').map((val) => val.trim()));
210 + const keys = lines[0];
211 + return lines.slice(1).map((vals) => Object.fromEntries(vals.map((val, vi) => [keys[vi], val])));
212 + },
213 + ssv: (item) => {
214 + const text = this.getname(item);
215 + const lines = text.split('\n').filter((line) => line !== '').map((line) => line.split(/ +/).map((val) => val.trim()));
216 + const keys = lines[0];
217 + return lines.slice(1).map((vals) => Object.fromEntries(vals.map((val, vi) => [keys[vi], val])));
218 + },
219 + json: (item) => this.vals(item).flatMap((v) => JSON.parse(v)),
220 + year: (item) => {
221 + if (item.date?.length > 0) return item.date[0].match(/\d\d\d\d/)[0];
222 + let yearmatch = this.getname(item).match(/\d\d\d\d/);
223 + if (yearmatch) {
224 + return yearmatch[0];
225 + } else {
226 + yearmatch = this.getid(item.of[0] || '').toString().match(/\d\d\d\d/);
227 + if (yearmatch) {
228 + return yearmatch[0];
229 + }
230 + return null;
231 + }
232 + },
233 + month: (item) => {
234 + if (item.date?.length > 0) return item.date[0].match(/\d\d\d\d-(\d\d)-\d\d/)[1];
235 + let monthmatch = this.getname(item).match(/\d\d\d\d-(\d\d)-\d\d/);
236 + if (monthmatch) {
237 + return monthmatch[1];
238 + } else {
239 + monthmatch = this.getid(item.of[0] || '').toString().match(/\d\d\d\d-(\d\d)-\d\d/);
240 + if (monthmatch) {
241 + return monthmatch[1];
242 + }
243 + return null;
244 + }
245 + },
246 + date: (item) => {
247 + let itemvals = this.vals(item);
248 + if (itemvals) {
249 + let datematch = itemvals[0].match(/\d\d\d\d-\d\d-\d\d/);
250 + if (datematch) {
251 + return datematch[0];
252 + } else {
253 + datematch = this.getid(item.of[0] || '').toString().match(/\d\d\d\d-\d\d-\d\d/);
254 + if (datematch) {
255 + return datematch[0];
256 + }
257 + }
258 + }
259 + return null;
260 + },
261 + weekday: (item) => {
262 + const days = ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'];
263 + if (item.date?.length > 0) return days[new Date(item.date).getDay()];
264 + },
265 + timeshift: (item) => {
266 + const vals = this.vals(item);
267 + let tsx = new Date(vals[0]);
268 + let adjust = Number(vals[1]) * 60*60*1000;
269 + tsx.setTime(tsx.getTime() + adjust);
270 + return tsx.toISOString();
271 + },
272 + hour: (item) => this.vals(item).map((v) => v.split('T')[1].split(':')[0]),
273 + datediff: (item) => {
274 + const [d1, d2] = this.vals(item);
275 + return (new Date(d2) - new Date(d1)) / (24*60*60*1000);
276 + },
277 + timediff: (item) => {
278 + const [d1, d2] = this.vals(item);
279 + return (new Date(d2) - new Date(d1));
280 + },
281 + dateforms: (item) => {
282 + const basedate = this.vals(item)[0];
283 + const [baseyear, basemonth, baseday] = basedate.split('-');
284 + return [
285 + basedate,
286 + `${basemonth}/${baseday}/${baseyear}`,
287 + basemonth.startsWith('0') ? `${basemonth.replace(/^0/,'')}/${baseday}/${baseyear}` : null,
288 + basemonth.startsWith('0') && baseday.startsWith('0') ? `${basemonth.replace(/^0/,'')}/${baseday.replace(/^0/,'')}/${baseyear}` : null,
289 + `${basemonth}/${baseday}/${baseyear.slice(2)}`,
290 + `${basemonth.replace(/^0/,'')}/${baseday.replace(/^0/,'')}/${baseyear.slice(2)}`,
291 + `${['', 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'][Number(basemonth)]} ${baseday.replace(/^0/,'')}, ${baseyear}`
292 + ].filter(x => x);
293 + },
294 + now: (item) => performance.now(),
295 + round: (item) => this.numvals(item).map((v) => Math.round(v)),
296 + roundaway: (item) => this.numvals(item).map((v) => Math.sign(v) * Math.round(Math.abs(v))),
297 + roundm: (item) => {
298 + const vals = this.numvals(item);
299 + const multiple = vals.pop();
300 + return vals.map((v) => Math.round(v / multiple) * multiple);
301 + },
302 + roundd: (item) => {
303 + const vals = this.numvals(item);
304 + const digits = vals.pop();
305 + return vals.map((v) => {
306 + const factor = 10 ** (Math.floor(Math.log10(v)) - digits + 1);
307 + return Math.round(v / factor) * factor;
308 + });
309 + },
310 + floor: (item) => this.numvals(item).map((v) => Math.floor(v)),
311 + ceil: (item) => this.numvals(item).map((v) => Math.ceil(v)),
312 + n: (item) => {
313 + let i = 0;
314 + const test = this.getname(item.of[0]);
315 + for (const key in item) {
316 + if (key !== 'of') {
317 + i--;
318 + if (key === test) return i;
319 + }
320 + }
321 + return i - 1;
322 + },
323 + numbers: (item) => {
324 + const res = [];
325 + const firstnum = item.from !== undefined && item.from != null ? Number(item.from) : 1;
326 + const lastnum = item.to !== undefined && item.to != null ? Number(item.to) : Number(this.getname(item.of[0]));
327 + for(let x=firstnum; x<=lastnum; x++) {
328 + res.push(x)
329 + }
330 + return res;
331 + },
332 + dehyphenate: (item) => this.vals(item).flatMap((v) => v.replaceAll(/(?<=\w)-\n(?=\w)/g, '')),
333 + sentences: (item) => this.vals(item).flatMap((v) => v.split(/[.?!…]['"’”»]?\s+/)),
334 + allwords: (item) => this.vals(item).flatMap((v) => Array.from(v.toString().toLowerCase().matchAll(/[\p{Letter}\p{Number}]+(?:['‘’][\p{Letter}\p{Number}]+)?/gu).map((m) => m[0]))),
335 + words: (item) => this.vals(item).flatMap((v) => Array.from(v.toString().toLowerCase().matchAll(/[\p{Letter}\p{Number}]+(?:['‘’][\p{Letter}\p{Number}]+)?/gu).map((m) => m[0])).filter((w) => w.length >= 4)),
336 + characters: (item) => this.vals(item).flatMap((v) => Array.from(v)),
337 + 'character count': (item) => this.vals(item).flatMap((v) => v.length),
338 + case: (item) => this.vals(item).map((v) => {
339 + const hasupper = v.match(/[A-Z]/);
340 + const haslower = v.match(/[a-z]/);
341 + if (!hasupper && !haslower) {
342 + return 'none';
343 + } else if (hasupper && !haslower) {
344 + return 'upper';
345 + } else if (haslower && !hasupper) {
346 + return 'lower';
347 + } else {
348 + if (v[0].match(/[A-Z]/) && !(v.slice(1).match(/[A-Z]/))) {
349 + return 'initial';
350 + } else {
351 + return 'mixed';
352 + }
353 + }
354 + }),
355 + uppercase: (item) => this.vals(item).flatMap((v) => v.toString().toUpperCase()),
356 + lowercase: (item) => this.vals(item).flatMap((v) => v.toString().toLowerCase().replaceAll(/[‘’`]/g, "'").replaceAll(/[“”]/g, '"')),
357 + list: (item) => Object.keys(item).filter((k) => k !== 'of'),
358 + items: (item) => item?.of?.flatMap((typename) => this.data?.[this.getname(typename)]),
359 + random: (item) => Math.random(),
360 + shuffle: (item) => {
361 + const newArray = [].concat(item?.of || []);
362 + for (let i = newArray.length - 1; i > 0; i--) {
363 + const j = Math.floor(Math.random() * (i + 1));
364 + [newArray[i], newArray[j]] = [newArray[j], newArray[i]];
365 + }
366 + return newArray;
367 + },
368 + 'weighted shuffle': (item) => {
369 + return item.of.map((subitem) => ({weight: Number(subitem.weight || 1) * Math.random(), subitem: subitem})).sort((a, b) => b.weight - a.weight).map((ws) => ws.subitem);
370 + },
371 + pick: (item) => item?.of?.[~~(Math.random() * item?.of?.length)],
372 + link: (item) => `<a href="${item.url}"${item.target ? ' target="' + item.target + '"' : ''}>${item.text}</a>`,
373 + img: (item) => {
374 + if (item.uri) {
375 + return `<a href="${item.uri}"><img src="${item.src}" height=${item.height}px width=${item.width}px></a>`;
376 + } else {
377 + return `<img src="${item.src}" height=${item.height}px width=${item.width}px>`;
378 + }
379 + },
380 + sign: (item) => item.of.map((subitem) => {
381 + if (subitem.id) {
382 + return subitem;
383 + } else {
384 + const str = JSON.stringify(subitem);
385 + let hash = 0;
386 + for (let i = 0; i < str.length; i++) {
387 + const char = str.charCodeAt(i);
388 + hash = (hash << 5) - hash + char;
389 + }
390 + // Convert to 32bit unsigned integer in base 36 and pad with "0" to ensure length is 7.
391 + return Object.assign({id: (hash >>> 0).toString(36).padStart(7, '0')}, subitem);
392 + }
393 + }),
394 + 'parse query': (item) => item.of.map((subitem) => ({name: subitem.name, query: subitem.query, assembly: this.parse(subitem.query)}))
395 + }
396 + this.data.aggregators = Object.keys(this.aggregators);
397 + // this.cachehitcount = 0;
398 + // this.cachemisscount = {};
399 + this.destinations = new Set();
400 + }
401 +
402 + survey() {
403 + this.destinations = new Set(Object.keys(this.data).concat(this.data?.queries?.map((q) => q.name) || []).concat(Object.keys(this.adapters)));
404 + }
405 +
406 + vals(item) {
407 + const getname = this.getname;
408 + const itemvals = [];
409 + if (Object.keys(item).length > 1) {
410 + Object.entries(item).filter(([k, v]) => k !== 'of').flatMap(([k, v]) => Array.isArray(v) ? v : [v]).map(getname).forEach((v) => itemvals.push(v));
411 + } else {
412 + item.of.map(getname).forEach((v) => itemvals.push(v));
413 + }
414 + return itemvals;
415 + }
416 +
417 + numvals(item) {
418 + return this.vals(item).filter((val) => this.dtype(val, 'number')).map((val) => Number(val));
419 + }
420 +
421 + async querylive(query, inputlist=null) {
422 + this.recache = new Set();
423 + return await this.query(query, inputlist);
424 + }
425 +
426 + async query(query, inputlist=null, loop=50) {
427 + this.survey();
428 + const operations = Array.isArray(query) ? query : this.assemble(this.tokenize(query));
429 + const result = this.execute(inputlist, operations);
430 + this.data['current results'] = result;
431 + const queued = Object.keys(this.adapters).filter((key) => this.adapters[key].queue.length > 0);
432 + if (loop > 0 && queued.length > 0) {
433 + await this.adapt();
434 + if (this.recache) queued.forEach((q) => this.recache.add(q));
435 + return await this.query(operations, inputlist, loop - 1);
436 + } else if (loop === 0) {
437 + this.recache = false;
438 + Object.keys(this.adapters).forEach((key) => {
439 + if (!this.adapters[key].annotator) {
440 + this.adapters[key].queue.forEach((id) => {
441 + (this.index[key] ||= {})[id] = null;
442 + })
443 + }
444 + })
445 + }
446 + return result;
447 + }
448 +
449 + async adapt() {
450 + for (const key in this.adapters) {
451 + const adapter = this.adapters[key];
452 + if (adapter.queue.length > 0) {
453 + if (adapter.annotator) {
454 + while (adapter.queue.length > 0) {
455 + const item_to_annotate = adapter.queue[0];
456 + try {
457 + (this.index[key] ||= {})[this.getid(item_to_annotate)] = await adapter.f(item_to_annotate);
458 + this.index_modified = true;
459 + adapter.queue.shift();
460 + } catch (e) {}
461 + }
462 + } else {
463 + const newids = adapter.queue.filter((id) => !adapter.pending.has(id));
464 + newids.forEach((id) => adapter.pending.add(id));
465 + const res = await adapter.f(newids);
466 + newids.forEach((id) => adapter.pending.delete(id));
467 + for (let i=0; i<adapter.queue.length; i++) {
468 + const item = res[i];
469 + const id = this.getid(item) ?? adapter.queue[i];
470 + (this.index[key] ||= {})[id] = item;
471 + this.index_modified = true;
472 + if (key in this.data) {
473 + this.data[key].push(item);
474 + }
475 + }
476 + this.adapters[key].queue = [];
477 + }
478 + }
479 + }
480 + }
481 +
482 + async rehope() {
483 + Object.keys(this.index).forEach((k) => {
484 + const todelete = Object.keys(this.index[k]).filter((dk) => this.index[k][dk] == null || this.index[k][dk] === undefined);
485 + todelete.forEach((dk) => delete(this.index[k][dk]));
486 + });
487 + }
488 +
489 + load(something, named, append=false) {
490 + if (!named) return
491 + if (!append || !(named in this.data)) this.data[named] = [];
492 + if (Array.isArray(something)) {
493 + something.forEach((somethingx) => this.data[named].push(somethingx));
494 + } else if (typeof something == 'object') {
495 + const firstkey = Object.keys(something)[0];
496 + const firstobj = something[firstkey];
497 + if (typeof firstobj === 'object' && !Array.isArray(firstobj) && (firstkey in Object.values(firstobj) || 'id' in firstobj)) {
498 + this.data[named].push(...Object.values(something));
499 + } else if (typeof firstobj === 'object' && !Array.isArray(firstobj)) {
500 + this.data[named].push(...Object.entries(firstobj).map(([key, val]) => ({id: key, ...val})));
501 + } else if (typeof firstobj === 'string') {
502 + this.data[named].push(...Object.entries(something).map(([key, val]) => ({id: key, name: val})));
503 + } else {
504 + this.data[named].push(...Object.entries(something).map(([key, val]) => ({id: key, value: val})));
505 + }
506 + } else {
507 + this.data[named].push(something);
508 + }
509 + return this.data[named];
510 + }
511 +
512 + async loadjsonl(something, named, append=false) {
513 + if (!named) return
514 + if (!append || !(named in this.data)) this.data[named] = [];
515 + if (typeof something == 'string' && (something.startsWith('http') || something.startsWith('file://'))) {
516 + const fetchres = await fetch(something);
517 + something = await fetchres.text();
518 + }
519 + const rows = something.trim().split(/[\n\r]+/);
520 + for (const row of rows) {
521 + const rowdata = JSON.parse(row);
522 + if (rowdata) this.data[named].push(rowdata);
523 + }
524 + }
525 +
526 + async loadcsv(something, named, quoteChar = '"', delimiter = ',', headerrows=1) {
527 + if (typeof something == 'string' && (something.startsWith('http') || something.startsWith('file://'))) {
528 + const fetchres = await fetch(something);
529 + something = await fetchres.text();
530 + }
531 + const rows = something.split(/[\n\r]+/);
532 +
533 + const regex = new RegExp(`\\s*(${quoteChar})?(.*?)\\1\\s*(?:${delimiter}|$)`, 'gs');
534 +
535 + const match = (line) => Array.from(line.matchAll(regex), (m) => m[2]);
536 +
537 + const headers = [];
538 + for (let hrowx=0; hrowx<headerrows; hrowx++) {
539 + const hrow = rows.shift();
540 + match(hrow).forEach((h, hx) => {
541 + if (hrowx === 0) {
542 + headers.push(h);
543 + } else {
544 + headers[hx] = headers[hx] + ' ' + h;
545 + }
546 + });
547 + }
548 + const heads = headers ?? match(rows.shift());
549 + const lines = rows.slice(0).filter((line) => line);
550 + const parsed = lines.map((line) => {
551 + return match(line).reduce((acc, cur, i) => {
552 + // replace blank matches with `null`
553 + const val = cur.length <= 0 ? null : (!isNaN(cur) ? Number(cur) : cur);
554 + const key = heads[i] ?? `{i}`;
555 + if (key === '') {
556 + return { ...acc};
557 + } else {
558 + return { ...acc, [key]: val };
559 + }
560 + }, {});
561 + });
562 + this.load(parsed, named);
563 + }
564 +
565 + apacheLogToDate(apacheTimestamp) {
566 + // Apache log format: [10/Oct/2000:13:55:36 -0700]
567 + // Remove brackets if present
568 + const cleanTimestamp = apacheTimestamp.replace(/^\[|]$/g, '');
569 +
570 + // Split into date/time and timezone parts
571 + const [dateTimePart, timezone] = cleanTimestamp.split(' ');
572 +
573 + // Parse the date/time part: dd/MMM/yyyy:HH:mm:ss
574 + const [datePart, hour, minute, second] = dateTimePart.split(':');
575 + const [day, month, year] = datePart.split('/');
576 +
577 + // Month mapping
578 + const months = {
579 + 'Jan': 0, 'Feb': 1, 'Mar': 2, 'Apr': 3, 'May': 4, 'Jun': 5,
580 + 'Jul': 6, 'Aug': 7, 'Sep': 8, 'Oct': 9, 'Nov': 10, 'Dec': 11
581 + };
582 +
583 + // Create Date object (months are 0-indexed in JS)
584 + const date = new Date(
585 + parseInt(year),
586 + months[month],
587 + parseInt(day),
588 + parseInt(hour),
589 + parseInt(minute),
590 + parseInt(second)
591 + );
592 +
593 + // Handle timezone offset if present
594 + if (timezone) {
595 + const sign = timezone[0] === '+' ? 1 : -1;
596 + const tzHours = parseInt(timezone.slice(1, 3));
597 + const tzMinutes = parseInt(timezone.slice(3, 5));
598 + const offsetMs = sign * (tzHours * 60 + tzMinutes) * 60 * 1000;
599 +
600 + // Adjust for timezone (Apache logs are in local time, JS Date assumes UTC)
601 + date.setTime(date.getTime() - offsetMs);
602 + }
603 +
604 + return date;
605 + }
606 +
607 + async loadclf(loglines, named, apiroutes=[]) {
608 + const lines = loglines.trim().split('\n');
609 + const cols9 = ['ip', 'name', 'username', 'timestamp', 'requestraw', 'status', 'bytes', 'referrer', 'useragent'];
610 + const cols10 = ['host'].concat(cols9);
611 + const parsed = lines.map((line) => {
612 + const vals = Array.from(line.matchAll(/"(?:\\"|.)*?"|\[.*?]|\S+/g)).map((m) => m[0]);
613 + const cols = vals.length === 10 ? cols10 : cols9;
614 + const obj = Object.fromEntries(vals.map((v, vx) => ([cols[vx], v])));
615 + const date = this.apacheLogToDate(obj.timestamp);
616 + obj.timestamp = date.toISOString();
617 + [obj.date, obj.time] = obj.timestamp.slice(0, -1).split('T');
618 + if (obj.requestraw.match(/ [^ ]+ /) && !obj.requestraw.startsWith('"{')) {
619 + [obj.method, obj.request, obj.protocol] = obj.requestraw.slice(1, -1).split(' ');
620 + if (this.data['.clf API routes']) {
621 + for (const apiroute of this.data['.clf API routes'].sort((a, b) => b.length - a.length)) {
622 + if (obj.request.startsWith(apiroute)) {
623 + obj.page = apiroute;
624 + break;
625 + }
626 + }
627 + }
628 + obj.page ??= obj.request.split('?')[0];
629 + }
630 + obj.referrer = obj.referrer.slice(1, -1);
631 + obj.useragent = obj.useragent.slice(1, -1);
632 + obj.logline = line;
633 + return obj;
634 + });
635 + this.load(parsed, named);
636 + }
637 +
638 + async loadrss(rsstext, named) {
639 + const rssval = (k, rawval) => {
640 + if (k.match(/date/i)) {
641 + return new Date(rawval).toISOString();
642 + } else if (!isNaN(rawval)) {
643 + return Number(rawval);
644 + } else {
645 + return rawval;
646 + }
647 + }
648 + const rssdom = new window.DOMParser().parseFromString(rsstext, "text/xml");
649 + const items = Array.from(rssdom.querySelectorAll('item')).map((i) => {
650 + const obj = {};
651 + Array.from(i.children).forEach((c) => {
652 + const k = c.tagName;
653 + const rawval = c.textContent;
654 + obj[k] = rssval(k, rawval);
655 + });
656 + return obj;
657 + });
658 + this.load(items, named);
659 + }
660 +
661 + connect(key, adapter, annotator=false) {
662 + this.adapters[key] = {queue: [], pending: new Set(), f: adapter, annotator: annotator};
663 + this.data.adapters ??= [];
664 + if (!this.data.adapters.includes(key)) this.data.adapters.push(key);
665 + }
666 +
667 + connect_annotator(key, adapter) {
668 + this.connect(key, adapter, true);
669 + }
670 +
671 + register(key, aggregator) {
672 + this.aggregators[key] = aggregator;
673 + this.data.aggregators ??= [];
674 + if (!this.data.aggregators.includes(key)) this.data.aggregators.push(key);
675 + }
676 +
677 + unbracket(token) {
678 + if (!(typeof token == 'string')) token = token.toString();
679 + if (token.startsWith('[') && token.endsWith(']')) {
680 + return token.slice(1, -1).replaceAll(']]', ']');
681 + }
682 + return token;
683 + }
684 +
685 + bracket(token) {
686 + if (!(typeof token == 'string')) token = token?.toString() ?? '';
687 + if (token.match(/[?.:#\/|!<>=~@\[\](),;+-]/) || token.startsWith(' ') || token.endsWith(' ')) {
688 + return '[' + token.replaceAll(']', ']]') + ']';
689 + }
690 + return token;
691 + }
692 +
693 + escapeRegExp(string) {
694 + return string.replace(/[.*+?${}()|[\]\\]/g, "\\$&");
695 + }
696 +
697 + tokenize(text) {
698 + if (typeof text != 'string') text = String(text);
699 + const matches = text.matchAll(/(\[(?:]]|[^\]])*])|(\?{1,3})|(\.{1,4})|(\|{1,2})|(\/{1,2})|([:#!])|(\()|(\))|([@~=<>+-]+)|(,)|(;)|([^\[\]().?:\/#|!~=<>@,;+-]+)|([\[\]])/gms);
700 + return Array.from(matches, m => m[0].trim()).filter((token) => token.length > 0);
701 + }
702 +
703 + assemble(tokenized) {
704 + const isOperator = (token) => '???....://#||!'.includes(token);
705 + const tokenlist = '= ~ =~ ~< ~> > < <> >< >= <= - =- ~- + =+ @ @@ @- @= @@= =@ =@@ @< @@< @<= @@<= @> @@> @>= @@>= >> >>= << <<= ~~ =>'.split(' ');
706 + const isSubop = (token) => tokenlist.includes(token) || (token?.startsWith('-') && (token.length === 1 || tokenlist.includes(token.slice(1))));
707 + const isSeparator = (token) => ',;'.includes(token);
708 + const isValue = (token) => !isOperator(token) && !isSubop(token) && !isSeparator(token);
709 +
710 + let tokens = tokenized.slice();
711 + let level = 0;
712 + const operations = [];
713 + while (tokens.length > 0) {
714 + const op = {operator: null, args: []};
715 + let token = tokens.shift();
716 + if (isOperator(token) || ((isValue(token) || isSubop(token)) && operations.length === 0)) {
717 + if ((isValue(token) || isSubop(token)) && operations.length === 0) {
718 + op.operator = '?';
719 + tokens.unshift(token);
720 + } else {
721 + op.operator = token;
722 + }
723 + while (tokens.length > 0 && !isOperator(tokens[0])) {
724 + const arg = {separator: null, label: null, subop: null, value: null};
725 + if (isSeparator(tokens[0])) {
726 + arg.separator = tokens.shift();
727 + }
728 + while (tokens.length > 0 && !isOperator(tokens[0]) && !isSeparator(tokens[0])) {
729 + const frag = tokens.shift();
730 + if (frag !== '(' && isValue(frag) && isSubop(tokens[0])) {
731 + arg.label = this.unbracket(frag);
732 + arg.subop = tokens.shift();
733 + } else if (isSubop(frag)) {
734 + arg.subop = frag;
735 + } else if (frag === '(') {
736 + const subquery = [];
737 + level++;
738 + while (tokens.length > 0 && level > 0) {
739 + const sub = tokens.shift();
740 + if (sub === '(') {
741 + level++;
742 + if (level > 0) {
743 + subquery.push(sub);
744 + }
745 + } else if (sub === ')') {
746 + level--;
747 + if (level > 0) {
748 + subquery.push(sub);
749 + }
750 + } else if (level > 0) {
751 + subquery.push(sub);
752 + }
753 + }
754 + arg.value = this.assemble(subquery);
755 + } else if (!arg.value) {
756 + arg.value = this.unbracket(frag);
757 + } else {
758 + const failure = {tokens: tokenized, assembled: operations.slice(0), assembling: {op: op, arg: arg, unexpected: frag}, unassembled: tokens.slice(0)}
759 + throw new Error("Unexpected token", {cause: failure});
760 + }
761 + }
762 + op.args.push(arg);
763 + }
764 + }
765 + operations.push(op);
766 +
767 + }
768 + if (level > 0) console.warn({parentropy: level, operations: operations});
769 + return operations;
770 + }
771 +
772 + parse(querystr) {
773 + return this.assemble(this.tokenize(querystr));
774 + }
775 +
776 + disassemble(assembly) {
777 + return assembly.map((operation, ox) => operation.operator + ((operation.args.length === 0 && ox < assembly.length - 1 && operation.operator[0] === assembly[ox + 1].operator[0]) ? ' ' : operation.args.map((arg) => (arg.separator ?? '') + (arg.label ? this.bracket(arg.label) : '') + (arg.subop ?? '') + (Array.isArray(arg.value) ? ('(' + this.disassemble(arg.value) + ')') : (arg.value ? this.bracket(arg.value) : ''))).join(''))).join('');
778 + }
779 +
780 + compact(querystr) {
781 + return this.disassemble(this.parse(querystr));
782 + }
783 +
784 + executeq(querystr) {
785 + return this.execute([], this.assemble(this.tokenize(querystr)));
786 + }
787 +
788 + timecheck(timer, i, count, op) {
789 + if (i === 1) timer.loopstart = new Date();
790 + if (i >= 10) {
791 + const taken = new Date() - timer.loopstart;
792 + const projected = count * taken / i;
793 + if (projected > this.timelimit) throw new Error('Query overrun.', {cause: {operation: dactal.disassemble([op]), items: count, done: i, elapsed: taken + 'ms', projected: Math.round(projected / 60000) + ' minutes', timelimit: Math.round(this.timelimit / 60000) + ' minutes'}});
794 + }
795 + }
796 +
797 + execute(inputlist, operations, labeled=null, level=null) {
798 + let currentlist = inputlist?.slice(0) || [];
799 + const getname = this.getname;
800 + const getid = this.getid;
801 + const dtype = this.dtype;
802 + const dcopy = this.dcopy;
803 + const step = this.step;
804 + const escapeRegExp = this.escapeRegExp;
805 + const unmmss = (mmssstr) => {
806 + const parts = mmssstr.split(':');
807 + let s = 0;
808 + const seconds = parts.pop();
809 + if (seconds) s += Number(seconds);
810 + const minutes = parts.pop();
811 + if (minutes) s += 60 * Number(minutes);
812 + const hours = parts.pop();
813 + if (hours) s += 60 * 60 * Number(hours);
814 + return s;
815 + };
816 + const compvals = (araw, braw) => {
817 + const a = araw.toString();
818 + const b = braw.toString();
819 + const atime = a.match(/^\d+(?::\d{2,})+$/);
820 + const btime = b.match(/^\d+(?::\d{2,})+$/);
821 + if (atime && btime) {
822 + return unmmss(btime[0]) - unmmss(atime[0]);
823 + }
824 + return dethe(a).localeCompare(dethe(b));
825 + }
826 +
827 + const dethe = (value) => value.toLowerCase().replace(/^the /, '');
828 +
829 + labeled ||= {};
830 + let outputlist = [];
831 + for (let opx=0; opx<operations.length; opx++) {
832 + const op = operations[opx];
833 + outputlist = [];
834 + switch (op.operator) {
835 + case '?': // start
836 + if (op.args.length === 0) {
837 + outputlist = Object.keys(this.data).sort((a, b) => dethe(a).localeCompare(dethe(b)));
838 + break;
839 + }
840 + outputlist = [];
841 + op.args.forEach((arg) => {
842 + let startitems;
843 + if (arg.label && arg.subop?.match(/\+/)) labeled[arg.label] ||= [];
844 + if (arg.separator !== ';' || outputlist.length === 0) {
845 + if (Array.isArray(arg.value)) {
846 + startitems = this.execute([], arg.value, labeled);
847 + if (arg.label) {
848 + if (arg.subop?.match(/\+/)) {
849 + startitems.forEach((i) => labeled[arg.label].push(i));
850 + } else {
851 + labeled[arg.label] = startitems;
852 + this.index[arg.label] = {};
853 + }
854 + labeled[arg.label].forEach((i) => outputlist.push(i));
855 + } else {
856 + startitems.forEach((i) => outputlist.push(i));
857 + }
858 + } else if (arg.subop === '~') {
859 + outputlist.push(arg.value);
860 + } else if (arg.value in labeled) {
861 + labeled[arg.value].forEach((i) => outputlist.push(i));
862 + } else if (startitems = this.gettype(arg.value, false)) {
863 + startitems.forEach((i) => outputlist.push(i));
864 + } else if (arg.value?.trim().length > 0 && this.dtype(arg.value, 'number')) {
865 + if (arg.subop === '-' && arg.value === '0') {
866 + outputlist.push(-0);
867 + } else {
868 + outputlist.push((arg.subop === '-' ? -1 : 1) * Number(arg.value));
869 + }
870 + } else if (arg.value && arg.subop !== '=') {
871 + outputlist.push(arg.value);
872 + }
873 + }
874 + });
875 + break;
876 + case '??': // label
877 + outputlist = currentlist;
878 + let ended = false;
879 + op.args.forEach((arg) => {
880 + if (!ended) {
881 + if (arg.label === '_timelimit' && !isNaN(arg.value)) {
882 + this.timelimit = Number(arg.value) * 60000;
883 + } else if (arg.label) {
884 + if (arg.subop.includes('~') && dtype(arg.value, 'string')) {
885 + labeled[arg.label] = [arg.value];
886 + } else if (arg.labeled) {
887 + labeled[arg.label] = arg.labeled;
888 + } else {
889 + if (arg.subop?.match(/\+/)) labeled[arg.label] ||= [];
890 + const newvals = this.execute(currentlist, Array.isArray(arg.value) ? arg.value : '.' + arg.value, labeled, level);
891 + if (arg.subop?.match(/\+/)) {
892 + newvals.forEach((nv) => labeled[arg.label].push(nv));
893 + } else {
894 + labeled[arg.label] = newvals;
895 + }
896 + if (!level && opx === 0) arg.labeled = newvals;
897 + }
898 + this.index[arg.label] = {};
899 + } else if (dtype(arg.value, 'string')) {
900 + if (arg.subop?.match(/\+/)) {
901 + labeled[arg.value] ??= [];
902 + currentlist.forEach((i) => labeled[arg.value].push(i));
903 + } else {
904 + labeled[arg.value] = currentlist.slice(0);
905 + }
906 + this.index[arg.value] = {};
907 + if (arg.value === 'end') {
908 + ended = true;
909 + }
910 + }
911 + }
912 + });
913 + if (ended) return (this.debug && !inputlist) ? operations : outputlist;
914 + break;
915 + case '!': // repeat
916 + if (opx > 0) {
917 + const repeat_ops = operations.slice(opx - 1, opx + 1);
918 + outputlist = currentlist.slice(0);
919 + level ??= 0;
920 + level += 1;
921 + const maxrecursion = ((op.args.length > 0 && op.args[0].value) || 1000);
922 + if (outputlist.length > 0 && level < maxrecursion && (!this.samearray(inputlist, outputlist) || level === 1)) {
923 + const recursed = this.execute(outputlist, repeat_ops, labeled, level);
924 + if (recursed.length > 0 && !this.samearray(recursed, outputlist)) outputlist = recursed.slice(0);
925 + }
926 + }
927 + break;
928 + case '.': // traverse
929 + case '..': // traverse with duplicates
930 + const seen = new Set();
931 + const sofar = Array.isArray(op.args?.[0]?.value) && op.args?.[0]?.label;
932 + let transq = null;
933 + if (sofar) labeled[sofar] = [];
934 + if (op.operator === '.' && op.args?.length === 1) {
935 + const firstarg = op.args[0];
936 + const firstval = firstarg?.value;
937 + if (dtype(firstval, 'string')) {
938 + transq = this.data.queries?.find((q) => q.relative === true && q.name === firstval);
939 + }
940 + }
941 + if (transq) {
942 + outputlist = this.execute(currentlist, this.parse(transq.query), labeled, level);
943 + } else {
944 + const traversetimer = {};
945 + outputlist = currentlist.reduce((acc, item, i) => {
946 + this.timecheck(traversetimer, i, currentlist.length, op);
947 + if (op.args.length === 0) {
948 + const itemkey = getid(item);
949 + if (op.operator === '..' || !seen.has(itemkey)) {
950 + acc.push(item);
951 + seen.add(itemkey);
952 + }
953 + return acc;
954 + }
955 + if (op.args[0].subop?.includes('<') && acc.length > 0) return acc;
956 + let itemvals = [];
957 + let toremoveids = {};
958 + for (const arg of op.args.filter((arg) => arg.value != null)) {
959 + let passdown = null;
960 + if (arg.subop?.includes('>') && typeof arg.value == 'string') {
961 + if (arg?.label in item) {
962 + passdown = item[arg.label];
963 + } else {
964 + passdown = JSON.parse(JSON.stringify(item, Object.keys(item).filter((k) => k !== arg.value)));
965 + }
966 + }
967 + if (arg.separator !== ';' || itemvals.length === 0) {
968 + const itemval = step(item, arg.value, arg.subop, labeled);
969 + if (arg.subop?.includes('-') && isNaN(arg.value)) {
970 + itemval.forEach((subitem) => {
971 + const subid = getid(subitem);
972 + toremoveids[subid] ||= 0;
973 + toremoveids[subid] += 1;
974 + });
975 + } else {
976 + itemval.forEach((subitem) => {
977 + if (passdown) {
978 + if (!dtype(subitem, 'object')) subitem = {value: subitem};
979 + if (arg.label) {
980 + subitem[arg.label] = [passdown];
981 + } else {
982 + Object.assign(subitem, passdown);
983 + }
984 + }
985 + const subitemkey = getid(subitem);
986 + if (op.operator === '..' || !seen.has(subitemkey)) {
987 + seen.add(subitemkey);
988 + itemvals.push(subitem);
989 + if (sofar) labeled[sofar].push(subitem);
990 + }
991 + })
992 + }
993 + }
994 + if (Object.keys(toremoveids).length > 0) {
995 + if (op.operator === '.') {
996 + itemvals = itemvals.filter((subitem) => !(getid(subitem) in toremoveids));
997 + } else {
998 + itemvals = itemvals.filter((subitem) => {
999 + const subid = getid(subitem);
1000 + if (toremoveids[subid] > 0) {
1001 + toremoveids[subid] -= 1;
1002 + return false;
1003 + } else {
1004 + return true;
1005 + }
1006 + })
1007 + }
1008 + }
1009 + }
1010 + itemvals.forEach((x) => acc.push(x));
1011 + return acc;
1012 + }, []);
1013 + }
1014 + outputlist = outputlist.filter((item) => item != null);
1015 + break;
1016 + case ':': // filter
1017 + if (!op?.args?.length > 0) {
1018 + outputlist = currentlist;
1019 + break;
1020 + }
1021 + for(const arg of op.args) {
1022 + if (arg.subop && ['>>', '>>=', '<<', '<<=', '~~'].includes(arg.subop.replace(/^-/, ''))) {
1023 + arg.tests = {id: new Set(), name: new Set()};
1024 + if (Array.isArray(arg.value)) {
1025 + const testitems = this.execute(currentlist, arg.value, labeled);
1026 + testitems.forEach((testitem) => arg.tests.id.add(getid(testitem)));
1027 + testitems.forEach((testitem) => arg.tests.name.add(getname(testitem)));
1028 + }
1029 +
1030 + if (['>>', '>>=', '<<', '<<='].includes(arg.subop)) {
1031 + for (const [tryval, tryf] of [['id', getid], ['name', getname]]) {
1032 + for (let j=0; j < currentlist.length; j++) {
1033 + if (arg.tests[tryval].has(tryf(currentlist[j]))) {
1034 + arg.matchpos = j;
1035 + break;
1036 + }
1037 + }
1038 + if (arg.matchpos) break
1039 + }
1040 + }
1041 + } else if ([null, '', '=', '~'].includes((arg.subop || '').replace(/-/, '')) && dtype(arg.value, 'string') && arg.value.startsWith('~')) {
1042 + const flags = arg.value.startsWith('~~') ? 'i' : '';
1043 + const pattern = arg.value.replace(/^~*/, '');
1044 + const fullpattern = (arg.subop || '').replace(/-/, '') === '=' ? ((pattern.startsWith('^') ? '' : '^') + pattern + (pattern.endsWith('$') ? '' : '$')) : pattern;
1045 + arg.re = new RegExp(fullpattern, flags);
1046 + }
1047 + }
1048 +
1049 + const ands = [[]];
1050 + for (const arg of op.args) {
1051 + if (arg.separator === ';') {
1052 + ands.push([arg]);
1053 + } else {
1054 + ands[ands.length - 1].push(arg)
1055 + }
1056 + }
1057 + const filtertimer = {};
1058 + outputlist = currentlist.filter((item, i) => {
1059 + this.timecheck(filtertimer, i, currentlist.length, op);
1060 + return ands.filter((and) => {
1061 + return and.filter((arg) => {
1062 + let comparator = arg.subop;
1063 + let polarize = (x) => x;
1064 + if (comparator?.startsWith('-') || comparator?.endsWith('-')) {
1065 + polarize = (x) => !x;
1066 + comparator = comparator.replace(/-|-$/, '');
1067 + }
1068 + if (!comparator && arg.label == null && Array.isArray(arg.value)) {
1069 + return polarize(this.execute([item], arg.value, labeled).length > 0);
1070 + } else if (comparator === '~~') {
1071 + if (arg.label == null && dtype(arg.value, 'number')) {
1072 + return polarize(currentlist.length === Number(arg.value));
1073 + } else {
1074 + if (arg.label) {
1075 + const argitems = step(item, arg.label, null, labeled);
1076 + return polarize(argitems.find((argitem) => arg.tests.id.has(getid(argitem)) || arg.tests.name.has(getname(argitem))));
1077 + } else {
1078 + return polarize(arg.tests.id.has(getid(item)) || (item?.id === undefined && arg.tests.name.has(getname(item))));
1079 + }
1080 + }
1081 + } else if (['~~', '<<', '<<=', '>>', '>>='].includes(comparator) && dtype(arg.value, 'number')) {
1082 + const argval = Number(arg.value);
1083 + let testlength;
1084 + if (typeof arg.label == 'string') {
1085 + testlength = step(item, arg.label, null, labeled).length;
1086 + } else {
1087 + testlength = currentlist.length;
1088 + }
1089 + switch (comparator) {
1090 + case '~~': return polarize(testlength === argval);
1091 + case '<<': return polarize(testlength < argval);
1092 + case '<<=': return polarize(testlength <= argval);
1093 + case '>>': return polarize(testlength > argval);
1094 + case '>>=': return polarize(testlength >= argval);
1095 + }
1096 + } else if (['+', ''].includes(comparator) && arg.label && !arg.value && dtype(arg.label, 'string') && dtype(item, 'object')) {
1097 + const propval = item[arg.label];
1098 + // console.log({plusminus: comparator, label: arg.label, propval: propval})
1099 + return polarize(Array.isArray(propval) ? propval.length > 0 : propval);
1100 + }
1101 +
1102 + let testitems = [item];
1103 + if (arg.label && dtype(item, 'object')) {
1104 + let found = false;
1105 + for (const tryval of [arg.label, arg.label + 's']) {
1106 + if (item?.[tryval] != null) {
1107 + const propitems = item[tryval];
1108 + if (propitems != null) {
1109 + found = true;
1110 + testitems = Array.isArray(propitems) ? propitems : [propitems];
1111 + break;
1112 + }
1113 + }
1114 + }
1115 + if (!found) {
1116 + if (dactal.savedquerynames.has(arg.label) || (labeled['=>'] && arg.label in labeled['=>'])) {
1117 + testitems = step(item, arg.label, null, labeled);
1118 + } else {
1119 + testitems = [];
1120 + }
1121 + }
1122 + }
1123 +
1124 + const testvals = testitems.map((testitem) => {
1125 + if (comparator || arg.re) {
1126 + return getname(testitem);
1127 + } else if (this.dtype(testitem, 'literal')) {
1128 + return testitem;
1129 + } else if ('id' in testitem) {
1130 + return testitem.id;
1131 + } else {
1132 + return getname(testitem)
1133 + }
1134 + });
1135 +
1136 + let argvals;
1137 + if (Array.isArray(arg.value)) {
1138 + const argvalitems = this.execute([item], arg.value, labeled);
1139 + argvals = argvalitems.map((argvalitem) => {
1140 + let argval;
1141 + if (dtype(argvalitem, 'literal')) {
1142 + argval = argvalitem;
1143 + } else {
1144 + argval = getname(argvalitem);
1145 + }
1146 + if (argval == null) {
1147 + const argvalitemkeys = Object.keys(argvalitem).filter((key) => key !== 'id');
1148 + if (argvalitemkeys.length === 1) {
1149 + argval = argvalitem[argvalitemkeys[0]];
1150 + }
1151 + }
1152 + return argval;
1153 + });
1154 + }
1155 +
1156 + return testvals.find((testval) => {
1157 + if (!argvals) {
1158 + if (dtype(testval, 'number') && dtype(arg.value, 'number')) {
1159 + testval = Number(testval);
1160 + argvals = [Number(arg.value)];
1161 + } else {
1162 + argvals = [arg.value];
1163 + }
1164 + }
1165 + if (comparator?.startsWith('@')) {
1166 + if (dtype(arg.value, 'number')) {
1167 + argvals = [Number(arg.value)];
1168 + } else if (arg.value in labeled) {
1169 + argvals = Number(labeled[arg.value]);
1170 + if (!Array.isArray(argvals)) argvals = [argvals];
1171 + }
1172 + if (comparator.startsWith('@@')) {
1173 + testval = currentlist.length - i;
1174 + comparator = comparator.slice(2);
1175 + } else {
1176 + testval = i + 1;
1177 + comparator = comparator.slice(1);
1178 + }
1179 + } else if (['<<', '<<=', '>>', '>>='].includes(comparator) && dtype(arg.value, 'number')) {
1180 + const argval = Number(arg.value);
1181 + switch (comparator) {
1182 + case '<<': return polarize(currentlist.length < argval);
1183 + case '<<=': return polarize(currentlist.length <= argval);
1184 + case '>>': return polarize(currentlist.length > argval);
1185 + case '>>=': return polarize(currentlist.length >= argval);
1186 + }
1187 + }
1188 + if (testval == null || argvals == null || argvals.length === 0) return false;
1189 + return argvals.find((argval) => {
1190 + if (arg.re) {
1191 + return polarize(testval.match(arg.re));
1192 + } else {
1193 + switch (comparator) {
1194 + case null: return polarize(testval === argval);
1195 + case '': return polarize(testval === argval);
1196 + case '=': return polarize(testval === argval);
1197 + case '>=': return polarize(testval >= argval);
1198 + case '<=': return polarize(testval <= argval);
1199 + case '>': return polarize(testval > argval);
1200 + case '<': return polarize(testval < argval);
1201 + case '~': return polarize(testval?.toString().toLowerCase().includes(argval?.toString().toLowerCase()));
1202 + case '~<': return polarize(testval?.toString().toLowerCase().startsWith(argval?.toString().toLowerCase()));
1203 + case '~>': return polarize(testval?.toString().toLowerCase().endsWith(argval?.toString().toLowerCase()));
1204 + case '>>': return polarize(i > arg.matchpos);
1205 + case '>>=': return polarize(i >= arg.matchpos);
1206 + case '<<': return polarize(i < arg.matchpos);
1207 + case '<<=': return polarize(i <= arg.matchpos);
1208 + }
1209 + }
1210 + }) != null;
1211 + }) != null;
1212 + }).length > 0;
1213 + }).length === ands.length;
1214 + });
1215 + break;
1216 + case '#': // sort
1217 + const sortargs = op.args.slice(0);
1218 + const lastarg = sortargs[sortargs.length - 1];
1219 + let temped = false;
1220 + if (!lastarg || ![';', '=;'].includes(Object.values(lastarg).join(''))) sortargs.push(...[{subop: null, value: 'name'}, {subop: null, value: 'id'}]);
1221 + if (!dtype(currentlist[0], 'object') && lastarg && lastarg.separator === ';') {
1222 + currentlist = currentlist.map((v) => ({_value: v}));
1223 + temped = true;
1224 + } else if (sortargs[0]?.label) {
1225 + currentlist = currentlist.map((i) => dcopy(i));
1226 + }
1227 + sortargs.forEach((arg) => {
1228 + arg.extraindex = {};
1229 + const stablesort = (arg.label == null && arg.value == null && arg.separator === ';' ) ? (arg.subop === '-' ? -1 : 1) : null;
1230 + if (arg.subop?.includes('~')) {
1231 + arg.sortmode = 'literal';
1232 + } else if (arg.subop?.endsWith('-')) {
1233 + arg.sortmode = 'numeric';
1234 + } else if (arg.subop?.endsWith('+')) {
1235 + arg.sortmode = 'rank';
1236 + } else {
1237 + arg.sortmode = stablesort || ['rank', 'index', 'number', 'id'].includes(arg.value) ? 'rank' : 'numeric';
1238 + }
1239 + const vals = new Set();
1240 + const extradone = new Set();
1241 + for (let ix=0; ix<currentlist.length; ix++) {
1242 + const item = currentlist[ix];
1243 + const vallist = stablesort ? [ix] : (arg.value ? step(item, arg.value, null, labeled) : (Array.isArray(item) ? item : [item]));
1244 + if (vallist && !stablesort) vallist.forEach((val) => vals.add(val));
1245 + if (typeof item == 'object') {
1246 + (item._sortindex ||= []).push(vallist);
1247 + } else if (!dtype(item, 'number') && !extradone.has(item)) {
1248 + arg.extraindex[item] = vallist;
1249 + extradone.add(item);
1250 + }
1251 + }
1252 + if (arg.sortmode !== 'literal') {
1253 + for (const val of vals) {
1254 + if (val != null && !dtype(val, 'number') && !(dtype(val, 'object') && dtype(getname(val), 'number'))) {
1255 + arg.sortmode = null;
1256 + break;
1257 + }
1258 + }
1259 + }
1260 + })
1261 +
1262 + outputlist = currentlist.sort((a, b) => {
1263 + let comp = 0;
1264 + let ii = 0;
1265 + for (const arg of sortargs) {
1266 + const alist = typeof a == 'object' ? a._sortindex[ii] : dtype(a, 'number') ? [Number(a)] : arg.extraindex[a];
1267 + const blist = typeof b == 'object' ? b._sortindex[ii] : dtype(b, 'number') ? [Number(b)] : arg.extraindex[b];
1268 + ii++;
1269 + if (alist && blist) {
1270 + if (arg?.subop?.includes('@')) {
1271 + const alookup = alist.indexOf(getname(a));
1272 + const blookup = blist.indexOf(getname(b));
1273 + if (alookup >-1 && blookup === -1) {
1274 + comp = -1;
1275 + } else if (alookup === -1 && blookup > -1) {
1276 + comp = 1;
1277 + } else {
1278 + comp = alist.indexOf(getname(a)) - alist.indexOf(getname(b))
1279 + }
1280 + } else {
1281 + for (let i=0; i < Math.min(alist.length, blist.length); i++) {
1282 + let aitem = alist[i];
1283 + let bitem = blist[i];
1284 + if (aitem != null && bitem == null) {
1285 + comp = -1;
1286 + } else if (aitem == null && bitem != null) {
1287 + comp = 1;
1288 + } else {
1289 + switch (arg.sortmode) {
1290 + case 'literal':
1291 + comp = aitem < bitem ? -1 : (bitem < aitem ? 1 : 0);
1292 + break;
1293 + case 'numeric':
1294 + let bnum = Number(bitem);
1295 + let anum = Number(aitem);
1296 + if (isNaN(bnum) || isNaN(anum)) {
1297 + bnum = Number(getname(bitem));
1298 + anum = Number(getname(aitem));
1299 + }
1300 + if (isNaN(bnum) || isNaN(anum)) {
1301 + comp = compvals(aitem, bitem)
1302 + } else {
1303 + comp = bnum - anum;
1304 + }
1305 + break;
1306 + case 'rank':
1307 + comp = Number(aitem) - Number(bitem);
1308 + break;
1309 + default:
1310 + if (dtype(aitem, 'literal') && dtype(bitem, 'literal')) {
1311 + comp = compvals(aitem, bitem);
1312 + } else if (typeof aitem == 'object' && typeof bitem == 'object') {
1313 + let aval = getname(aitem);
1314 + let bval = getname(bitem);
1315 + if (aval != null && bval != null) {
1316 + comp = compvals(aval, bval);
1317 + } else {
1318 + aval = getid(aitem);
1319 + bval = getid(bitem);
1320 + if (aval != null && bval != null) {
1321 + comp = compvals(aval, bval);
1322 + } else {
1323 + comp = 0;
1324 + }
1325 + }
1326 + } else if (aitem != null && bitem == null) {
1327 + comp = -1;
1328 + } else if (aitem == null && bitem != null) {
1329 + comp = 1;
1330 + }
1331 + break;
1332 + }
1333 + }
1334 + if (comp !== 0) break;
1335 + }
1336 + }
1337 + }
1338 + if (comp === 0) {
1339 + if (alist && blist) {
1340 + comp = blist.length - alist.length;
1341 + } else if (alist) {
1342 + comp = -1;
1343 + } else if (blist) {
1344 + comp = 1;
1345 + }
1346 + }
1347 + if (arg?.subop?.includes('-') && (arg.sortmode === 'literal' || !arg.sortmode)) {
1348 + comp = -comp;
1349 + }
1350 + if (comp !== 0) break;
1351 + }
1352 + return comp;
1353 + })
1354 + if (temped) outputlist = outputlist.map((vt) => vt._value);
1355 + outputlist.forEach((item, i) => {
1356 + delete item._sortindex;
1357 + if (op.args.length > 0 && op.args[0].label) item[op.args[0].label] = i + 1;
1358 + });
1359 + break;
1360 + case '/': // group
1361 + case '//': // merge
1362 + const groupindex = {};
1363 + let keylists = {};
1364 + let ofname = 'of';
1365 + let countname = 'count';
1366 + let sortgroups = true;
1367 + const groupargs = [];
1368 + const merge_ands = [];
1369 + for (const arg of op.args) {
1370 + if (arg.separator === ';' && arg.label == null && arg.value == null) {
1371 + sortgroups = false;
1372 + } else if (op.operator === '//' && (arg.separator === ';' || merge_ands.length > 0)) {
1373 + if (arg.separator === ';') {
1374 + merge_ands.push([arg.value]);
1375 + } else {
1376 + merge_ands[merge_ands.length - 1].push(arg.value)
1377 + }
1378 + } else {
1379 + groupargs.push(arg);
1380 + }
1381 + }
1382 + if (groupargs.length === 0) groupargs.push({value: null});
1383 +
1384 + groupargs.forEach((arg) => arg.groupcounter = 0);
1385 +
1386 + const grouptimer = {};
1387 + for (let ix=0; ix<currentlist.length; ix++) {
1388 + this.timecheck(grouptimer, ix, currentlist.length, op);
1389 + const item = currentlist[ix];
1390 + let keys = null;
1391 + let keyi = 0;
1392 + for (const arg of groupargs) {
1393 + if (arg.label === 'of') {
1394 + ofname = arg.value;
1395 + continue;
1396 + } else if (arg.label === 'count') {
1397 + countname = arg.value;
1398 + continue;
1399 + }
1400 + keyi++;
1401 + let groupnumber = null;
1402 + if (op.operator === '/' && dtype(arg.value, 'number')) {
1403 + if (arg.subop?.endsWith('@')) {
1404 + arg.divisor = Number(arg.value);
1405 + } else {
1406 + arg.divisor = currentlist.length / Number(arg.value);
1407 + }
1408 + groupnumber = Math.floor(ix / arg.divisor) + 1;
1409 + } else if (arg.value != null && arg.subop?.endsWith('@@')) {
1410 + const groupval = step(item, arg.value, null, labeled);
1411 + if (ix === 0 || groupval?.length > 0) arg.groupcounter += 1;
1412 + groupnumber = arg.groupcounter;
1413 + }
1414 + const label = arg.label ?? (typeof arg.value === 'string' ? arg.value : null) ?? keyi;
1415 + const newkeyitems = groupnumber != null ? [groupnumber] : arg.value ? step(item, arg.value, arg.subop, labeled) : [null];
1416 + const newkeys = newkeyitems.map((newkey) => ([{arglabel: arg.label, label: label, keyitem: this.resolve(arg.value, newkey, labeled)}]));
1417 + if (keys) {
1418 + keys = keys.flatMap((oldkeys) => newkeys.filter((newkey) => arg.separator === ',' || oldkeys.filter((oldkey) => compvals(getname(oldkey.keyitem) || oldkey.keyitem, getname(newkey[0].keyitem) || newkey[0].keyitem) >= 0).length === 0).map((newkey) => oldkeys.concat(newkey)));
1419 + } else {
1420 + keys = newkeys;
1421 + }
1422 + if (arg.subop?.endsWith('@')) {
1423 + keys.forEach((key) => {
1424 + const testkey = JSON.stringify(key.slice(0, -1));
1425 + const newkeytest = JSON.stringify(key[key.length - 1]);
1426 + if (testkey in keylists) {
1427 + const lastkey = keylists[testkey][keylists[testkey].length - 1];
1428 + if (newkeytest !== lastkey) keylists[testkey].push(newkeytest);
1429 + } else {
1430 + keylists[testkey] = [newkeytest];
1431 + }
1432 + key[key.length - 1].keyindex = keylists[testkey].length;
1433 + });
1434 + }
1435 + }
1436 + if (keys) {
1437 + for (const key of keys) {
1438 + const keystr = JSON.stringify(key);
1439 + (groupindex[keystr] ||= []).push(item);
1440 + }
1441 + }
1442 + }
1443 + for (const [keystr, items] of Object.entries(groupindex)) {
1444 + if (op.operator === '/') {
1445 + const newgroup = {};
1446 + const keydata = JSON.parse(keystr);
1447 + if (keydata.length === 1 && keydata[0].keyitem != null) {
1448 + let keyitemname = keydata[0].keyindex || getname(keydata[0].keyitem);
1449 + if (keyitemname != null) {
1450 + newgroup.name = keyitemname;
1451 + }
1452 + }
1453 + let skip = false;
1454 + let keys = [];
1455 + for (const {arglabel, label, keyitem, keyindex} of keydata) {
1456 + if (keyitem != null) {
1457 + const keyobj = (dtype(keyitem, 'object') || keyindex == null || false) ? keyitem : {name: keyitem};
1458 + if (keyindex != null) {
1459 + keyobj.index = keyindex;
1460 + }
1461 + keys.push(keyobj)
1462 + if (label && dtype(label, 'string') && isNaN(label) && label !== '_') {
1463 + newgroup[label] = [keyobj];
1464 + }
1465 + }
1466 + }
1467 + newgroup[countname] = items.length;
1468 + if (keys) newgroup.key = keys;
1469 + newgroup[ofname] = items;
1470 + outputlist.push(newgroup);
1471 + } else if (op.operator === '//') {
1472 + if (merge_ands.length === 0 || !merge_ands.find((mand) => !mand.find((mr) => items.find((item) => mr in item)))) {
1473 + const newgroup = items.reduce((acc, item) => {
1474 + for(const prop in item) {
1475 + if (prop in acc) {
1476 + if (Array.isArray(acc[prop])) {
1477 + (Array.isArray(item[prop]) ? item[prop] : [item[prop]]).filter((val) => !acc[prop].includes(val)).forEach((val) => acc[prop].push(val));
1478 + }
1479 + } else {
1480 + acc[prop] = item[prop];
1481 + }
1482 + }
1483 + return acc;
1484 + }, {});
1485 + const keydata = JSON.parse(keystr);
1486 + for (const {arglabel, label, keyitem, keyindex} of keydata) {
1487 + if (arglabel && typeof arglabel === 'string' && arglabel !== '_') {
1488 + newgroup[arglabel] = [keyitem];
1489 + }
1490 + }
1491 + outputlist.push(newgroup);
1492 + }
1493 + }
1494 + }
1495 + if (sortgroups) {
1496 + const sortquery = '#' + groupargs.map((arg, argx) => (arg.subop?.endsWith('@') || arg.divisor ? '+' : '') + '(..key:@' + (argx + 1) + (arg.subop?.endsWith('@') ? '.index;_' : '') + ')').join(',');
1497 + outputlist = this.execute(outputlist, this.assemble(this.tokenize(sortquery)), labeled);
1498 + }
1499 + break;
1500 + case '...': // aggregate
1501 + case '....': // aggregate to value
1502 + if (!(op?.args?.length > 0)) {
1503 + if (op.operator === '...') {
1504 + outputlist = [{of: currentlist.map((item) => dcopy(item))}];
1505 + } else {
1506 + outputlist = [currentlist.length]
1507 + }
1508 + break;
1509 + }
1510 + const tempitem = {};
1511 + if (op.operator === '...') {
1512 + tempitem.of = currentlist.map((item) => dcopy(item));
1513 + } else {
1514 + tempitem.of = currentlist;
1515 + }
1516 + const finalitem = {};
1517 + let aggregated = null;
1518 + let afteraggregated = 0;
1519 + let finalvalue = null;
1520 + op.args.forEach((arg, propx) => {
1521 + if (dtype(arg.value, 'string') && arg.subop === '~') {
1522 + tempitem[arg.label || '_' + (propx + 1).toString()] = [arg.value];
1523 + } else {
1524 + const prop = arg.label ?? (typeof arg.value === 'string' ? arg.value : null) ?? '_' + (propx + 1).toString();
1525 + const aggname = (arg.label == null && typeof arg.value == 'string' && arg.value) || (arg.value == null && arg.label);
1526 + const firstitem = currentlist[0];
1527 + const isprop = typeof firstitem == 'object' && prop in firstitem;
1528 + if ((arg.separator === ';' || !isprop) && aggname in this.aggregators) {
1529 + const aggval = this.aggregators[aggname](tempitem);
1530 + const agglabel = arg.label || aggname;
1531 + if (aggval != null) finalitem[agglabel] = aggval;
1532 + aggregated = agglabel;
1533 + finalvalue = dcopy(aggval);
1534 + } else if (!aggregated) {
1535 + let propitems;
1536 + if (op === operations[0] && op.operator === '....' && arg.label && !Array.isArray(arg.value) && !arg.value.startsWith('=')) {
1537 + propitems = [arg.value]
1538 + } else {
1539 + propitems = this.execute(currentlist, Array.isArray(arg.value) ? arg.value.map((x) => structuredClone(x)) : [{operator: '..', args: [{value: arg.value}]}], labeled)
1540 + }
1541 + tempitem[prop] = propitems;
1542 + } else if (aggregated in finalitem && Array.isArray(finalitem[aggregated]) && finalitem[aggregated].length > afteraggregated) {
1543 + tempitem[prop] = finalitem[aggregated][afteraggregated];
1544 + afteraggregated++;
1545 + }
1546 + }
1547 + });
1548 + if (op.operator === '....' && finalvalue != null) {
1549 + if (this.dtype(finalvalue, 'array')) {
1550 + outputlist = finalvalue;
1551 + } else {
1552 + outputlist = [finalvalue];
1553 + }
1554 + } else {
1555 + for (const prop in tempitem) {
1556 + if (!(prop in finalitem) && prop !== 'of') finalitem[prop] = tempitem[prop];
1557 + }
1558 + if (op.operator === '...') {
1559 + finalitem.of = tempitem.of;
1560 + } else {
1561 + for (const prop in finalitem) {
1562 + if (Array.isArray(finalitem[prop]) && finalitem[prop].length === 1) finalitem[prop] = finalitem[prop][0];
1563 + }
1564 + }
1565 + outputlist = [finalitem];
1566 + }
1567 + break;
1568 + case '|': // annotate
1569 + case '||': // annotate with values
1570 + outputlist = currentlist.slice(0).map((item) => dcopy(item));
1571 + op.args.filter((arg) => arg.subop?.includes('>') && arg.label != null && true && Array.isArray(arg.value))
1572 + .forEach((arg) => (labeled['=>'] ??= {})[arg.label] = arg.value);
1573 +
1574 + const annotatetimer = {};
1575 + for (const i in outputlist) {
1576 + this.timecheck(annotatetimer, i, outputlist.length, op);
1577 + const baseitem = outputlist[i];
1578 + if (typeof baseitem != 'object') {
1579 + outputlist[i] = {};
1580 + if (op.args?.[0]?.value !== '_') outputlist[i].name = baseitem;
1581 + }
1582 + const item = outputlist[i];
1583 + const newprops = op.args.filter((arg) => arg.subop !== '<' && (arg.label || !arg.subop?.includes('-'))).map((arg) => arg.label || arg.value);
1584 + if (typeof item == 'object') {
1585 + let argx = -1;
1586 + for (const arg of op.args) {
1587 + argx++;
1588 + if (arg?.subop === '<' && dtype(arg.value, 'string') && Array.isArray(item[arg.value]) && argx < op.args.length - 1) {
1589 + const sublabeled = {...labeled};
1590 + if (arg.label) sublabeled[arg.label] = [item];
1591 + item[arg.value] = dactal.execute(item[arg.value], [{operator: op.operator, args: op.args.slice(argx + 1).map((arg) => structuredClone(arg))}], sublabeled);
1592 + break;
1593 + } else if (arg.separator === ';' && argx === op.args.length - 1 && arg.label == null && arg.value == null && arg.subop == null) {
1594 + for (const oldprop in item) {
1595 + if (!(newprops.includes(oldprop))) {
1596 + const tempval = item[oldprop];
1597 + delete item[oldprop];
1598 + item[oldprop] = tempval;
1599 + }
1600 + }
1601 + } else if (arg.subop?.endsWith('-')) {
1602 + if (arg.value && typeof arg.value == 'string' && arg.value in item) {
1603 + if (arg.label && typeof arg.label == 'string') item[arg.label] = item[arg.value];
1604 + delete item[arg.value];
1605 + } else if (arg.label && Array.isArray(arg.value)) {
1606 + const toremoveids = new Set(step(item, arg.value, null, labeled).map((subitem) => getid(subitem)));
1607 + item[arg.label] = item[arg.label].filter((subitem) => !toremoveids.has(getid(subitem)));
1608 + }
1609 + } else if (argx === 0 && arg.label != null && arg.value === '_') {
1610 + item[arg.label] = op.operator === '||' ? baseitem : [baseitem];
1611 + } else {
1612 + if (!arg.label && arg.value && typeof arg.value === 'string' && arg.value !== '') {
1613 + const moveprop = arg.value;
1614 + const value = structuredClone(item[moveprop]);
1615 + delete item[moveprop];
1616 + item[moveprop] = value;
1617 + }
1618 + if (arg.label) {
1619 + let vals = [];
1620 + if (arg.subop.endsWith('@')) {
1621 + if (arg.subop.endsWith('@@')) {
1622 + arg.counter ??= arg.value == null ? outputlist.length + 1 : 0;
1623 + } else {
1624 + arg.counter ??= 1;
1625 + vals = [arg.counter];
1626 + }
1627 + if (Array.isArray(arg.value)) {
1628 + arg.counter += step(item, arg.value, null, labeled).length;
1629 + } else if (dtype(arg.value, 'string') && dtype(item[arg.value], 'number')) {
1630 + arg.counter += Number(item[arg.value]);
1631 + } else {
1632 + arg.counter += arg.subop.endsWith('@@') ? -1 : 1;
1633 + }
1634 + if (arg.subop.endsWith('@@')) {
1635 + vals = [arg.counter];
1636 + }
1637 + } else {
1638 + vals = step(item, arg.value, arg.subop, labeled);
1639 + }
1640 + const base = (arg.subop?.endsWith('+') && arg.label in item && Array.isArray(item[arg.label])) ? item[arg.label] : [];
1641 + if (this.dtype(vals, 'array')) {
1642 + if (op.operator === '|') {
1643 + item[arg.label] = base.concat(vals.map((v) => dcopy(v)));
1644 + } else if (vals.length > 0 ) {
1645 + if (typeof vals[0] === 'object') {
1646 + item[arg.label] = getname(vals[0]) != null ? getname(vals[0]) : vals[0].id != null ? vals[0].id : dcopy(vals[0]);
1647 + } else {
1648 + item[arg.label] = vals[0];
1649 + }
1650 + }
1651 + } else if (vals) {
1652 + item[arg.label] = base.concat(vals);
1653 + }
1654 + }
1655 + }
1656 + }
1657 + }
1658 + }
1659 + break;
1660 + case '???':
1661 + outputlist = currentlist;
1662 + const commentval = op.args?.[0]?.value;
1663 + if (commentval in labeled) console.log({[commentval]: labeled[commentval]});
1664 + if (commentval === 'end') return (this.debug && !inputlist) ? operations : outputlist;
1665 + break;
1666 + default:
1667 + outputlist = []
1668 + }
1669 + currentlist = outputlist.slice(0);
1670 + if (this.debug && !inputlist) op.results = outputlist.slice(0, typeof this.debug == 'number' ? this.debug : outputlist.length);
1671 + }
1672 + return (this.debug && !inputlist) ? operations : outputlist;
1673 + }
1674 +
1675 + gettype(value, allow_relative=true) {
1676 + if (value == null) return null;
1677 + const trylist = [value];
1678 + if (value.endsWith('s')) {
1679 + trylist.push(value.slice(0, value.length - 1));
1680 + trylist.push(value + 'es');
1681 + } else {
1682 + trylist.push(value + 's');
1683 + }
1684 + for (const tryval of trylist) {
1685 + if (tryval in this.data) {
1686 + return this.data[tryval];
1687 + }
1688 + }
1689 + if (this.savedquerynames.has(value)) {
1690 + const savedqueries = this.data.queries.filter((q) => q.name === value);
1691 + if (savedqueries?.length === 1) {
1692 + const sq = savedqueries[0];
1693 + if (allow_relative && sq.query.startsWith('?')) {
1694 + return sq;
1695 + } else {
1696 + if (!sq.results) sq.results = this.executeq(sq.query);
1697 + return sq.results.slice(0);
1698 + }
1699 + }
1700 + }
1701 + return null;
1702 + }
1703 +
1704 + dcopy = (item) => {
1705 + if (this.dtype(item, 'literal')) return item;
1706 + if (this.dtype(item, 'array')) return item.slice(0);
1707 + return Object.assign({}, item);
1708 + }
1709 +
1710 + dtype(item, test=null) {
1711 + let type = null;
1712 + if (item && Array.isArray(item)) {
1713 + type = 'array';
1714 + } else if (item && typeof item === 'object') {
1715 + type = 'object';
1716 + } else if (typeof item === 'string' || typeof item === 'number' || typeof item === 'boolean') {
1717 + type = 'literal';
1718 + }
1719 + if (test) {
1720 + if (test === 'number') {
1721 + return type === 'literal' && !isNaN(item);
1722 + } else if (test === 'string') {
1723 + return type === 'literal' && typeof item === 'string';
1724 + } else if (test === 'boolean') {
1725 + return type === 'literal' && typeof item === 'boolean';
1726 + }
1727 + return type === test;
1728 + }
1729 + return type;
1730 + }
1731 +
1732 + getname(item) {
1733 + if (typeof item === 'object' && item != null) {
1734 + if (item?.name != null) {
1735 + return item.name;
1736 + } else if (item?.key?.length > 0) {
1737 + return item.key.join(' / ');
1738 + }
1739 + const {id, ...nonidprops} = item;
1740 + const names = Object.values(nonidprops).filter((val) => typeof val === 'string' || typeof val === 'number');
1741 + if (names.length > 0) {
1742 + return names[0];
1743 + }
1744 + for(const key in nonidprops) {
1745 + if (Array.isArray(item[key]) && item[key].length === 1 && typeof item[key][0] == 'string') {
1746 + return item[key][0];
1747 + }
1748 + }
1749 + } else if (typeof item === 'string' || typeof item === 'number') {
1750 + return item;
1751 + } else if (typeof item === 'boolean') {
1752 + return item.toString();
1753 + }
1754 + return '';
1755 + }
1756 +
1757 + getid(item) {
1758 + if (typeof item === 'object' && item != null) {
1759 + const iditems = (item.id || item.uri || item.name) ? [item] : item.key?.length > 0 ? item.key : Object.keys(item).length === 3 && item.of?.length > 0 ? item.of : [item];
1760 + return iditems.map((item) => {
1761 + if (item.id) {
1762 + if (Array.isArray(item.id)) {
1763 + return item.id[0];
1764 + } else {
1765 + return item.id;
1766 + }
1767 + } else if (item.uri) {
1768 + if (Array.isArray(item.uri)) {
1769 + return item.uri[0];
1770 + } else {
1771 + return item.uri;
1772 + }
1773 + } else if (item.name) {
1774 + return item.name;
1775 + } else {
1776 + // if (stringifiedid.length > 128) console.warn({idstringify: item, idlength: stringifiedid.length});
1777 + return JSON.stringify(item);
1778 + }
1779 + }).join(',');
1780 + } else if (typeof item === 'string' || typeof item === 'number') {
1781 + return item;
1782 + }
1783 + return null;
1784 + }
1785 +
1786 + step = (item, property, subop, labeled) => {
1787 + // (this.data.trace ??= []).push({stepitem: item, property: property, subop: subop, labeled: labeled});
1788 + const dtype = this.dtype;
1789 + const dcopy = this.dcopy;
1790 + const resolve = this.resolve;
1791 + const escapeRegExp = this.escapeRegExp;
1792 + const vals = [];
1793 + if (subop?.includes('~') && dtype(property, 'literal')) {
1794 + vals.push(property);
1795 + } else if (Array.isArray(property)) {
1796 + this.execute([item], property.map((x) => structuredClone(x)), labeled).forEach((val) => vals.push(val));
1797 + } else if (property === '_') {
1798 + vals.push(dcopy(item));
1799 + } else if (item?.toString()?.match(/^-?\d+$/) && property.match(/^\d+$/)) {
1800 + vals.push((item === 0 && 1 / item === -Infinity ? -1 : 1) * Number(`${item}.${property}`));
1801 + } else if (item?.of && dtype(property, 'number')) {
1802 + const propval = Number(property);
1803 + (subop === '-' || propval < 0 ? item.of.slice(-1 * Math.abs(Number(property))) : item.of.slice(0, Math.abs(Number(property)))).forEach((val) => vals.push(val));
1804 + } else if (property === 'id' && dtype(item, 'literal')) {
1805 + vals.push(item);
1806 + } else if (property === 'name') {
1807 + vals.push(this.getname(item));
1808 + } else if ((dtype(item, 'number') || dtype(item, 'object')) && property.startsWith('=')) {
1809 + let calculation = property.slice(1);
1810 + const mathwords = Object.getOwnPropertyNames(Math).filter((mathword) => mathword.match(/^[a-z0-9]+$/)).sort((a, b) => b.length - a.length || a.localeCompare(b));
1811 + const otherwords = ['split'];
1812 + const variables = Object.entries(item).concat(Object.entries(labeled))
1813 + .map(([k, v]) => k)
1814 + .sort((a, b) => b.length - a.length || a.localeCompare(b));
1815 + if (dtype(this.getname(item), 'number')) variables.push('_');
1816 + const allowedwords = mathwords.concat(otherwords).concat(variables).sort((a, b) => b.length - a.length || a.localeCompare(b));
1817 + const allowed = new RegExp(`^((\b(${allowedwords.map((w) => escapeRegExp(w)).join('|')})\b)|([0-9_\+\/\*\(\)\[\].%=,'" -]*))*$`);
1818 + let val;
1819 + if (calculation.match(allowed)) {
1820 + for (const variable of variables) {
1821 + const variableex = new RegExp(`\\b${variable}\\b`, 'g');
1822 + if (calculation.match(variableex)) {
1823 + let vval = variable === '_' ? Number(this.getname(item)) : item[variable] ?? labeled[variable];
1824 + if (Array.isArray(vval) && vval.length === 1) vval = vval[0];
1825 + calculation = calculation.replaceAll(variableex, dtype(vval, 'number') ? vval : JSON.stringify(vval));
1826 + }
1827 + }
1828 + if (calculation.match(/[A-Za-z]/)) {
1829 + for (const mathword of mathwords) {
1830 + const mathwordex = new RegExp(`\\b${mathword}\\b`, 'g');
1831 + calculation = calculation.replaceAll(mathwordex, `Math.${mathword}`);
1832 + if (!calculation.match(/[A-Za-z]/)) break;
1833 + }
1834 + }
1835 + calculation = calculation.replaceAll(/\b=\b/g, '==');
1836 + try {
1837 + val = eval(calculation);
1838 + } catch (error) {
1839 + val = calculation;
1840 + }
1841 + } else {
1842 + val = calculation;
1843 + }
1844 + if (val !== false) vals.push(val);
1845 + } else if (item) {
1846 + let found = false;
1847 + if (property != null && !dtype(item, 'literal')) {
1848 + const tryvals = [property];
1849 + tryvals.push(property.endsWith('s') ? property.slice(0, -1) : property + 's');
1850 + if (property.match(/ /)) tryvals.push(property.replaceAll(/ /g, '_'));
1851 + for (const tryval of tryvals) {
1852 + if (tryval in item) {
1853 + const resolved = resolve(tryval, item[tryval], labeled);
1854 + if (Array.isArray(resolved)) {
1855 + resolved.forEach((x) => vals.push(x));
1856 + } else {
1857 + vals.push(resolved);
1858 + }
1859 + found = true;
1860 + break;
1861 + }
1862 + }
1863 + }
1864 + if (!found && labeled?.['=>']?.[property]) {
1865 + this.execute([item], labeled['=>'][property], labeled).forEach((val) => vals.push(val));
1866 + found = true;
1867 + }
1868 + if (!found && (property in this.data || property in this.adapters || this.savedquerynames.has(property) || property in labeled)) {
1869 + let typenav;
1870 + if (!dtype(item, 'literal')) {
1871 + if (this.adapters?.[property]?.annotator) {
1872 + typenav = resolve(property, item, labeled);
1873 + // } else if (property in this.aggregators) {
1874 + // typenav = this.aggregators[property](item);
1875 + } else if ('id' in item || 'uri' in item) {
1876 + typenav = resolve(property, this.getid(item), labeled);
1877 + } else if (this.savedquerynames.has(property)) {
1878 + const relative_query = dactal.data.queries.find((q) => q.relative && q.name === property);
1879 + if (relative_query) {
1880 + typenav = this.step(item, this.parse(relative_query.query), subop, labeled);
1881 + } else {
1882 + typenav = this.getname(item);
1883 + }
1884 + } else {
1885 + typenav = this.getname(item);
1886 + }
1887 + if (typenav) vals.push(typenav);
1888 + } else {
1889 + typenav = resolve(property, item, labeled);
1890 + if (typenav) {
1891 + if (Array.isArray(typenav)) {
1892 + typenav.forEach((t) => vals.push(t));
1893 + } else {
1894 + vals.push(typenav);
1895 + }
1896 + }
1897 + }
1898 + }
1899 + }
1900 + return vals;
1901 + }
1902 +
1903 + resolve = (property, item, labeled) => {
1904 + const dtype = this.dtype;
1905 + if (dtype(item, 'object')) {
1906 + if (this.adapters?.[property]?.annotator) {
1907 + if ((!this.recache || this.recache.has(property)) && property in this.index && this.getid(item) in this.index[property]) {
1908 + return this.index[property][this.getid(item)];
1909 + } else {
1910 + this.adapters[property].queue.push(this.adapters[property].annotator ? item : this.getid(item));
1911 + return null;
1912 + }
1913 + } else {
1914 + return item;
1915 + }
1916 + } else if (dtype(item, 'literal') && typeof property == 'string' && (property in labeled || this.destinations.has(property) || this.destinations.has(property + 's'))) {
1917 + if ((!this.recache || this.recache.has(property)) && property in this.index && item in this.index[property]) {
1918 + // this.cachehitcount++;
1919 + return this.index[property][item];
1920 + }
1921 + const typeitems = labeled[property] ?? this.gettype(property);
1922 + if (Array.isArray(typeitems)) {
1923 + for (const lookupkey of ['id', 'uri', 'name', property, property + 's']) {
1924 + const found = typeitems.filter((typeitem) => {
1925 + return dtype(typeitem, 'object') && (lookupkey in typeitem) && (typeitem[lookupkey] === item || (Array.isArray(typeitem[lookupkey]) && typeitem[lookupkey].length === 1 && typeitem[lookupkey][0] === item));
1926 + });
1927 + if (found.length === 1) {
1928 + (this.index[property] ||= {})[item] = found[0];
1929 + return found[0];
1930 + }
1931 + }
1932 + if (property in this.adapters && (!this.recache || !this.recache.has(property))) {
1933 + this.adapters[property].queue.push(this.adapters[property].annotator ? item : this.getid(item));
1934 + }
1935 + return null;
1936 + } else if (typeitems && typeof typeitems == 'object' && 'query' in typeitems) {
1937 + const relative_query = this.assemble(this.tokenize(typeitems.query));
1938 + while (['?', ':'].includes(relative_query[0].operator)) {
1939 + relative_query.shift()
1940 + }
1941 + return this.execute([item], relative_query, labeled);
1942 + }
1943 + if (property in this.adapters) {
1944 + this.adapters[property].queue.push(this.adapters[property].annotator ? item : this.getid(item));
1945 + return null;
1946 + }
1947 + }
1948 + return item;
1949 + }
1950 +
1951 + samearray(a, b) {
1952 + if ((a && !b) || (!a && b) || a.length !== b.length) return false;
1953 + for(let x=0; x<a.length; x++) {
1954 + if (this.getid(a[x]) !== this.getid(b[x])) return false;
1955 + }
1956 + return true;
1957 + }
1958 +
1959 + async verify(i = null) {
1960 + let toverify = this.data.queries;
1961 + if (!isNaN(i)) toverify = toverify.slice(i - 1, i);
1962 + let allsame = true;
1963 + for (const savedq of toverify) {
1964 + console.log('verifying ' + savedq.name);
1965 + const testres = await this.query(savedq.query);
1966 + if (testres.length !== savedq.results.length) {
1967 + console.log('--x result count changed from ' + savedq.results.length + ' to ' + testres.length);
1968 + allsame = false;
1969 + } else {
1970 + for (let i=0; i<testres.length; i++) {
1971 + const testrow = testres[i];
1972 + const savedrow = savedq.results[i];
1973 + if (typeof savedrow == 'object') {
1974 + for (const prop in savedrow) {
1975 + if (!(prop in testrow)) {
1976 + console.log('--x row ' + (i+1) + ': new results missing property ' + prop);
1977 + allsame = false;
1978 + } else {
1979 + const testval = JSON.stringify(testrow[prop]);
1980 + const savedval = JSON.stringify(savedrow[prop]);
1981 + if (testval !== savedval) {
1982 + console.log('--x row ' + (i+1) + ': different value for property ' + prop);
1983 + console.log({was: savedrow[prop], now: testrow[prop]})
1984 + allsame = false;
1985 + }
1986 + }
1987 + }
1988 + } else {
1989 + if (testrow !== savedrow) {
1990 + console.log('--x row ' + (i+1) + ': different value');
1991 + console.log({was: savedrow, now: testrow});
1992 + allsame = false;
1993 + }
1994 + }
1995 + }
1996 + }
1997 + if (allsame) console.log('--- results unchanged');
1998 + }
1999 + }
2000 +
2001 + index_check() {
2002 + console.table(Object.entries(this.index).map(([key, vals]) => ({key: key, vals: Object.keys(vals).length, size: Object.keys(vals).length * JSON.stringify(Object.entries(vals).slice(0, 1)).length})).sort((a, b) => b.size - a.size || b.vals - a.vals || a.key.localeCompare(b.key)))
2003 + }
2004 +
2005 + index_materialize() {
2006 + this.load(Object.keys(this.index).flatMap((i) => Object.keys(this.index[i]).flatMap((k) => ({index: i, indexed: k, value: this.index[i][k]}))), 'index contents')
2007 + }
2008 +}