update dactal

author
Marijn Besseling <njirambem@gmail.com> · 2026-07-07 15:32 UTC
commit
10de9bcb21904430ecabbbb68366e9b68370e3f5
parent
093ed2adcb
tree
browse at this commit

1 file changed +1201 -669

Blog/wwwroot/dactal.js +1201 -669

Blog/wwwroot/dactal.js +1201 -669

@@ -1,61 +1,111 @@
1 1 export class DACTAL {
2 - constructor(data={}) {
2 + constructor(data = {}) {
3 3 this.data = data;
4 4 this.index = {};
5 - this.index_modified = false;
5 + this.index_modified = new Set;
6 6 this.adapters = {};
7 + this.adaptive = false;
8 + this.features = { // earnest magic you can disable
9 + autoresolve: true, // if dataset X exists, following prop X to a literal is treated as an ID lookup
10 + plurality: true, // prop and props may be used interchangeably
11 + inlinemath: true, // traverse literals starting with = do inline math, like .[=score/total]
12 + unscore: true, // props_like_this can also be referred to as props like this
13 + guessid: true, // items without ids may have them inferred from their names
14 + guessname: true // items without names may have them inferred from other properties
15 + }
7 16 this.data['query history'] = [];
8 17 this.savedquerynames = new Set();
9 18 this.debug = false;
10 19 this.recache = false;
11 20 this.timelimit = 120000;
12 - this.statusf = (statusmsg) => console.log(statusmsg);
13 - this.aggregators = {
21 + this.statusf = (statusmsg) => {
22 + if (statusmsg) console.log(statusmsg)
23 + };
24 + this.internal_datasets = ['queries', 'query history', 'connectors', 'adapters', 'annotators', 'current results', 'data updates', 'assistance', '.clf API routes'];
25 +
26 + this.annotators = {
14 27 group: (item) => item.of,
28 + label: (item) => item.of,
29 + as: (item) => item.of,
15 30 count: (item) => this.vals(item).length,
16 - total: (item) => this.numvals(item).reduce((acc, val) => acc + (this.dtype(val, 'number') ? val : 0), 0),
17 - average: (item) => this.numvals(item).reduce((acc, val) => acc + (this.dtype(val, 'number') ? val : 0), 0) / this.numvals(item).length,
31 + total: (item) => this.numvals(item).reduce((acc, val) => acc + val, 0),
32 + average: (item) => this.numvals(item).reduce((acc, val) => acc + val, 0) / this.numvals(item).length,
33 + median: (item) => {
34 + const nums = this.numvals(item).sort();
35 + return nums[Math.floor(nums.length / 2)];
36 + },
18 37 min: (item) => {
19 38 const nums = this.numvals(item);
20 - return nums.length === 0 ? [] : Math.min(...nums)
39 + return nums.length == 0 ? [] : Math.min(...nums)
21 40 },
22 41 max: (item) => {
23 42 const nums = this.numvals(item);
24 - return nums.length === 0 ? [] : Math.max(...nums)
43 + return nums.length == 0 ? [] : Math.max(...nums)
25 44 },
26 - product: (item) => this.numvals(item).reduce((acc, val) => acc * (this.dtype(val, 'number') ? val : 1), 1),
27 - difference: (item) => this.numvals(item).reduce((acc, val) => acc - (this.dtype(val, 'number') ? val : 0)),
28 - quotient: (item) => this.numvals(item).reduce((acc, val) => acc / (this.dtype(val, 'number') ? val : 1)),
29 - percent: (item) => Math.round(this.numvals(item).reduce((acc, val) => acc / (this.dtype(val, 'number') ? val : 1)) * 100),
45 + product: (item) => this.numvals(item).reduce((acc, val) => acc * val, 1),
46 + difference: (item) => this.numvals(item).reduce((acc, val) => acc - val),
47 + quotient: (item) => this.numvals(item).reduce((acc, val) => acc / val),
48 + percent: (item) => Math.round(this.numvals(item).reduce((acc, val) => acc / val) * 100),
30 49 sqrt: (item) => Math.sqrt(this.numvals(item)[0]),
31 50 log: (item) => Math.log(this.numvals(item)[0]),
32 51 log10: (item) => Math.log10(this.numvals(item)[0]),
33 52 abs: (item) => Math.abs(this.numvals(item)[0]),
34 53 is: (item) => (item.of?.length > 0) ? 1 : 0,
35 54 isnt: (item) => (item.of?.length === 0) ? 1 : 0,
36 - yesno: (item) => (item.of?.length > 0) ? 'yes' : 'no',
37 - missing: (item) => (item.of?.length === 0) ? true : null,
55 + yesno: (item) => (item.of?.length > 0) ? 'yes' : 'no',
56 + missing: (item) => (item.of?.length === 0) ? true : [],
57 + otherwise: (item) => {
58 + const ikeys = this.kkeys(item);
59 + if (item.of?.length > 0 || ikeys.length == 0) {
60 + return item.of;
61 + } else {
62 + return item[ikeys[0]];
63 + }
64 + },
65 + sortsame: (item) => {
66 + const sortforms = this.vals(item).map((v) => v?.toString()?.toLowerCase()?.replace(/^the /, ''));
67 + const sortformset = new Set(sortforms);
68 + return (sortformset.size == 1 ? 'sortsame' : []);
69 + },
70 + startsame: (item) => {
71 + const vals = this.vals(item);
72 + const shortest = vals.sort((a, b) => a.length - b.length)[0];
73 + return (vals.filter((v) => v.startsWith(shortest)).length == vals.length ? shortest : []);
74 + },
38 75 concatenate: (item) => this.vals(item).join(' '),
39 76 join: (item) => item.of.map(this.getname).join(this.vals(item)[0]),
40 77 str: (item) => item.of.map(this.getname).join(''),
41 78 'to json': (item) => JSON.stringify(item.of),
42 - quote: (item) => `“${this.vals(item)[0]}”`,
79 + quote: (item) => `“${this.vals(item)[0]}”`,
43 80 url: (item) => {
44 81 let u = item.of.map(this.getname).join('');
45 82 if (!u.startsWith('https://')) u = 'https://' + u;
46 - for (const prop of Object.keys(item).filter((key) => key !== 'of')) {
83 + for (const prop of this.kkeys(item)) {
47 84 const val = item[prop];
48 - (Array.isArray(val) ? val : [val]).forEach((vv) => u = u + (u.match(/\?/) ? '&' : '?') + encodeURIComponent(prop) + '=' + encodeURIComponent(vv));
85 + (Array.isArray(val) ? val : [val]).forEach((vv) => u = u + (u.match(/\?/) ? '&' : '?') + encodeURIComponent(prop) + '=' + encodeURIComponent(vv));
49 86 }
50 87 return u;
51 88 },
52 89 remove: (item) => this.vals(item).reduce((acc, val) => acc.replaceAll(val, '')),
53 - bmk: (item) => this.vals(item).map((v) => v.toString().toLowerCase().replaceAll(/[$\xA2-\xA5\u058F\u060B\u09F2\u09F3\u09FB\u0AF1\u0BF9\u0E3F\u17DB\u20A0-\u20BD\uA838\uFDFC\uFE69\uFF04\uFFE0\uFFE1\uFFE5\uFFE6,+]/g, '').replaceAll('b', 'kkk').replaceAll('m', 'kk').replaceAll('k', '000')),
54 - sortform: (item) => this.vals(item).map((val) => val.toString().toLowerCase().replace(/^the /, '')),
90 + replace: (item) => {
91 + return item.of.map((i) => {
92 + let newi = this.getname(i);
93 + this.kkeys(item).map((k) => {
94 + newi = newi.replaceAll(k, item[k].toString());
95 + })
96 + return newi;
97 + });
98 + },
99 + matches: (item) => {
100 + const matchers = item.match.map((m) => m.toLowerCase());
101 + return matchers.filter((m) => item.text.find((i) => this.getname(i).toLowerCase().includes(m)));
102 + },
103 + bmk: (item) => this.vals(item).map((v) => v.toString().toLowerCase().replaceAll(/[\$\xA2-\xA5\u058F\u060B\u09F2\u09F3\u09FB\u0AF1\u0BF9\u0E3F\u17DB\u20A0-\u20BD\uA838\uFDFC\uFE69\uFF04\uFFE0\uFFE1\uFFE5\uFFE6,+]/g, '').replaceAll('b', 'kkk').replaceAll('m', 'kk').replaceAll('k', '000')),
104 + sortform: (item) => this.vals(item).map((val) => val?.toString()?.toLowerCase()?.replace(/^the /, '')),
55 105 zip: (item) => {
56 - const itemkeys = Object.keys(item).filter((k) => k !== 'of');
106 + const itemkeys = this.kkeys(item);
57 107 const zipped = [];
58 - for (let i=0; i<item[itemkeys[0]].length; i++) {
108 + for (let i = 0; i < item[itemkeys[0]].length; i++) {
59 109 const zipline = {};
60 110 for (const key of itemkeys) {
61 111 zipline[key] = item[key][i];
@@ -65,25 +115,25 @@ export class DACTAL {
65 115 return zipped;
66 116 },
67 117 pairs: (item) => {
68 - return item.of.slice(0, -1).map((val, vx) => ({pair: [val, item.of[vx+1]]}));
118 + return item.of.slice(0, -1).map((val, vx) => ({pair: [val, item.of[vx + 1]]}));
69 119 },
70 120 triples: (item) => {
71 - return item.of.slice(0, -2).map((val, vx) => ({triple: [val, item.of[vx+1], item.of[vx+2]]}));
121 + return item.of.slice(0, -2).map((val, vx) => ({triple: [val, item.of[vx + 1], item.of[vx + 2]]}));
72 122 },
73 123 quads: (item) => {
74 - return item.of.slice(0, -3).map((val, vx) => ({quad: [val, item.of[vx+1], item.of[vx+2], item.of[vx+3]]}));
124 + return item.of.slice(0, -3).map((val, vx) => ({quad: [val, item.of[vx + 1], item.of[vx + 2], item.of[vx + 3]]}));
75 125 },
76 126 sequences: (item) => {
77 - return item.of.map((val, valx, vallist) => ({sequence: vallist.slice(0, valx+1).map((val) => this.dcopy(val))}));
127 + return item.of.map((val, valx, vallist) => ({sequence: vallist.slice(0, valx + 1).map((val) => this.dcopy(val))}));
78 128 },
79 129 split: (item) => {
80 130 const itemvals = this.vals(item);
81 131 let splitter;
82 132 let tobesplit;
83 - if (Object.keys(item).length === 1) itemvals.push(' ');
84 - if (itemvals.length === 1) {
133 + if (Object.keys(item).length == 1) itemvals.push(' ');
134 + if (itemvals.length == 1) {
85 135 splitter = itemvals[0];
86 - tobesplit = [this.getname(item)];
136 + tobesplit = item.of.slice(0);
87 137 } else {
88 138 splitter = itemvals.pop();
89 139 tobesplit = itemvals.slice(0);
@@ -91,26 +141,46 @@ export class DACTAL {
91 141 if (splitter.startsWith('~')) splitter = new RegExp(splitter.replace(/^~*/, ''), splitter.startsWith('~~') ? 'i' : '');
92 142 return tobesplit.flatMap((v) => v.toString().split(splitter));
93 143 },
144 + unpack: (item) => {
145 + return item.of.flatMap((val) => {
146 + let valstr = val.toString();
147 + return this.vals(item).reduce((acc, size, x) => {
148 + acc.push(valstr.substring(0, size));
149 + valstr = valstr.substring(size);
150 + return acc;
151 + }, []);
152 + });
153 + },
94 154 extract: (item) => {
95 155 const itemvals = this.vals(item);
96 156 const delimiters = itemvals.pop();
97 157 const res = [];
98 - for (const itemval of itemvals) {
99 - for(let i=0; i<delimiters.length; i+=2) {
100 - const d1 = delimiters[i];
101 - const d2 = delimiters[i+1];
102 - const d1x = itemval.indexOf(d1);
103 - const d2x = itemval.indexOf(d2);
104 - if (d1x > -1 && d2x > d1x) {
105 - res.push(itemval.slice(d1x+1, d2x).trim());
106 - break;
158 + for (const itemval of (itemvals.length > 0 ? itemvals : item.of)) {
159 + if (delimiters.startsWith('~')) {
160 + itemval.matchAll(new RegExp(delimiters.slice(1), 'g')).forEach((match) => {
161 + if (match.groups) {
162 + res.push(match.groups);
163 + } else {
164 + match.slice(1).forEach((m) => res.push(m));
165 + }
166 + });
167 + } else {
168 + for (let i = 0; i < delimiters.length; i += 2) {
169 + const d1 = delimiters[i];
170 + const d2 = delimiters[i + 1];
171 + const d1x = itemval.indexOf(d1);
172 + const d2x = itemval.indexOf(d2);
173 + if (d1x > -1 && d2x > d1x) {
174 + res.push(itemval.slice(d1x + 1, d2x).trim());
175 + break;
176 + }
107 177 }
108 178 }
109 179 }
110 180 return res;
111 181 },
112 182 unchain: (item) => {
113 - const chainprops = Object.keys(item).filter((key) => key !== 'of');
183 + const chainprops = this.kkeys(item);
114 184 const unchained = [];
115 185 const queue = item.of.slice(0);
116 186 while (queue.length > 0) {
@@ -128,15 +198,18 @@ export class DACTAL {
128 198 }
129 199 }
130 200 }
131 - return unchained;
201 + return unchained;
132 202 },
133 203 itemize: (item) => {
134 204 const propname = item?.property ?? 'property';
135 205 const valname = item?.value ?? 'value';
136 - return item.of.flatMap((subitem) => Object.entries(subitem).map(([key, val]) => ({[propname]: key, [valname]: val})));
206 + return item.of.flatMap((subitem) => Object.entries(subitem).map(([key, val]) => ({
207 + [propname]: key,
208 + [valname]: val
209 + })));
137 210 },
138 211 schematize: (item) => {
139 - const itemkeys = Object.keys(item).filter((k) => k !== 'of');
212 + const itemkeys = this.kkeys(item);
140 213 const schematized = {};
141 214 item[itemkeys[0]].forEach((subitem) => {
142 215 let subkey;
@@ -151,10 +224,13 @@ export class DACTAL {
151 224 return schematized;
152 225 },
153 226 index: (item) => {
154 - return Object.entries(item).filter(([k, v]) => k !== 'of').map(([k, v]) => ({id: k, name: !isNaN(v) ? Number(v) : v}));
227 + return Object.entries(item).filter(([k, v]) => k != 'of').map(([k, v]) => ({
228 + id: k,
229 + name: !isNaN(v) ? Number(v) : v
230 + }));
155 231 },
156 232 unflatten: (item) => {
157 - const itemkeys = Object.keys(item).filter((k) => k !== 'of');
233 + const itemkeys = this.kkeys(item);
158 234 if (itemkeys.length === 0) itemkeys.push('');
159 235 const newindex = {};
160 236 const neworder = [];
@@ -185,20 +261,20 @@ export class DACTAL {
185 261 detupled: (item) => {
186 262 const newobj = {};
187 263 item.of.forEach((subitem) => {
188 - if (Array.isArray(subitem) && subitem.length === 2) {
264 + if (Array.isArray(subitem) && subitem.length == 2) {
189 265 newobj[subitem[0]] = subitem[1];
190 266 }
191 267 })
192 268 return [newobj];
193 269 },
194 270 csv: (item) => {
195 - const keys = Object.entries(item).find(([k, v]) => k !== 'of')[1];
271 + const keys = Object.entries(item).find(([k, v]) => k != 'of')[1];
196 272 const res = [];
197 273 const vals = item.of.flatMap((val) => this.dtype(val, 'string') ? val.split('\n').map((val) => val.trim()) : val);
198 - for (let i=0; i<vals.length; i+=keys.length) {
274 + for (let i = 0; i < vals.length; i += keys.length) {
199 275 const newitem = {};
200 - for (let k=0; k<keys.length; k++) {
201 - newitem[keys[k]] = vals[i+k];
276 + for (let k = 0; k < keys.length; k++) {
277 + newitem[keys[k]] = vals[i + k];
202 278 }
203 279 res.push(newitem);
204 280 }
@@ -206,20 +282,20 @@ export class DACTAL {
206 282 },
207 283 tsv: (item) => {
208 284 const text = this.getname(item);
209 - const lines = text.split('\n').filter((line) => line !== '').map((line) => line.split('\t').map((val) => val.trim()));
285 + const lines = text.split('\n').filter((line) => line != '').map((line) => line.split('\t').map((val) => val.trim()));
210 286 const keys = lines[0];
211 287 return lines.slice(1).map((vals) => Object.fromEntries(vals.map((val, vi) => [keys[vi], val])));
212 288 },
213 289 ssv: (item) => {
214 290 const text = this.getname(item);
215 - const lines = text.split('\n').filter((line) => line !== '').map((line) => line.split(/ +/).map((val) => val.trim()));
291 + const lines = text.split('\n').filter((line) => line != '').map((line) => line.split(/ +/).map((val) => val.trim()));
216 292 const keys = lines[0];
217 293 return lines.slice(1).map((vals) => Object.fromEntries(vals.map((val, vi) => [keys[vi], val])));
218 294 },
219 295 json: (item) => this.vals(item).flatMap((v) => JSON.parse(v)),
220 296 year: (item) => {
221 - if (item.date?.length > 0) return item.date[0].match(/\d\d\d\d/)[0];
222 - let yearmatch = this.getname(item).match(/\d\d\d\d/);
297 + if (item.date?.toString()?.length > 0) return item.date?.[0]?.toString()?.match(/\d\d\d\d/)?.[0];
298 + let yearmatch = this.getname(item.toString()).match(/\d\d\d\d/);
223 299 if (yearmatch) {
224 300 return yearmatch[0];
225 301 } else {
@@ -245,14 +321,21 @@ export class DACTAL {
245 321 },
246 322 date: (item) => {
247 323 let itemvals = this.vals(item);
248 - if (itemvals) {
249 - let datematch = itemvals[0].match(/\d\d\d\d-\d\d-\d\d/);
324 + if (itemvals?.length > 0) {
325 + let datematch = itemvals[0].toString().match(/\d\d\d\d-\d\d-\d\d/);
250 326 if (datematch) {
251 327 return datematch[0];
252 328 } else {
253 329 datematch = this.getid(item.of[0] || '').toString().match(/\d\d\d\d-\d\d-\d\d/);
254 330 if (datematch) {
255 331 return datematch[0];
332 + } else {
333 + if (this.dtype(itemvals[0], 'number')) {
334 + datematch = itemvals[0].toString().slice(0, 4);
335 + if (datematch) {
336 + return datematch;
337 + }
338 + }
256 339 }
257 340 }
258 341 }
@@ -262,17 +345,18 @@ export class DACTAL {
262 345 const days = ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'];
263 346 if (item.date?.length > 0) return days[new Date(item.date).getDay()];
264 347 },
348 + time: (item) => this.vals(item).map((v) => v.split('T')[1].slice(0, 5)),
265 349 timeshift: (item) => {
266 350 const vals = this.vals(item);
267 351 let tsx = new Date(vals[0]);
268 - let adjust = Number(vals[1]) * 60*60*1000;
352 + let adjust = Number(vals[1]) * 60 * 60 * 1000;
269 353 tsx.setTime(tsx.getTime() + adjust);
270 354 return tsx.toISOString();
271 355 },
272 356 hour: (item) => this.vals(item).map((v) => v.split('T')[1].split(':')[0]),
273 357 datediff: (item) => {
274 358 const [d1, d2] = this.vals(item);
275 - return (new Date(d2) - new Date(d1)) / (24*60*60*1000);
359 + return ((d2 ? new Date(d2) : new Date()) - new Date(d1)) / (24 * 60 * 60 * 1000);
276 360 },
277 361 timediff: (item) => {
278 362 const [d1, d2] = this.vals(item);
@@ -284,14 +368,29 @@ export class DACTAL {
284 368 return [
285 369 basedate,
286 370 `${basemonth}/${baseday}/${baseyear}`,
287 - basemonth.startsWith('0') ? `${basemonth.replace(/^0/,'')}/${baseday}/${baseyear}` : null,
288 - basemonth.startsWith('0') && baseday.startsWith('0') ? `${basemonth.replace(/^0/,'')}/${baseday.replace(/^0/,'')}/${baseyear}` : null,
371 + basemonth.startsWith('0') ? `${basemonth.replace(/^0/, '')}/${baseday}/${baseyear}` : null,
372 + basemonth.startsWith('0') && baseday.startsWith('0') ? `${basemonth.replace(/^0/, '')}/${baseday.replace(/^0/, '')}/${baseyear}` : null,
289 373 `${basemonth}/${baseday}/${baseyear.slice(2)}`,
290 - `${basemonth.replace(/^0/,'')}/${baseday.replace(/^0/,'')}/${baseyear.slice(2)}`,
291 - `${['', 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'][Number(basemonth)]} ${baseday.replace(/^0/,'')}, ${baseyear}`
374 + `${basemonth.replace(/^0/, '')}/${baseday.replace(/^0/, '')}/${baseyear.slice(2)}`,
375 + `${['', 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'][Number(basemonth)]} ${baseday.replace(/^0/, '')}, ${baseyear}`
292 376 ].filter(x => x);
293 377 },
294 378 now: (item) => performance.now(),
379 + spandays: (item) => {
380 + const startdate = Array.isArray(item.start) ? item.start[0] : item.start;
381 + const enddate = Array.isArray(item.end) ? item.end[0] : item.end;
382 + const days = [];
383 + const d = new Date(startdate);
384 + d.setUTCHours(0, 0, 0, 0);
385 + const stop = new Date(enddate);
386 + stop.setUTCHours(0, 0, 0, 0);
387 +
388 + while (d < stop) {
389 + days.push(d.toISOString().slice(0, 10));
390 + d.setUTCDate(d.getUTCDate() + 1);
391 + }
392 + return days;
393 + },
295 394 round: (item) => this.numvals(item).map((v) => Math.round(v)),
296 395 roundaway: (item) => this.numvals(item).map((v) => Math.sign(v) * Math.round(Math.abs(v))),
297 396 roundm: (item) => {
@@ -313,27 +412,28 @@ export class DACTAL {
313 412 let i = 0;
314 413 const test = this.getname(item.of[0]);
315 414 for (const key in item) {
316 - if (key !== 'of') {
415 + if (key != 'of') {
317 416 i--;
318 - if (key === test) return i;
417 + if (key == test) return i;
319 418 }
320 419 }
321 420 return i - 1;
322 421 },
323 422 numbers: (item) => {
324 423 const res = [];
325 - const firstnum = item.from !== undefined && item.from != null ? Number(item.from) : 1;
326 - const lastnum = item.to !== undefined && item.to != null ? Number(item.to) : Number(this.getname(item.of[0]));
327 - for(let x=firstnum; x<=lastnum; x++) {
424 + const firstnum = item.from != undefined && item.from != null ? Number(item.from) : 1;
425 + const lastnum = item.to != undefined && item.to != null ? Number(item.to) : Number(this.getname(item.of[0]));
426 + for (let x = firstnum; x <= lastnum; x++) {
328 427 res.push(x)
329 428 }
330 429 return res;
331 430 },
332 431 dehyphenate: (item) => this.vals(item).flatMap((v) => v.replaceAll(/(?<=\w)-\n(?=\w)/g, '')),
333 - sentences: (item) => this.vals(item).flatMap((v) => v.split(/[.?!…]['"’”»]?\s+/)),
334 - allwords: (item) => this.vals(item).flatMap((v) => Array.from(v.toString().toLowerCase().matchAll(/[\p{Letter}\p{Number}]+(?:['‘’][\p{Letter}\p{Number}]+)?/gu).map((m) => m[0]))),
335 - words: (item) => this.vals(item).flatMap((v) => Array.from(v.toString().toLowerCase().matchAll(/[\p{Letter}\p{Number}]+(?:['‘’][\p{Letter}\p{Number}]+)?/gu).map((m) => m[0])).filter((w) => w.length >= 4)),
336 - characters: (item) => this.vals(item).flatMap((v) => Array.from(v)),
432 + deparen: (item) => this.vals(item).flatMap((v) => v.replaceAll(/\s*\(.*?\)\s*/g, '')),
433 + sentences: (item) => this.vals(item).flatMap((v) => v.split(/[.?!…]['"’”»]?\s+/)),
434 + allwords: (item) => this.vals(item).flatMap((v) => Array.from(v.toString().toLowerCase().matchAll(/[\p{Letter}\p{Number}]+(?:['‘’][\p{Letter}\p{Number}]+)?/gu).map((m) => m[0]))),
435 + words: (item) => this.vals(item).flatMap((v) => Array.from(v.toString().toLowerCase().matchAll(/[\p{Letter}\p{Number}]+(?:['‘’][\p{Letter}\p{Number}]+)?/gu).map((m) => m[0])).filter((w) => w.length >= 4)),
436 + characters: (item) => this.vals(item).flatMap((v) => Array.from(v.toString())),
337 437 'character count': (item) => this.vals(item).flatMap((v) => v.length),
338 438 case: (item) => this.vals(item).map((v) => {
339 439 const hasupper = v.match(/[A-Z]/);
@@ -353,9 +453,13 @@ export class DACTAL {
353 453 }
354 454 }),
355 455 uppercase: (item) => this.vals(item).flatMap((v) => v.toString().toUpperCase()),
356 - lowercase: (item) => this.vals(item).flatMap((v) => v.toString().toLowerCase().replaceAll(/[‘’`]/g, "'").replaceAll(/[“”]/g, '"')),
357 - list: (item) => Object.keys(item).filter((k) => k !== 'of'),
456 + lowercase: (item) => this.vals(item).flatMap((v) => v.toString().toLowerCase().replaceAll(/[‘’`]/g, "'").replaceAll(/[“”]/g, '"')),
457 + list: (item) => this.kkeys(item),
358 458 items: (item) => item?.of?.flatMap((typename) => this.data?.[this.getname(typename)]),
459 + traverse: (item) => {
460 + const props = this.vals(item);
461 + return item?.of?.flatMap((subitem) => props.flatMap((prop) => this.step(subitem, prop, null, {})));
462 + },
359 463 random: (item) => Math.random(),
360 464 shuffle: (item) => {
361 465 const newArray = [].concat(item?.of || []);
@@ -366,7 +470,10 @@ export class DACTAL {
366 470 return newArray;
367 471 },
368 472 'weighted shuffle': (item) => {
369 - return item.of.map((subitem) => ({weight: Number(subitem.weight || 1) * Math.random(), subitem: subitem})).sort((a, b) => b.weight - a.weight).map((ws) => ws.subitem);
473 + return item.of.map((subitem) => ({
474 + weight: Number(subitem.weight || 1) * Math.random(),
475 + subitem: subitem
476 + })).sort((a, b) => b.weight - a.weight).map((ws) => ws.subitem);
370 477 },
371 478 pick: (item) => item?.of?.[~~(Math.random() * item?.of?.length)],
372 479 link: (item) => `<a href="${item.url}"${item.target ? ' target="' + item.target + '"' : ''}>${item.text}</a>`,
@@ -387,27 +494,29 @@ export class DACTAL {
387 494 const char = str.charCodeAt(i);
388 495 hash = (hash << 5) - hash + char;
389 496 }
390 - // Convert to 32bit unsigned integer in base 36 and pad with "0" to ensure length is 7.
391 497 return Object.assign({id: (hash >>> 0).toString(36).padStart(7, '0')}, subitem);
392 498 }
393 499 }),
394 - 'parse query': (item) => item.of.map((subitem) => ({name: subitem.name, query: subitem.query, assembly: this.parse(subitem.query)}))
500 + 'parse query': (item) => item.of.map((subitem) => ({
501 + name: subitem.name,
502 + query: subitem.query,
503 + assembly: this.parse(subitem.query)
504 + })),
505 + results: (item) => this.execute(item.of, this.parse(item.query || item.of?.[0]))
395 506 }
396 - this.data.aggregators = Object.keys(this.aggregators);
397 - // this.cachehitcount = 0;
398 - // this.cachemisscount = {};
507 + this.data.annotators = Object.entries(this.annotators).map(([key, code]) => ({id: key, code: code}));
399 508 this.destinations = new Set();
400 509 }
401 -
510 +
402 511 survey() {
403 512 this.destinations = new Set(Object.keys(this.data).concat(this.data?.queries?.map((q) => q.name) || []).concat(Object.keys(this.adapters)));
404 513 }
405 -
514 +
406 515 vals(item) {
407 516 const getname = this.getname;
408 517 const itemvals = [];
409 518 if (Object.keys(item).length > 1) {
410 - Object.entries(item).filter(([k, v]) => k !== 'of').flatMap(([k, v]) => Array.isArray(v) ? v : [v]).map(getname).forEach((v) => itemvals.push(v));
519 + Object.entries(item).filter(([k, v]) => k != 'of').flatMap(([k, v]) => Array.isArray(v) ? v : [v]).map(getname).forEach((v) => itemvals.push(v));
411 520 } else {
412 521 item.of.map(getname).forEach((v) => itemvals.push(v));
413 522 }
@@ -418,12 +527,17 @@ export class DACTAL {
418 527 return this.vals(item).filter((val) => this.dtype(val, 'number')).map((val) => Number(val));
419 528 }
420 529
421 - async querylive(query, inputlist=null) {
530 + kkeys(item) {
531 + return Object.keys(item).filter((k) => k != 'of');
532 + }
533 +
534 + async querylive(query, inputlist = null) {
422 535 this.recache = new Set();
423 - return await this.query(query, inputlist);
536 + const res = await this.query(query, inputlist);
537 + return res;
424 538 }
425 539
426 - async query(query, inputlist=null, loop=50) {
540 + async query(query, inputlist = null, loop = 50) {
427 541 this.survey();
428 542 const operations = Array.isArray(query) ? query : this.assemble(this.tokenize(query));
429 543 const result = this.execute(inputlist, operations);
@@ -432,7 +546,8 @@ export class DACTAL {
432 546 if (loop > 0 && queued.length > 0) {
433 547 await this.adapt();
434 548 if (this.recache) queued.forEach((q) => this.recache.add(q));
435 - return await this.query(operations, inputlist, loop - 1);
549 + const reres = await this.query(operations, inputlist, loop - 1);
550 + return reres;
436 551 } else if (loop === 0) {
437 552 this.recache = false;
438 553 Object.keys(this.adapters).forEach((key) => {
@@ -443,32 +558,35 @@ export class DACTAL {
443 558 }
444 559 })
445 560 }
561 + this.recache = false;
446 562 return result;
447 563 }
448 -
564 +
449 565 async adapt() {
450 566 for (const key in this.adapters) {
451 567 const adapter = this.adapters[key];
452 568 if (adapter.queue.length > 0) {
453 569 if (adapter.annotator) {
454 570 while (adapter.queue.length > 0) {
455 - const item_to_annotate = adapter.queue[0];
571 + this.statusf('annotating ' + key + ' ' + adapter.queue.length);
572 + const item_to_annotate = adapter.queue.shift();
456 573 try {
457 - (this.index[key] ||= {})[this.getid(item_to_annotate)] = await adapter.f(item_to_annotate);
458 - this.index_modified = true;
459 - adapter.queue.shift();
460 - } catch (e) {}
574 + const annotation = await adapter.f(item_to_annotate);
575 + (this.index[key] ||= {})[this.getid(item_to_annotate)] = annotation;
576 + this.indexlogit(key, 'write');
577 + } catch (e) {
578 + console.error(e);
579 + }
461 580 }
462 581 } else {
463 582 const newids = adapter.queue.filter((id) => !adapter.pending.has(id));
464 583 newids.forEach((id) => adapter.pending.add(id));
465 - const res = await adapter.f(newids);
584 + const res = await adapter.f(Array.from(new Set(newids)));
466 585 newids.forEach((id) => adapter.pending.delete(id));
467 - for (let i=0; i<adapter.queue.length; i++) {
468 - const item = res[i];
469 - const id = this.getid(item) ?? adapter.queue[i];
586 + for (const item of res) {
587 + const id = this.getid(item);
470 588 (this.index[key] ||= {})[id] = item;
471 - this.index_modified = true;
589 + this.indexlogit(key, 'write');
472 590 if (key in this.data) {
473 591 this.data[key].push(item);
474 592 }
@@ -477,16 +595,22 @@ export class DACTAL {
477 595 }
478 596 }
479 597 }
598 + this.adaptive = false;
480 599 }
481 -
482 - async rehope() {
600 +
601 + async rehope(only = null) {
602 + var reset = 0;
483 603 Object.keys(this.index).forEach((k) => {
484 - const todelete = Object.keys(this.index[k]).filter((dk) => this.index[k][dk] == null || this.index[k][dk] === undefined);
485 - todelete.forEach((dk) => delete(this.index[k][dk]));
604 + if (!only || only == k) {
605 + const todelete = Object.keys(this.index[k]).filter((dk) => this.index[k][dk] == null || this.index[k][dk] == undefined);
606 + todelete.forEach((dk) => delete (this.index[k][dk]));
607 + reset += todelete.length;
608 + }
486 609 });
610 + return reset;
487 611 }
488 612
489 - load(something, named, append=false) {
613 + load(something, named, append = false) {
490 614 if (!named) return
491 615 if (!append || !(named in this.data)) this.data[named] = [];
492 616 if (Array.isArray(something)) {
@@ -508,34 +632,34 @@ export class DACTAL {
508 632 }
509 633 return this.data[named];
510 634 }
511 -
512 - async loadjsonl(something, named, append=false) {
635 +
636 + async loadjsonl(something, named, append = false) {
513 637 if (!named) return
514 638 if (!append || !(named in this.data)) this.data[named] = [];
515 639 if (typeof something == 'string' && (something.startsWith('http') || something.startsWith('file://'))) {
516 640 const fetchres = await fetch(something);
517 641 something = await fetchres.text();
518 642 }
519 - const rows = something.trim().split(/[\n\r]+/);
643 + var rows = something.trim().split(/[\n\r]+/);
520 644 for (const row of rows) {
521 645 const rowdata = JSON.parse(row);
522 646 if (rowdata) this.data[named].push(rowdata);
523 647 }
524 648 }
525 649
526 - async loadcsv(something, named, quoteChar = '"', delimiter = ',', headerrows=1) {
650 + async loadcsv(something, named, quoteChar = '"', delimiter = ',', headerrows = 1) {
527 651 if (typeof something == 'string' && (something.startsWith('http') || something.startsWith('file://'))) {
528 652 const fetchres = await fetch(something);
529 653 something = await fetchres.text();
530 654 }
531 - const rows = something.split(/[\n\r]+/);
655 + var rows = something.split(/[\n\r]+/);
532 656
533 657 const regex = new RegExp(`\\s*(${quoteChar})?(.*?)\\1\\s*(?:${delimiter}|$)`, 'gs');
534 -
658 +
535 659 const match = (line) => Array.from(line.matchAll(regex), (m) => m[2]);
536 660
537 661 const headers = [];
538 - for (let hrowx=0; hrowx<headerrows; hrowx++) {
662 + for (let hrowx = 0; hrowx < headerrows; hrowx++) {
539 663 const hrow = rows.shift();
540 664 match(hrow).forEach((h, hx) => {
541 665 if (hrowx === 0) {
@@ -545,41 +669,41 @@ export class DACTAL {
545 669 }
546 670 });
547 671 }
548 - const heads = headers ?? match(rows.shift());
549 - const lines = rows.slice(0).filter((line) => line);
672 + const heads = headers.length > 0 ? headers : match(rows.shift());
673 + var lines = rows.slice(0).filter((line) => line);
550 674 const parsed = lines.map((line) => {
551 - return match(line).reduce((acc, cur, i) => {
552 - // replace blank matches with `null`
553 - const val = cur.length <= 0 ? null : (!isNaN(cur) ? Number(cur) : cur);
554 - const key = heads[i] ?? `{i}`;
555 - if (key === '') {
556 - return { ...acc};
557 - } else {
558 - return { ...acc, [key]: val };
559 - }
560 - }, {});
675 + return match(line).reduce((acc, cur, i) => {
676 + // replace blank matches with `null`
677 + const val = cur.length <= 0 ? null : (!isNaN(cur) ? Number(cur) : cur);
678 + const key = heads[i] ?? `{i}`;
679 + if (key == '') {
680 + return {...acc};
681 + } else {
682 + return {...acc, [key]: val};
683 + }
684 + }, {});
561 685 });
562 686 this.load(parsed, named);
563 687 }
564 -
688 +
565 689 apacheLogToDate(apacheTimestamp) {
566 690 // Apache log format: [10/Oct/2000:13:55:36 -0700]
567 691 // Remove brackets if present
568 - const cleanTimestamp = apacheTimestamp.replace(/^\[|]$/g, '');
569 -
692 + const cleanTimestamp = apacheTimestamp.replace(/^\[|\]$/g, '');
693 +
570 694 // Split into date/time and timezone parts
571 695 const [dateTimePart, timezone] = cleanTimestamp.split(' ');
572 -
696 +
573 697 // Parse the date/time part: dd/MMM/yyyy:HH:mm:ss
574 698 const [datePart, hour, minute, second] = dateTimePart.split(':');
575 699 const [day, month, year] = datePart.split('/');
576 -
700 +
577 701 // Month mapping
578 702 const months = {
579 703 'Jan': 0, 'Feb': 1, 'Mar': 2, 'Apr': 3, 'May': 4, 'Jun': 5,
580 704 'Jul': 6, 'Aug': 7, 'Sep': 8, 'Oct': 9, 'Nov': 10, 'Dec': 11
581 705 };
582 -
706 +
583 707 // Create Date object (months are 0-indexed in JS)
584 708 const date = new Date(
585 709 parseInt(year),
@@ -589,28 +713,28 @@ export class DACTAL {
589 713 parseInt(minute),
590 714 parseInt(second)
591 715 );
592 -
716 +
593 717 // Handle timezone offset if present
594 718 if (timezone) {
595 719 const sign = timezone[0] === '+' ? 1 : -1;
596 720 const tzHours = parseInt(timezone.slice(1, 3));
597 721 const tzMinutes = parseInt(timezone.slice(3, 5));
598 722 const offsetMs = sign * (tzHours * 60 + tzMinutes) * 60 * 1000;
599 -
723 +
600 724 // Adjust for timezone (Apache logs are in local time, JS Date assumes UTC)
601 725 date.setTime(date.getTime() - offsetMs);
602 726 }
603 -
727 +
604 728 return date;
605 729 }
606 -
607 - async loadclf(loglines, named, apiroutes=[]) {
730 +
731 + async loadclf(loglines, named, apiroutes = []) {
608 732 const lines = loglines.trim().split('\n');
609 733 const cols9 = ['ip', 'name', 'username', 'timestamp', 'requestraw', 'status', 'bytes', 'referrer', 'useragent'];
610 734 const cols10 = ['host'].concat(cols9);
611 - const parsed = lines.map((line) => {
612 - const vals = Array.from(line.matchAll(/"(?:\\"|.)*?"|\[.*?]|\S+/g)).map((m) => m[0]);
613 - const cols = vals.length === 10 ? cols10 : cols9;
735 + const parsed = lines.filter((line) => line?.length > 0).map((line) => {
736 + const vals = Array.from(line.matchAll(/\"(?:\\"|.)*?\"|\[.*?\]|\S+/g)).map((m) => m[0]);
737 + const cols = vals.length == 10 ? cols10 : cols9;
614 738 const obj = Object.fromEntries(vals.map((v, vx) => ([cols[vx], v])));
615 739 const date = this.apacheLogToDate(obj.timestamp);
616 740 obj.timestamp = date.toISOString();
@@ -634,7 +758,7 @@ export class DACTAL {
634 758 });
635 759 this.load(parsed, named);
636 760 }
637 -
761 +
638 762 async loadrss(rsstext, named) {
639 763 const rssval = (k, rawval) => {
640 764 if (k.match(/date/i)) {
@@ -650,30 +774,58 @@ export class DACTAL {
650 774 const obj = {};
651 775 Array.from(i.children).forEach((c) => {
652 776 const k = c.tagName;
653 - const rawval = c.textContent;
777 + const rawval = c.textContent.trim();
654 778 obj[k] = rssval(k, rawval);
779 + if (k.match(/date/i) && obj[k].length > 10 && !('date' in obj)) obj.date = obj[k].slice(0, 10)
655 780 });
656 781 return obj;
657 782 });
658 - this.load(items, named);
783 + if (named) {
784 + this.load(items, named);
785 + } else {
786 + return items;
787 + }
659 788 }
660 -
661 - connect(key, adapter, annotator=false) {
662 - this.adapters[key] = {queue: [], pending: new Set(), f: adapter, annotator: annotator};
789 +
790 + async loadopml(opmltext, named) {
791 + const opmldom = new window.DOMParser().parseFromString(opmltext, "text/xml");
792 + const feeds = Array.from(opmldom.querySelectorAll('outline[type="rss"]')).map((i) => ({
793 + title: i.getAttribute('title'),
794 + feed: i.getAttribute('xmlUrl'),
795 + site: i.getAttribute('htmlUrl')
796 + }));
797 + if (named) {
798 + this.load(feeds, named);
799 + } else {
800 + return feeds;
801 + }
802 + }
803 +
804 + connect(key, adapter, doc = {}, annotator = null) {
805 + this.adapters[key] = {queue: [], pending: new Set(), f: adapter, annotator: annotator, doc: doc};
663 806 this.data.adapters ??= [];
664 - if (!this.data.adapters.includes(key)) this.data.adapters.push(key);
807 + if (!this.data.adapters.find((a) => a.id == key)) this.data.adapters.push({
808 + id: key,
809 + requires: doc.requires,
810 + produces: doc.produces,
811 + code: this.adapters[key].f
812 + });
813 + if (annotator) this.register(key, (item) => this.resolve(key, item), adapter);
665 814 }
666 815
667 - connect_annotator(key, adapter) {
668 - this.connect(key, adapter, true);
816 + connect_annotator(key, adapter, required = [], doc = {}) {
817 + this.connect(key, adapter, doc, required);
669 818 }
670 -
671 - register(key, aggregator) {
672 - this.aggregators[key] = aggregator;
673 - this.data.aggregators ??= [];
674 - if (!this.data.aggregators.includes(key)) this.data.aggregators.push(key);
819 +
820 + register(key, annotator, displayf = null) {
821 + this.annotators[key] = annotator;
822 + this.data.annotators ??= [];
823 + if (!this.data.annotators.find((a) => a.id == key)) this.data.annotators.push({
824 + id: key,
825 + code: displayf || this.annotators[key]
826 + });
675 827 }
676 -
828 +
677 829 unbracket(token) {
678 830 if (!(typeof token == 'string')) token = token.toString();
679 831 if (token.startsWith('[') && token.endsWith(']')) {
@@ -684,7 +836,7 @@ export class DACTAL {
684 836
685 837 bracket(token) {
686 838 if (!(typeof token == 'string')) token = token?.toString() ?? '';
687 - if (token.match(/[?.:#\/|!<>=~@\[\](),;+-]/) || token.startsWith(' ') || token.endsWith(' ')) {
839 + if (token.match(/[?.:#\/|!<>=~@\[\]\(\),;\+-]/) || token.startsWith(' ') || token.endsWith(' ')) {
688 840 return '[' + token.replaceAll(']', ']]') + ']';
689 841 }
690 842 return token;
@@ -696,17 +848,18 @@ export class DACTAL {
696 848
697 849 tokenize(text) {
698 850 if (typeof text != 'string') text = String(text);
699 - const matches = text.matchAll(/(\[(?:]]|[^\]])*])|(\?{1,3})|(\.{1,4})|(\|{1,2})|(\/{1,2})|([:#!])|(\()|(\))|([@~=<>+-]+)|(,)|(;)|([^\[\]().?:\/#|!~=<>@,;+-]+)|([\[\]])/gms);
700 - return Array.from(matches, m => m[0].trim()).filter((token) => token.length > 0);
851 + const matches = text.matchAll(/(\[(?:\]\]|[^\]])*\])|(\?{1,3})|(\.{1,4})|(\/{1,2})|([\:\#\|\!])|(\()|(\))|([@~=<>\+-]+)|(\,)|(\;)|([^\[\]\(\)\.\?\:\/\#\|\!~=<>@,;\+-]+)|([\[\]])/gms);
852 + const tokenlist = Array.from(matches, m => m[0].trim()).filter((token) => token.length > 0);
853 + return tokenlist;
701 854 }
702 855
703 856 assemble(tokenized) {
704 - const isOperator = (token) => '???....://#||!'.includes(token);
705 - const tokenlist = '= ~ =~ ~< ~> > < <> >< >= <= - =- ~- + =+ @ @@ @- @= @@= =@ =@@ @< @@< @<= @@<= @> @@> @>= @@>= >> >>= << <<= ~~ =>'.split(' ');
706 - const isSubop = (token) => tokenlist.includes(token) || (token?.startsWith('-') && (token.length === 1 || tokenlist.includes(token.slice(1))));
857 + const isOperator = (token) => '???....://#|!'.includes(token);
858 + const tokenlist = '= ~ =~ ~< ~> > < <> >< >= <= - =- ~- + =+ @ @@ @- @= @@= =@ =@@ @< @@< @<= @@<= @> @@> @>= @@>= =>'.split(' ');
859 + const isSubop = (token) => tokenlist.includes(token) || (token?.startsWith('-') && (token.length == 1 || tokenlist.includes(token.slice(1))));
707 860 const isSeparator = (token) => ',;'.includes(token);
708 861 const isValue = (token) => !isOperator(token) && !isSubop(token) && !isSeparator(token);
709 -
862 +
710 863 let tokens = tokenized.slice();
711 864 let level = 0;
712 865 const operations = [];
@@ -727,7 +880,7 @@ export class DACTAL {
727 880 }
728 881 while (tokens.length > 0 && !isOperator(tokens[0]) && !isSeparator(tokens[0])) {
729 882 const frag = tokens.shift();
730 - if (frag !== '(' && isValue(frag) && isSubop(tokens[0])) {
883 + if (frag != '(' && isValue(frag) && isSubop(tokens[0])) {
731 884 arg.label = this.unbracket(frag);
732 885 arg.subop = tokens.shift();
733 886 } else if (isSubop(frag)) {
@@ -751,11 +904,25 @@ export class DACTAL {
751 904 subquery.push(sub);
752 905 }
753 906 }
754 - arg.value = this.assemble(subquery);
907 + const subexpr = this.assemble(subquery);
908 + arg.value = subexpr;
909 + } else if (frag.match(/^[=<>@~-]+$/)) {
910 + const failure = {
911 + tokens: tokenized,
912 + assembled: operations.slice(0),
913 + assembling: {op: op, arg: arg, unexpected: frag},
914 + unassembled: tokens.slice(0)
915 + }
916 + throw new Error("Invalid subop", {cause: failure});
755 917 } else if (!arg.value) {
756 918 arg.value = this.unbracket(frag);
757 919 } else {
758 - const failure = {tokens: tokenized, assembled: operations.slice(0), assembling: {op: op, arg: arg, unexpected: frag}, unassembled: tokens.slice(0)}
920 + const failure = {
921 + tokens: tokenized,
922 + assembled: operations.slice(0),
923 + assembling: {op: op, arg: arg, unexpected: frag},
924 + unassembled: tokens.slice(0)
925 + }
759 926 throw new Error("Unexpected token", {cause: failure});
760 927 }
761 928 }
@@ -763,112 +930,157 @@ export class DACTAL {
763 930 }
764 931 }
765 932 operations.push(op);
766 -
933 +
767 934 }
768 935 if (level > 0) console.warn({parentropy: level, operations: operations});
769 936 return operations;
770 937 }
771 -
938 +
772 939 parse(querystr) {
773 940 return this.assemble(this.tokenize(querystr));
774 941 }
775 -
942 +
776 943 disassemble(assembly) {
777 - return assembly.map((operation, ox) => operation.operator + ((operation.args.length === 0 && ox < assembly.length - 1 && operation.operator[0] === assembly[ox + 1].operator[0]) ? ' ' : operation.args.map((arg) => (arg.separator ?? '') + (arg.label ? this.bracket(arg.label) : '') + (arg.subop ?? '') + (Array.isArray(arg.value) ? ('(' + this.disassemble(arg.value) + ')') : (arg.value ? this.bracket(arg.value) : ''))).join(''))).join('');
944 + return assembly.map((operation, ox) => operation.operator + ((operation.args.length == 0 && ox < assembly.length - 1 && operation.operator[0] == assembly[ox + 1].operator[0]) ? ' ' : operation.args.map((arg) => (arg.separator ?? '') + (arg.label ? this.bracket(arg.label) : '') + (arg.subop ?? '') + (Array.isArray(arg.value) ? ('(' + this.disassemble(arg.value) + ')') : (arg.value ? this.bracket(arg.value) : ''))).join(''))).join('').replace(/^\?(?=[a-z])/, '');
778 945 }
779 -
946 +
780 947 compact(querystr) {
781 948 return this.disassemble(this.parse(querystr));
782 949 }
783 -
950 +
784 951 executeq(querystr) {
785 952 return this.execute([], this.assemble(this.tokenize(querystr)));
786 953 }
787 -
954 +
788 955 timecheck(timer, i, count, op) {
789 - if (i === 1) timer.loopstart = new Date();
790 - if (i >= 10) {
956 + if (i == 1) timer.loopstart = new Date();
957 + if (i >= 10 && i >= count / 100) {
791 958 const taken = new Date() - timer.loopstart;
792 959 const projected = count * taken / i;
793 - if (projected > this.timelimit) throw new Error('Query overrun.', {cause: {operation: dactal.disassemble([op]), items: count, done: i, elapsed: taken + 'ms', projected: Math.round(projected / 60000) + ' minutes', timelimit: Math.round(this.timelimit / 60000) + ' minutes'}});
960 + if (this.timelimit && projected > this.timelimit) throw new Error('Query overrun.', {
961 + cause: {
962 + operation: this.disassemble([op]),
963 + items: count,
964 + done: i,
965 + elapsed: taken + 'ms',
966 + projected: Math.round(projected / 60000) + ' minutes',
967 + timelimit: Math.round(this.timelimit / 60000) + ' minutes'
968 + }
969 + });
794 970 }
795 971 }
796 972
797 - execute(inputlist, operations, labeled=null, level=null) {
973 + unmmss = (mmssstr) => {
974 + const parts = mmssstr.split(':');
975 + let s = 0;
976 + const seconds = parts.pop();
977 + if (seconds) s += Number(seconds);
978 + const minutes = parts.pop();
979 + if (minutes) s += 60 * Number(minutes);
980 + const hours = parts.pop();
981 + if (hours) s += 60 * 60 * Number(hours);
982 + return s;
983 + }
984 +
985 + dethe = (value) => {
986 + return value.toLowerCase().replace(/^the /, '')
987 + };
988 +
989 + compvals = (araw, braw) => {
990 + const a = araw.toString().trim();
991 + const b = braw.toString().trim();
992 + const atime = a.match(/^(?:\d+)(?:\:\d{2,})+$/);
993 + const btime = b.match(/^(?:\d+)(?:\:\d{2,})+$/);
994 + if (atime && btime) {
995 + return this.unmmss(btime[0]) - this.unmmss(atime[0]);
996 + }
997 + return this.dethe(a).localeCompare(this.dethe(b));
998 + }
999 +
1000 + execute(inputlistraw, operations, labeled = null, level = null) {
1001 + let inputlist = Array.isArray(inputlistraw) ? inputlistraw : inputlistraw ? [inputlistraw] : [];
798 1002 let currentlist = inputlist?.slice(0) || [];
799 1003 const getname = this.getname;
800 1004 const getid = this.getid;
801 1005 const dtype = this.dtype;
1006 + const nullish = this.nullish;
802 1007 const dcopy = this.dcopy;
1008 + const opclone = this.opclone;
803 1009 const step = this.step;
804 - const escapeRegExp = this.escapeRegExp;
805 - const unmmss = (mmssstr) => {
806 - const parts = mmssstr.split(':');
807 - let s = 0;
808 - const seconds = parts.pop();
809 - if (seconds) s += Number(seconds);
810 - const minutes = parts.pop();
811 - if (minutes) s += 60 * Number(minutes);
812 - const hours = parts.pop();
813 - if (hours) s += 60 * 60 * Number(hours);
814 - return s;
815 - };
816 - const compvals = (araw, braw) => {
817 - const a = araw.toString();
818 - const b = braw.toString();
819 - const atime = a.match(/^\d+(?::\d{2,})+$/);
820 - const btime = b.match(/^\d+(?::\d{2,})+$/);
821 - if (atime && btime) {
822 - return unmmss(btime[0]) - unmmss(atime[0]);
823 - }
824 - return dethe(a).localeCompare(dethe(b));
825 - }
826 -
827 - const dethe = (value) => value.toLowerCase().replace(/^the /, '');
828 -
1010 + const dethe = this.dethe;
1011 + const compvals = this.compvals;
1012 +
829 1013 labeled ||= {};
830 1014 let outputlist = [];
831 - for (let opx=0; opx<operations.length; opx++) {
1015 + const toplevel = !inputlistraw && !level;
1016 + if (toplevel && operations[0]?.progress) {
1017 + currentlist = operations[0].progress;
1018 + labeled = operations[0].labeled;
1019 + outputlist = currentlist;
1020 + }
1021 + for (let opx = 0; opx < operations.length; opx++) {
832 1022 const op = operations[opx];
1023 + if (op.completed) continue;
1024 + const opstart = performance.now();
833 1025 outputlist = [];
834 1026 switch (op.operator) {
835 1027 case '?': // start
836 - if (op.args.length === 0) {
837 - outputlist = Object.keys(this.data).sort((a, b) => dethe(a).localeCompare(dethe(b)));
1028 + if (op.args.length == 0) {
1029 + outputlist = Object.keys(this.data).filter((x) => !this.internal_datasets.includes(x)).sort((a, b) => dethe(a).localeCompare(dethe(b)));
838 1030 break;
839 1031 }
840 1032 outputlist = [];
841 1033 op.args.forEach((arg) => {
842 - let startitems;
843 - if (arg.label && arg.subop?.match(/\+/)) labeled[arg.label] ||= [];
844 - if (arg.separator !== ';' || outputlist.length === 0) {
1034 + let startitems = [];
1035 + if (arg.subop?.includes('+')) {
1036 + if (arg.label) {
1037 + if (arg.label in labeled) {
1038 + labeled[arg.label].forEach((i) => outputlist.push(i));
1039 + } else if (arg.label in this.data) {
1040 + this.data[arg.label].forEach((i) => outputlist.push(i));
1041 + }
1042 + } else {
1043 + currentlist.forEach((i) => outputlist.push(i));
1044 + }
1045 + }
1046 + let typeitems;
1047 + if (arg.separator != ';' || outputlist.length == 0) {
845 1048 if (Array.isArray(arg.value)) {
846 - startitems = this.execute([], arg.value, labeled);
1049 + startitems = this.execute(arg.subop?.includes('+') ? outputlist : [], arg.value, labeled);
1050 + } else if (arg.subop == '~') {
1051 + startitems = [arg.value];
1052 + } else if (arg.value in labeled) {
1053 + startitems = labeled[arg.value];
1054 + } else if (typeitems = this.gettype(arg.value)) {
1055 + startitems = typeitems;
1056 + } else if (arg.subop != '=') {
1057 + if (!isNaN(arg.value)) {
1058 + startitems = [Number(arg.value)];
1059 + } else {
1060 + startitems = [arg.value];
1061 + }
1062 + }
1063 + if (arg.label) this.index[arg.label] = {};
1064 + if (arg.subop?.includes('+')) {
847 1065 if (arg.label) {
848 - if (arg.subop?.match(/\+/)) {
849 - startitems.forEach((i) => labeled[arg.label].push(i));
850 - } else {
851 - labeled[arg.label] = startitems;
852 - this.index[arg.label] = {};
853 - }
854 - labeled[arg.label].forEach((i) => outputlist.push(i));
1066 + startitems.forEach((i) => {
1067 + if (arg.label in labeled) (labeled[arg.label] ||= []).push(i);
1068 + outputlist.push(i);
1069 + });
855 1070 } else {
856 1071 startitems.forEach((i) => outputlist.push(i));
857 1072 }
858 - } else if (arg.subop === '~') {
859 - outputlist.push(arg.value);
860 - } else if (arg.value in labeled) {
861 - labeled[arg.value].forEach((i) => outputlist.push(i));
862 - } else if (startitems = this.gettype(arg.value, false)) {
863 - startitems.forEach((i) => outputlist.push(i));
864 - } else if (arg.value?.trim().length > 0 && this.dtype(arg.value, 'number')) {
865 - if (arg.subop === '-' && arg.value === '0') {
866 - outputlist.push(-0);
1073 + } else if (arg.subop?.includes('-')) {
1074 + const startids = new Set(startitems.map((si) => getid(si)));
1075 + if (arg.label && labeled[arg.label]) {
1076 + labeled[arg.label] = labeled[arg.label].filter((li) => !startids.has(getid(li)));
1077 + outputlist = labeled[arg.label];
867 1078 } else {
868 - outputlist.push((arg.subop === '-' ? -1 : 1) * Number(arg.value));
1079 + outputlist = outputlist.filter((li) => !startids.has(getid(li)));
869 1080 }
870 - } else if (arg.value && arg.subop !== '=') {
871 - outputlist.push(arg.value);
1081 + } else {
1082 + if (arg.label) labeled[arg.label] = startitems;
1083 + startitems.forEach((si) => outputlist.push(si));
872 1084 }
873 1085 }
874 1086 });
@@ -878,33 +1090,46 @@ export class DACTAL {
878 1090 let ended = false;
879 1091 op.args.forEach((arg) => {
880 1092 if (!ended) {
881 - if (arg.label === '_timelimit' && !isNaN(arg.value)) {
1093 + if (arg.label == '_timelimit' && !isNaN(arg.value)) {
882 1094 this.timelimit = Number(arg.value) * 60000;
1095 + } else if (arg.label == 'status') {
1096 + if (arg.value) {
1097 + this.statusf(arg.value);
1098 + } else {
1099 + this.statusf(currentlist[0]);
1100 + }
883 1101 } else if (arg.label) {
884 1102 if (arg.subop.includes('~') && dtype(arg.value, 'string')) {
885 1103 labeled[arg.label] = [arg.value];
886 - } else if (arg.labeled) {
887 - labeled[arg.label] = arg.labeled;
1104 + // } else if (arg.labeled) {
1105 + // labeled[arg.label] = arg.labeled;
888 1106 } else {
889 1107 if (arg.subop?.match(/\+/)) labeled[arg.label] ||= [];
890 1108 const newvals = this.execute(currentlist, Array.isArray(arg.value) ? arg.value : '.' + arg.value, labeled, level);
891 1109 if (arg.subop?.match(/\+/)) {
892 1110 newvals.forEach((nv) => labeled[arg.label].push(nv));
1111 + } else if (arg.subop?.includes('-') && labeled[arg.label]) {
1112 + const newids = new Set(newvals.map((ni) => getid(ni)));
1113 + labeled[arg.label] = labeled[arg.label].filter((li) => !newids.has(getid(li)));
893 1114 } else {
894 1115 labeled[arg.label] = newvals;
895 1116 }
896 - if (!level && opx === 0) arg.labeled = newvals;
1117 + // if (!level && opx == 0) arg.labeled = newvals;
897 1118 }
898 1119 this.index[arg.label] = {};
899 1120 } else if (dtype(arg.value, 'string')) {
900 1121 if (arg.subop?.match(/\+/)) {
901 1122 labeled[arg.value] ??= [];
1123 + const currentids = new Set(currentlist.map((ci) => getid(ci)));
902 1124 currentlist.forEach((i) => labeled[arg.value].push(i));
1125 + } else if (arg.subop?.includes('-') && labeled[arg.value]) {
1126 + const currentids = new Set(currentlist.map((ci) => getid(ci)));
1127 + labeled[arg.value] = labeled[arg.value].filter((li) => !currentids.has(getid(li)));
903 1128 } else {
904 1129 labeled[arg.value] = currentlist.slice(0);
905 1130 }
906 1131 this.index[arg.value] = {};
907 - if (arg.value === 'end') {
1132 + if (arg.value == 'end') {
908 1133 ended = true;
909 1134 }
910 1135 }
@@ -915,11 +1140,16 @@ export class DACTAL {
915 1140 case '!': // repeat
916 1141 if (opx > 0) {
917 1142 const repeat_ops = operations.slice(opx - 1, opx + 1);
1143 + repeat_ops.forEach((rop) => {
1144 + delete (rop.progress);
1145 + delete (rop.labeled);
1146 + delete (rop.completed);
1147 + });
918 1148 outputlist = currentlist.slice(0);
919 1149 level ??= 0;
920 1150 level += 1;
921 1151 const maxrecursion = ((op.args.length > 0 && op.args[0].value) || 1000);
922 - if (outputlist.length > 0 && level < maxrecursion && (!this.samearray(inputlist, outputlist) || level === 1)) {
1152 + if (outputlist.length > 0 && level < maxrecursion && (!this.samearray(inputlist, outputlist) || level == 1)) {
923 1153 const recursed = this.execute(outputlist, repeat_ops, labeled, level);
924 1154 if (recursed.length > 0 && !this.samearray(recursed, outputlist)) outputlist = recursed.slice(0);
925 1155 }
@@ -928,44 +1158,41 @@ export class DACTAL {
928 1158 case '.': // traverse
929 1159 case '..': // traverse with duplicates
930 1160 const seen = new Set();
931 - const sofar = Array.isArray(op.args?.[0]?.value) && op.args?.[0]?.label;
932 - let transq = null;
1161 + const sofar = Array.isArray(op.args?.[0]?.value) && op.args?.[0]?.label;
933 1162 if (sofar) labeled[sofar] = [];
934 - if (op.operator === '.' && op.args?.length === 1) {
935 - const firstarg = op.args[0];
936 - const firstval = firstarg?.value;
937 - if (dtype(firstval, 'string')) {
938 - transq = this.data.queries?.find((q) => q.relative === true && q.name === firstval);
939 - }
940 - }
941 - if (transq) {
942 - outputlist = this.execute(currentlist, this.parse(transq.query), labeled, level);
943 - } else {
944 - const traversetimer = {};
945 - outputlist = currentlist.reduce((acc, item, i) => {
946 - this.timecheck(traversetimer, i, currentlist.length, op);
947 - if (op.args.length === 0) {
948 - const itemkey = getid(item);
949 - if (op.operator === '..' || !seen.has(itemkey)) {
950 - acc.push(item);
951 - seen.add(itemkey);
952 - }
953 - return acc;
1163 + const traversetimer = {};
1164 + outputlist = currentlist.reduce((acc, item, i) => {
1165 + this.timecheck(traversetimer, i, currentlist.length, op);
1166 + if (op.args.length == 0) {
1167 + const itemkey = getid(item);
1168 + if (op.operator == '..' || !seen.has(itemkey)) {
1169 + acc.push(item);
1170 + seen.add(itemkey);
954 1171 }
955 - if (op.args[0].subop?.includes('<') && acc.length > 0) return acc;
956 - let itemvals = [];
957 - let toremoveids = {};
958 - for (const arg of op.args.filter((arg) => arg.value != null)) {
959 - let passdown = null;
960 - if (arg.subop?.includes('>') && typeof arg.value == 'string') {
961 - if (arg?.label in item) {
962 - passdown = item[arg.label];
1172 + return acc;
1173 + }
1174 + if (op.args[0].subop?.includes('<') && acc.length > 0) return acc;
1175 + let itemvals = [];
1176 + let toremoveids = {};
1177 + for (const arg of op.args.filter((arg) => arg.value != null && arg.value != undefined)) {
1178 + if (arg.separator != ';' || itemvals.length == 0) {
1179 + let itemval;
1180 + if (arg.subop?.includes('@') && arg.label != null && !isNaN(arg.value)) {
1181 + const itemvalraw = step(item, arg.label, null, labeled);
1182 + if (arg.subop.includes('@@')) {
1183 + itemval = itemvalraw.slice(itemvalraw.length - Number(arg.value));
963 1184 } else {
964 - passdown = JSON.parse(JSON.stringify(item, Object.keys(item).filter((k) => k !== arg.value)));
1185 + itemval = itemvalraw.slice(0, Number(arg.value));
965 1186 }
1187 + } else if (arg.subop?.includes('=') && (arg.subop?.includes('>') || arg.label == null)) {
1188 + if (arg.value in item) {
1189 + itemval = item[arg.value];
1190 + }
1191 + } else {
1192 + itemval = step(item, arg.value, arg.subop, labeled);
966 1193 }
967 - if (arg.separator !== ';' || itemvals.length === 0) {
968 - const itemval = step(item, arg.value, arg.subop, labeled);
1194 + if (itemval) {
1195 + if (!Array.isArray(itemval)) itemval = [itemval];
969 1196 if (arg.subop?.includes('-') && isNaN(arg.value)) {
970 1197 itemval.forEach((subitem) => {
971 1198 const subid = getid(subitem);
@@ -973,17 +1200,25 @@ export class DACTAL {
973 1200 toremoveids[subid] += 1;
974 1201 });
975 1202 } else {
1203 + var passdown = null;
1204 + if (arg.subop?.includes('>')) {
1205 + if (arg?.label in item) {
1206 + passdown = item[arg.label];
1207 + } else {
1208 + passdown = JSON.parse(JSON.stringify(item, Object.keys(item).filter((k) => k != arg.value)));
1209 + }
1210 + }
976 1211 itemval.forEach((subitem) => {
977 1212 if (passdown) {
978 1213 if (!dtype(subitem, 'object')) subitem = {value: subitem};
979 1214 if (arg.label) {
980 - subitem[arg.label] = [passdown];
1215 + subitem[arg.label] = Array.isArray(passdown) ? passdown : [passdown];
981 1216 } else {
982 1217 Object.assign(subitem, passdown);
983 1218 }
984 1219 }
985 1220 const subitemkey = getid(subitem);
986 - if (op.operator === '..' || !seen.has(subitemkey)) {
1221 + if (op.operator == '..' || !seen.has(subitemkey)) {
987 1222 seen.add(subitemkey);
988 1223 itemvals.push(subitem);
989 1224 if (sofar) labeled[sofar].push(subitem);
@@ -991,64 +1226,104 @@ export class DACTAL {
991 1226 })
992 1227 }
993 1228 }
994 - if (Object.keys(toremoveids).length > 0) {
995 - if (op.operator === '.') {
996 - itemvals = itemvals.filter((subitem) => !(getid(subitem) in toremoveids));
997 - } else {
998 - itemvals = itemvals.filter((subitem) => {
999 - const subid = getid(subitem);
1000 - if (toremoveids[subid] > 0) {
1001 - toremoveids[subid] -= 1;
1002 - return false;
1003 - } else {
1004 - return true;
1005 - }
1006 - })
1007 - }
1229 + }
1230 + if (Object.keys(toremoveids).length > 0) {
1231 + if (op.operator == '.') {
1232 + itemvals = itemvals.filter((subitem) => !(getid(subitem) in toremoveids));
1233 + } else {
1234 + itemvals = itemvals.filter((subitem) => {
1235 + const subid = getid(subitem);
1236 + if (toremoveids[subid] > 0) {
1237 + toremoveids[subid] -= 1;
1238 + return false;
1239 + } else {
1240 + return true;
1241 + }
1242 + })
1008 1243 }
1009 1244 }
1010 - itemvals.forEach((x) => acc.push(x));
1011 - return acc;
1012 - }, []);
1013 - }
1245 + }
1246 + itemvals.forEach((x) => acc.push(x));
1247 + return acc;
1248 + }, []);
1014 1249 outputlist = outputlist.filter((item) => item != null);
1015 1250 break;
1016 1251 case ':': // filter
1017 - if (!op?.args?.length > 0) {
1252 + if (!(op?.args?.length > 0)) {
1018 1253 outputlist = currentlist;
1019 1254 break;
1020 1255 }
1021 - for(const arg of op.args) {
1022 - if (arg.subop && ['>>', '>>=', '<<', '<<=', '~~'].includes(arg.subop.replace(/^-/, ''))) {
1023 - arg.tests = {id: new Set(), name: new Set()};
1024 - if (Array.isArray(arg.value)) {
1025 - const testitems = this.execute(currentlist, arg.value, labeled);
1026 - testitems.forEach((testitem) => arg.tests.id.add(getid(testitem)));
1027 - testitems.forEach((testitem) => arg.tests.name.add(getname(testitem)));
1256 +
1257 + if (op?.args?.length == 1 && op.args[0].subop && op.args[0].label == null && op.args[0].value == null) {
1258 + let checklist;
1259 + if (currentlist.length == 0) {
1260 + outputlist = [];
1261 + break;
1262 + } else if (currentlist.length == 1) {
1263 + checklist = [currentlist[0], currentlist[0]];
1264 + } else {
1265 + checklist = currentlist;
1266 + }
1267 + let comparator = op.args[0].subop;
1268 + let polarize = (x) => x;
1269 + let check;
1270 + if (comparator?.startsWith('-') || comparator?.endsWith('-')) {
1271 + polarize = (x) => !x;
1272 + comparator = comparator.replace(/-|-$/, '');
1273 + }
1274 + for (let pi = 0; pi < checklist.length - 1; pi++) {
1275 + let a = getname(checklist[pi]);
1276 + let b = getname(checklist[pi + 1]);
1277 + switch (comparator) {
1278 + case '':
1279 + check = polarize(a == b);
1280 + break;
1281 + case '=':
1282 + check = polarize(a == b);
1283 + break;
1284 + case '>=':
1285 + check = polarize(a >= b);
1286 + break;
1287 + case '<=':
1288 + check = polarize(a <= b);
1289 + break;
1290 + case '>':
1291 + check = polarize(a > b);
1292 + break;
1293 + case '<':
1294 + check = polarize(a < b);
1295 + break;
1296 + case '~':
1297 + check = polarize(a?.toString().toLowerCase().includes(b?.toString().toLowerCase()));
1298 + break;
1299 + case '~<':
1300 + check = polarize(a?.toString().toLowerCase().startsWith(b?.toString().toLowerCase()));
1301 + break;
1302 + case '~>':
1303 + check = polarize(a?.toString().toLowerCase().endsWith(b?.toString().toLowerCase()));
1304 + break;
1028 1305 }
1029 -
1030 - if (['>>', '>>=', '<<', '<<='].includes(arg.subop)) {
1031 - for (const [tryval, tryf] of [['id', getid], ['name', getname]]) {
1032 - for (let j=0; j < currentlist.length; j++) {
1033 - if (arg.tests[tryval].has(tryf(currentlist[j]))) {
1034 - arg.matchpos = j;
1035 - break;
1036 - }
1037 - }
1038 - if (arg.matchpos) break
1039 - }
1306 + if (!check) {
1307 + outputlist = [];
1308 + break;
1040 1309 }
1041 - } else if ([null, '', '=', '~'].includes((arg.subop || '').replace(/-/, '')) && dtype(arg.value, 'string') && arg.value.startsWith('~')) {
1310 + }
1311 + if (check) outputlist = currentlist;
1312 + break;
1313 + }
1314 +
1315 + for (const arg of op.args) {
1316 + if ([null, '', '=', '~'].includes((arg.subop || '').replace(/-/, '')) && dtype(arg.value, 'string') && arg.value.startsWith('~')) {
1042 1317 const flags = arg.value.startsWith('~~') ? 'i' : '';
1043 1318 const pattern = arg.value.replace(/^~*/, '');
1044 - const fullpattern = (arg.subop || '').replace(/-/, '') === '=' ? ((pattern.startsWith('^') ? '' : '^') + pattern + (pattern.endsWith('$') ? '' : '$')) : pattern;
1319 + const fullpattern = (arg.subop || '').replace(/-/, '') == '=' ? ((pattern.startsWith('^') ? '' : '^') + pattern + (pattern.endsWith('$') ? '' : '$')) : pattern;
1045 1320 arg.re = new RegExp(fullpattern, flags);
1046 1321 }
1047 1322 }
1048 1323
1049 1324 const ands = [[]];
1050 1325 for (const arg of op.args) {
1051 - if (arg.separator === ';') {
1326 + if (arg.separator == ';') {
1052 1327 ands.push([arg]);
1053 1328 } else {
1054 1329 ands[ands.length - 1].push(arg)
@@ -1067,58 +1342,15 @@ export class DACTAL {
1067 1342 }
1068 1343 if (!comparator && arg.label == null && Array.isArray(arg.value)) {
1069 1344 return polarize(this.execute([item], arg.value, labeled).length > 0);
1070 - } else if (comparator === '~~') {
1071 - if (arg.label == null && dtype(arg.value, 'number')) {
1072 - return polarize(currentlist.length === Number(arg.value));
1073 - } else {
1074 - if (arg.label) {
1075 - const argitems = step(item, arg.label, null, labeled);
1076 - return polarize(argitems.find((argitem) => arg.tests.id.has(getid(argitem)) || arg.tests.name.has(getname(argitem))));
1077 - } else {
1078 - return polarize(arg.tests.id.has(getid(item)) || (item?.id === undefined && arg.tests.name.has(getname(item))));
1079 - }
1080 - }
1081 - } else if (['~~', '<<', '<<=', '>>', '>>='].includes(comparator) && dtype(arg.value, 'number')) {
1082 - const argval = Number(arg.value);
1083 - let testlength;
1084 - if (typeof arg.label == 'string') {
1085 - testlength = step(item, arg.label, null, labeled).length;
1086 - } else {
1087 - testlength = currentlist.length;
1088 - }
1089 - switch (comparator) {
1090 - case '~~': return polarize(testlength === argval);
1091 - case '<<': return polarize(testlength < argval);
1092 - case '<<=': return polarize(testlength <= argval);
1093 - case '>>': return polarize(testlength > argval);
1094 - case '>>=': return polarize(testlength >= argval);
1095 - }
1096 1345 } else if (['+', ''].includes(comparator) && arg.label && !arg.value && dtype(arg.label, 'string') && dtype(item, 'object')) {
1097 1346 const propval = item[arg.label];
1098 - // console.log({plusminus: comparator, label: arg.label, propval: propval})
1347 + // console.log({plusminus: comparator, label: arg.label, propval: propval, judgment: polarize(Array.isArray(propval) ? propval.length > 0 : propval)})
1099 1348 return polarize(Array.isArray(propval) ? propval.length > 0 : propval);
1100 1349 }
1101 1350
1102 1351 let testitems = [item];
1103 1352 if (arg.label && dtype(item, 'object')) {
1104 - let found = false;
1105 - for (const tryval of [arg.label, arg.label + 's']) {
1106 - if (item?.[tryval] != null) {
1107 - const propitems = item[tryval];
1108 - if (propitems != null) {
1109 - found = true;
1110 - testitems = Array.isArray(propitems) ? propitems : [propitems];
1111 - break;
1112 - }
1113 - }
1114 - }
1115 - if (!found) {
1116 - if (dactal.savedquerynames.has(arg.label) || (labeled['=>'] && arg.label in labeled['=>'])) {
1117 - testitems = step(item, arg.label, null, labeled);
1118 - } else {
1119 - testitems = [];
1120 - }
1121 - }
1353 + testitems = step(item, arg.label, null, labeled);
1122 1354 }
1123 1355
1124 1356 const testvals = testitems.map((testitem) => {
@@ -1134,8 +1366,8 @@ export class DACTAL {
1134 1366 });
1135 1367
1136 1368 let argvals;
1137 - if (Array.isArray(arg.value)) {
1138 - const argvalitems = this.execute([item], arg.value, labeled);
1369 + if (Array.isArray(arg.value) || arg.value?.startsWith('=')) {
1370 + const argvalitems = this.step(item, arg.value, null, labeled);
1139 1371 argvals = argvalitems.map((argvalitem) => {
1140 1372 let argval;
1141 1373 if (dtype(argvalitem, 'literal')) {
@@ -1144,8 +1376,8 @@ export class DACTAL {
1144 1376 argval = getname(argvalitem);
1145 1377 }
1146 1378 if (argval == null) {
1147 - const argvalitemkeys = Object.keys(argvalitem).filter((key) => key !== 'id');
1148 - if (argvalitemkeys.length === 1) {
1379 + const argvalitemkeys = Object.keys(argvalitem).filter((key) => key != 'id');
1380 + if (argvalitemkeys.length == 1) {
1149 1381 argval = argvalitem[argvalitemkeys[0]];
1150 1382 }
1151 1383 }
@@ -1159,6 +1391,7 @@ export class DACTAL {
1159 1391 testval = Number(testval);
1160 1392 argvals = [Number(arg.value)];
1161 1393 } else {
1394 + if (isNaN(testval) || !isFinite(testval)) testval = testval.toString();
1162 1395 argvals = [arg.value];
1163 1396 }
1164 1397 }
@@ -1176,49 +1409,50 @@ export class DACTAL {
1176 1409 testval = i + 1;
1177 1410 comparator = comparator.slice(1);
1178 1411 }
1179 - } else if (['<<', '<<=', '>>', '>>='].includes(comparator) && dtype(arg.value, 'number')) {
1180 - const argval = Number(arg.value);
1181 - switch (comparator) {
1182 - case '<<': return polarize(currentlist.length < argval);
1183 - case '<<=': return polarize(currentlist.length <= argval);
1184 - case '>>': return polarize(currentlist.length > argval);
1185 - case '>>=': return polarize(currentlist.length >= argval);
1186 - }
1187 1412 }
1188 - if (testval == null || argvals == null || argvals.length === 0) return false;
1413 + if (testval == null || argvals == null || argvals.length == 0) return polarize(false);
1189 1414 return argvals.find((argval) => {
1190 1415 if (arg.re) {
1191 1416 return polarize(testval.match(arg.re));
1192 1417 } else {
1193 1418 switch (comparator) {
1194 - case null: return polarize(testval === argval);
1195 - case '': return polarize(testval === argval);
1196 - case '=': return polarize(testval === argval);
1197 - case '>=': return polarize(testval >= argval);
1198 - case '<=': return polarize(testval <= argval);
1199 - case '>': return polarize(testval > argval);
1200 - case '<': return polarize(testval < argval);
1201 - case '~': return polarize(testval?.toString().toLowerCase().includes(argval?.toString().toLowerCase()));
1202 - case '~<': return polarize(testval?.toString().toLowerCase().startsWith(argval?.toString().toLowerCase()));
1203 - case '~>': return polarize(testval?.toString().toLowerCase().endsWith(argval?.toString().toLowerCase()));
1204 - case '>>': return polarize(i > arg.matchpos);
1205 - case '>>=': return polarize(i >= arg.matchpos);
1206 - case '<<': return polarize(i < arg.matchpos);
1207 - case '<<=': return polarize(i <= arg.matchpos);
1419 + case null:
1420 + return polarize(testval == argval);
1421 + case '':
1422 + return polarize(testval == argval);
1423 + case '=':
1424 + return polarize(testval == argval);
1425 + case '>=':
1426 + return polarize(testval >= argval);
1427 + case '<=':
1428 + return polarize(testval <= argval);
1429 + case '>':
1430 + return polarize(testval > argval);
1431 + case '<':
1432 + return polarize(testval < argval);
1433 + case '~':
1434 + return polarize(testval?.toString().toLowerCase().includes(argval?.toString().toLowerCase()));
1435 + case '~<':
1436 + return polarize(testval?.toString().toLowerCase().startsWith(argval?.toString().toLowerCase()));
1437 + case '~>':
1438 + return polarize(testval?.toString().toLowerCase().endsWith(argval?.toString().toLowerCase()));
1208 1439 }
1209 1440 }
1210 1441 }) != null;
1211 1442 }) != null;
1212 1443 }).length > 0;
1213 - }).length === ands.length;
1444 + }).length == ands.length;
1214 1445 });
1215 1446 break;
1216 1447 case '#': // sort
1217 1448 const sortargs = op.args.slice(0);
1218 1449 const lastarg = sortargs[sortargs.length - 1];
1219 1450 let temped = false;
1220 - if (!lastarg || ![';', '=;'].includes(Object.values(lastarg).join(''))) sortargs.push(...[{subop: null, value: 'name'}, {subop: null, value: 'id'}]);
1221 - if (!dtype(currentlist[0], 'object') && lastarg && lastarg.separator === ';') {
1451 + if (!lastarg || ![';', '=;'].includes(Object.values(lastarg).join(''))) sortargs.push(...[{
1452 + subop: null,
1453 + value: 'name'
1454 + }, {subop: null, value: 'id'}]);
1455 + if (!dtype(currentlist[0], 'object') && lastarg && lastarg.separator == ';') {
1222 1456 currentlist = currentlist.map((v) => ({_value: v}));
1223 1457 temped = true;
1224 1458 } else if (sortargs[0]?.label) {
@@ -1226,19 +1460,19 @@ export class DACTAL {
1226 1460 }
1227 1461 sortargs.forEach((arg) => {
1228 1462 arg.extraindex = {};
1229 - const stablesort = (arg.label == null && arg.value == null && arg.separator === ';' ) ? (arg.subop === '-' ? -1 : 1) : null;
1463 + const stablesort = (arg.label == null && arg.value == null && arg.separator == ';') ? (arg.subop == '-' ? -1 : 1) : null;
1230 1464 if (arg.subop?.includes('~')) {
1231 1465 arg.sortmode = 'literal';
1232 - } else if (arg.subop?.endsWith('-')) {
1466 + } else if (arg.subop?.endsWith('-') || arg.subop?.endsWith('>')) {
1233 1467 arg.sortmode = 'numeric';
1234 - } else if (arg.subop?.endsWith('+')) {
1468 + } else if (arg.subop?.endsWith('+') || arg.subop?.endsWith('<')) {
1235 1469 arg.sortmode = 'rank';
1236 1470 } else {
1237 1471 arg.sortmode = stablesort || ['rank', 'index', 'number', 'id'].includes(arg.value) ? 'rank' : 'numeric';
1238 1472 }
1239 1473 const vals = new Set();
1240 1474 const extradone = new Set();
1241 - for (let ix=0; ix<currentlist.length; ix++) {
1475 + for (let ix = 0; ix < currentlist.length; ix++) {
1242 1476 const item = currentlist[ix];
1243 1477 const vallist = stablesort ? [ix] : (arg.value ? step(item, arg.value, null, labeled) : (Array.isArray(item) ? item : [item]));
1244 1478 if (vallist && !stablesort) vallist.forEach((val) => vals.add(val));
@@ -1249,16 +1483,18 @@ export class DACTAL {
1249 1483 extradone.add(item);
1250 1484 }
1251 1485 }
1252 - if (arg.sortmode !== 'literal') {
1486 + ;
1487 + if (arg.sortmode != 'literal' && !arg.subop?.endsWith('>') && !arg.subop?.endsWith('<')) {
1253 1488 for (const val of vals) {
1254 1489 if (val != null && !dtype(val, 'number') && !(dtype(val, 'object') && dtype(getname(val), 'number'))) {
1255 1490 arg.sortmode = null;
1256 1491 break;
1257 1492 }
1258 1493 }
1494 + ;
1259 1495 }
1260 1496 })
1261 -
1497 +
1262 1498 outputlist = currentlist.sort((a, b) => {
1263 1499 let comp = 0;
1264 1500 let ii = 0;
@@ -1268,70 +1504,77 @@ export class DACTAL {
1268 1504 ii++;
1269 1505 if (alist && blist) {
1270 1506 if (arg?.subop?.includes('@')) {
1271 - const alookup = alist.indexOf(getname(a));
1272 - const blookup = blist.indexOf(getname(b));
1273 - if (alookup >-1 && blookup === -1) {
1507 + const ifunc = arg.subop == '@=' ? getname : getid;
1508 + const alookup = alist.indexOf(ifunc(a));
1509 + const blookup = blist.indexOf(ifunc(b));
1510 + if (alookup > -1 && blookup == -1) {
1274 1511 comp = -1;
1275 - } else if (alookup === -1 && blookup > -1) {
1512 + } else if (alookup == -1 && blookup > -1) {
1276 1513 comp = 1;
1277 1514 } else {
1278 - comp = alist.indexOf(getname(a)) - alist.indexOf(getname(b))
1515 + const aliststrs = alist.map((ax) => ax.toString());
1516 + comp = aliststrs.indexOf(ifunc(a).toString()) - aliststrs.indexOf(ifunc(b).toString());
1279 1517 }
1280 1518 } else {
1281 - for (let i=0; i < Math.min(alist.length, blist.length); i++) {
1519 + for (let i = 0; i < Math.min(alist.length, blist.length); i++) {
1282 1520 let aitem = alist[i];
1283 1521 let bitem = blist[i];
1284 - if (aitem != null && bitem == null) {
1522 + const anull = nullish(aitem);
1523 + const bnull = nullish(bitem);
1524 + if (anull && bnull) {
1525 + comp = 0;
1526 + } else if (bnull) {
1285 1527 comp = -1;
1286 - } else if (aitem == null && bitem != null) {
1528 + } else if (anull && !bnull) {
1287 1529 comp = 1;
1288 1530 } else {
1289 1531 switch (arg.sortmode) {
1290 1532 case 'literal':
1291 - comp = aitem < bitem ? -1 : (bitem < aitem ? 1 : 0);
1533 + const atrim = typeof aitem == 'string' ? aitem.trim() : aitem;
1534 + const btrim = typeof bitem == 'string' ? bitem.trim() : bitem;
1535 + comp = atrim < btrim ? -1 : (btrim < atrim ? 1 : 0);
1292 1536 break;
1293 1537 case 'numeric':
1538 + case 'rank':
1539 + const polarity = (arg.sortmode == 'numeric' ? -1 : 1);
1294 1540 let bnum = Number(bitem);
1295 1541 let anum = Number(aitem);
1296 1542 if (isNaN(bnum) || isNaN(anum)) {
1297 1543 bnum = Number(getname(bitem));
1298 1544 anum = Number(getname(aitem));
1299 1545 }
1300 - if (isNaN(bnum) || isNaN(anum)) {
1301 - comp = compvals(aitem, bitem)
1546 + if (isNaN(bnum) && isNaN(anum)) {
1547 + comp = polarity * compvals(aitem, bitem);
1548 + } else if (isNaN(anum)) {
1549 + comp = 1;
1550 + } else if (isNaN(bnum)) {
1551 + comp = -1;
1302 1552 } else {
1303 - comp = bnum - anum;
1553 + comp = polarity * (anum - bnum);
1304 1554 }
1305 1555 break;
1306 - case 'rank':
1307 - comp = Number(aitem) - Number(bitem);
1308 - break;
1309 1556 default:
1310 1557 if (dtype(aitem, 'literal') && dtype(bitem, 'literal')) {
1311 1558 comp = compvals(aitem, bitem);
1312 1559 } else if (typeof aitem == 'object' && typeof bitem == 'object') {
1313 1560 let aval = getname(aitem);
1314 1561 let bval = getname(bitem);
1315 - if (aval != null && bval != null) {
1562 + if (!nullish(aval) && !nullish(bval)) {
1316 1563 comp = compvals(aval, bval);
1317 1564 } else {
1318 1565 aval = getid(aitem);
1319 1566 bval = getid(bitem);
1320 - if (aval != null && bval != null) {
1567 + if (!nullish(aval) && !nullish(bval)) {
1321 1568 comp = compvals(aval, bval);
1322 1569 } else {
1323 1570 comp = 0;
1324 1571 }
1325 1572 }
1326 - } else if (aitem != null && bitem == null) {
1327 - comp = -1;
1328 - } else if (aitem == null && bitem != null) {
1329 - comp = 1;
1330 1573 }
1331 1574 break;
1332 1575 }
1333 1576 }
1334 - if (comp !== 0) break;
1577 + if (comp != 0) break;
1335 1578 }
1336 1579 }
1337 1580 }
@@ -1344,10 +1587,10 @@ export class DACTAL {
1344 1587 comp = 1;
1345 1588 }
1346 1589 }
1347 - if (arg?.subop?.includes('-') && (arg.sortmode === 'literal' || !arg.sortmode)) {
1590 + if (arg?.subop?.includes('-') && (arg.sortmode == 'literal' || !arg.sortmode)) {
1348 1591 comp = -comp;
1349 1592 }
1350 - if (comp !== 0) break;
1593 + if (comp != 0) break;
1351 1594 }
1352 1595 return comp;
1353 1596 })
@@ -1359,18 +1602,24 @@ export class DACTAL {
1359 1602 break;
1360 1603 case '/': // group
1361 1604 case '//': // merge
1362 - const groupindex = {};
1605 + const groupindex = new Map();
1363 1606 let keylists = {};
1364 1607 let ofname = 'of';
1365 1608 let countname = 'count';
1366 1609 let sortgroups = true;
1367 1610 const groupargs = [];
1611 + const accumulates = {};
1612 + const discards = new Set();
1368 1613 const merge_ands = [];
1369 1614 for (const arg of op.args) {
1370 - if (arg.separator === ';' && arg.label == null && arg.value == null) {
1615 + if (op.operator == '//' && arg.subop?.includes('+') && typeof arg.value == 'string') {
1616 + accumulates[arg.value] = arg.label || arg.value;
1617 + } else if (op.operator == '//' && arg.subop?.includes('-') && typeof arg.value == 'string') {
1618 + discards.add(arg.value);
1619 + } else if (arg.separator == ';' && arg.label == null && arg.value == null) {
1371 1620 sortgroups = false;
1372 - } else if (op.operator === '//' && (arg.separator === ';' || merge_ands.length > 0)) {
1373 - if (arg.separator === ';') {
1621 + } else if (op.operator == '//' && (arg.separator == ';' || merge_ands.length > 0)) {
1622 + if (arg.separator == ';') {
1374 1623 merge_ands.push([arg.value]);
1375 1624 } else {
1376 1625 merge_ands[merge_ands.length - 1].push(arg.value)
@@ -1379,53 +1628,59 @@ export class DACTAL {
1379 1628 groupargs.push(arg);
1380 1629 }
1381 1630 }
1382 - if (groupargs.length === 0) groupargs.push({value: null});
1383 -
1631 + if (groupargs.length == 0) groupargs.push({value: null});
1632 +
1384 1633 groupargs.forEach((arg) => arg.groupcounter = 0);
1385 -
1634 +
1386 1635 const grouptimer = {};
1387 - for (let ix=0; ix<currentlist.length; ix++) {
1636 + for (let ix = 0; ix < currentlist.length; ix++) {
1388 1637 this.timecheck(grouptimer, ix, currentlist.length, op);
1389 1638 const item = currentlist[ix];
1390 1639 let keys = null;
1391 1640 let keyi = 0;
1641 + let itemsleft = currentlist.length - ix;
1392 1642 for (const arg of groupargs) {
1393 - if (arg.label === 'of') {
1643 + if (arg.label == 'of') {
1394 1644 ofname = arg.value;
1395 1645 continue;
1396 - } else if (arg.label === 'count') {
1646 + } else if (arg.label == 'count') {
1397 1647 countname = arg.value;
1398 1648 continue;
1399 1649 }
1400 1650 keyi++;
1401 1651 let groupnumber = null;
1402 - if (op.operator === '/' && dtype(arg.value, 'number')) {
1652 + if (op.operator == '/' && dtype(arg.value, 'number')) {
1403 1653 if (arg.subop?.endsWith('@')) {
1404 1654 arg.divisor = Number(arg.value);
1405 1655 } else {
1406 - arg.divisor = currentlist.length / Number(arg.value);
1656 + arg.divisor = currentlist.length / Number(arg.value);
1407 1657 }
1408 1658 groupnumber = Math.floor(ix / arg.divisor) + 1;
1409 1659 } else if (arg.value != null && arg.subop?.endsWith('@@')) {
1410 1660 const groupval = step(item, arg.value, null, labeled);
1411 - if (ix === 0 || groupval?.length > 0) arg.groupcounter += 1;
1661 + if (ix == 0 || groupval?.length > 0) arg.groupcounter += 1;
1412 1662 groupnumber = arg.groupcounter;
1413 1663 }
1414 1664 const label = arg.label ?? (typeof arg.value === 'string' ? arg.value : null) ?? keyi;
1415 1665 const newkeyitems = groupnumber != null ? [groupnumber] : arg.value ? step(item, arg.value, arg.subop, labeled) : [null];
1416 - const newkeys = newkeyitems.map((newkey) => ([{arglabel: arg.label, label: label, keyitem: this.resolve(arg.value, newkey, labeled)}]));
1666 + const newkeys = newkeyitems.map((newkey) => ([{
1667 + arglabel: arg.label,
1668 + label: label,
1669 + keyitem: this.resolve(arg.value, newkey, labeled)
1670 + }]));
1671 + if (arg.subop?.includes('~') && Array.isArray(arg.value)) newkeys.forEach((newkey) => newkey[0].keyitem = getname(newkey[0].keyitem));
1417 1672 if (keys) {
1418 - keys = keys.flatMap((oldkeys) => newkeys.filter((newkey) => arg.separator === ',' || oldkeys.filter((oldkey) => compvals(getname(oldkey.keyitem) || oldkey.keyitem, getname(newkey[0].keyitem) || newkey[0].keyitem) >= 0).length === 0).map((newkey) => oldkeys.concat(newkey)));
1673 + keys = keys.flatMap((oldkeys) => newkeys.filter((newkey) => arg.separator == ',' || oldkeys.filter((oldkey) => compvals(getname(oldkey.keyitem) || oldkey.keyitem, getname(newkey[0].keyitem) || newkey[0].keyitem) >= 0).length === 0).map((newkey) => oldkeys.concat(newkey)));
1419 1674 } else {
1420 1675 keys = newkeys;
1421 1676 }
1422 - if (arg.subop?.endsWith('@')) {
1677 + if (arg.subop?.endsWith('@') || dtype(arg.value, 'number')) {
1423 1678 keys.forEach((key) => {
1424 1679 const testkey = JSON.stringify(key.slice(0, -1));
1425 1680 const newkeytest = JSON.stringify(key[key.length - 1]);
1426 1681 if (testkey in keylists) {
1427 1682 const lastkey = keylists[testkey][keylists[testkey].length - 1];
1428 - if (newkeytest !== lastkey) keylists[testkey].push(newkeytest);
1683 + if (newkeytest != lastkey) keylists[testkey].push(newkeytest);
1429 1684 } else {
1430 1685 keylists[testkey] = [newkeytest];
1431 1686 }
@@ -1436,30 +1691,35 @@ export class DACTAL {
1436 1691 if (keys) {
1437 1692 for (const key of keys) {
1438 1693 const keystr = JSON.stringify(key);
1439 - (groupindex[keystr] ||= []).push(item);
1694 + if (!groupindex.has(keystr)) groupindex.set(keystr, []);
1695 + groupindex.get(keystr).push(item);
1440 1696 }
1441 1697 }
1442 1698 }
1443 - for (const [keystr, items] of Object.entries(groupindex)) {
1444 - if (op.operator === '/') {
1699 + for (const [keystr, items] of groupindex) {
1700 + if (op.operator == '/') {
1445 1701 const newgroup = {};
1446 1702 const keydata = JSON.parse(keystr);
1447 - if (keydata.length === 1 && keydata[0].keyitem != null) {
1448 - let keyitemname = keydata[0].keyindex || getname(keydata[0].keyitem);
1449 - if (keyitemname != null) {
1450 - newgroup.name = keyitemname;
1703 + if (keydata.length == 1 && keydata[0].keyitem != null) {
1704 + if (keydata[0].keyindex) {
1705 + newgroup.keyindex = keydata[0].keyindex;
1706 + } else {
1707 + let keyitemname = getname(keydata[0].keyitem);
1708 + if (keyitemname != null) {
1709 + newgroup.name = keyitemname;
1710 + }
1451 1711 }
1452 1712 }
1453 1713 let skip = false;
1454 1714 let keys = [];
1455 1715 for (const {arglabel, label, keyitem, keyindex} of keydata) {
1456 1716 if (keyitem != null) {
1457 - const keyobj = (dtype(keyitem, 'object') || keyindex == null || false) ? keyitem : {name: keyitem};
1717 + const keyobj = (dtype(keyitem, 'object') || keyindex == null || keyindex == undefined) ? keyitem : (keyindex != null && keyindex != undefined) ? {} : {name: keyitem};
1458 1718 if (keyindex != null) {
1459 - keyobj.index = keyindex;
1719 + keyobj.keyindex = keyindex;
1460 1720 }
1461 1721 keys.push(keyobj)
1462 - if (label && dtype(label, 'string') && isNaN(label) && label !== '_') {
1722 + if (label && dtype(label, 'string') && isNaN(label) && label != '_') {
1463 1723 newgroup[label] = [keyobj];
1464 1724 }
1465 1725 }
@@ -1468,23 +1728,33 @@ export class DACTAL {
1468 1728 if (keys) newgroup.key = keys;
1469 1729 newgroup[ofname] = items;
1470 1730 outputlist.push(newgroup);
1471 - } else if (op.operator === '//') {
1472 - if (merge_ands.length === 0 || !merge_ands.find((mand) => !mand.find((mr) => items.find((item) => mr in item)))) {
1731 + } else if (op.operator == '//') {
1732 + const groupprops = groupargs.map((grouparg) => grouparg.value).filter((gp) => gp);
1733 + if (merge_ands.length == 0 || !merge_ands.find((mand) => !mand.find((mr) => items.find((item) => mr in item)))) {
1473 1734 const newgroup = items.reduce((acc, item) => {
1474 - for(const prop in item) {
1475 - if (prop in acc) {
1476 - if (Array.isArray(acc[prop])) {
1477 - (Array.isArray(item[prop]) ? item[prop] : [item[prop]]).filter((val) => !acc[prop].includes(val)).forEach((val) => acc[prop].push(val));
1735 + const proporder = groupprops.slice(0);
1736 + Object.keys(item).forEach((prop) => {
1737 + if (!proporder.includes(prop) && !discards.has(prop)) proporder.push(prop);
1738 + });
1739 + for (const prop of proporder) {
1740 + const writeprop = accumulates[prop] || prop;
1741 + if (writeprop in acc) {
1742 + if (Array.isArray(acc[writeprop])) {
1743 + (Array.isArray(item[prop]) ? item[prop] : [item[prop]]).filter((val) => !acc[writeprop].includes(val)).forEach((val) => acc[writeprop].push(val));
1478 1744 }
1479 1745 } else {
1480 - acc[prop] = item[prop];
1746 + if (prop in accumulates && !Array.isArray(item[prop])) {
1747 + acc[writeprop] = [item[prop]];
1748 + } else {
1749 + acc[writeprop] = item[prop];
1750 + }
1481 1751 }
1482 1752 }
1483 1753 return acc;
1484 1754 }, {});
1485 1755 const keydata = JSON.parse(keystr);
1486 1756 for (const {arglabel, label, keyitem, keyindex} of keydata) {
1487 - if (arglabel && typeof arglabel === 'string' && arglabel !== '_') {
1757 + if (arglabel && typeof arglabel === 'string' && arglabel != '_') {
1488 1758 newgroup[arglabel] = [keyitem];
1489 1759 }
1490 1760 }
@@ -1493,109 +1763,195 @@ export class DACTAL {
1493 1763 }
1494 1764 }
1495 1765 if (sortgroups) {
1496 - const sortquery = '#' + groupargs.map((arg, argx) => (arg.subop?.endsWith('@') || arg.divisor ? '+' : '') + '(..key:@' + (argx + 1) + (arg.subop?.endsWith('@') ? '.index;_' : '') + ')').join(',');
1766 + const sortquery = '#' + groupargs.map((arg, argx) => (arg.subop?.endsWith('@') || arg.divisor ? '+' : '') + '(..key:@' + (argx + 1) + (arg.subop?.endsWith('@') ? '.keyindex;_' : '') + ')').join(',');
1497 1767 outputlist = this.execute(outputlist, this.assemble(this.tokenize(sortquery)), labeled);
1498 1768 }
1499 1769 break;
1500 - case '...': // aggregate
1501 - case '....': // aggregate to value
1770 + case '...': // synthesize
1771 + case '....': // synthesize and extract
1502 1772 if (!(op?.args?.length > 0)) {
1503 - if (op.operator === '...') {
1773 + if (op.operator == '...') {
1504 1774 outputlist = [{of: currentlist.map((item) => dcopy(item))}];
1505 1775 } else {
1506 1776 outputlist = [currentlist.length]
1507 1777 }
1508 1778 break;
1509 1779 }
1510 - const tempitem = {};
1511 - if (op.operator === '...') {
1512 - tempitem.of = currentlist.map((item) => dcopy(item));
1513 - } else {
1514 - tempitem.of = currentlist;
1780 + const segments = [[]];
1781 + op.args.forEach((arg) => {
1782 + if (arg.separator == ';') segments.push([]);
1783 + segments[segments.length - 1].push(arg);
1784 + });
1785 + outputlist = [];
1786 + let labels = null;
1787 + let masterof = op.operator == '...';
1788 + if (
1789 + segments.length > 1 &&
1790 + segments[0].filter((s) => s.subop && s.value == null).length == segments[0].length &&
1791 + segments.slice(1).filter((s) => s.length == segments[0].length || (segments[0][segments[0].length - 1].label == null && s.length == segments[0].length - 1)).length == segments.length - 1
1792 + ) {
1793 + labels = segments.shift();
1794 + const lastlabel = labels[labels.length - 1];
1795 + if (lastlabel.subop == '~' && lastlabel.label == null) {
1796 + masterof = false;
1797 + labels.pop();
1798 + }
1515 1799 }
1516 - const finalitem = {};
1517 - let aggregated = null;
1518 - let afteraggregated = 0;
1519 - let finalvalue = null;
1520 - op.args.forEach((arg, propx) => {
1521 - if (dtype(arg.value, 'string') && arg.subop === '~') {
1522 - tempitem[arg.label || '_' + (propx + 1).toString()] = [arg.value];
1523 - } else {
1524 - const prop = arg.label ?? (typeof arg.value === 'string' ? arg.value : null) ?? '_' + (propx + 1).toString();
1525 - const aggname = (arg.label == null && typeof arg.value == 'string' && arg.value) || (arg.value == null && arg.label);
1526 - const firstitem = currentlist[0];
1527 - const isprop = typeof firstitem == 'object' && prop in firstitem;
1528 - if ((arg.separator === ';' || !isprop) && aggname in this.aggregators) {
1529 - const aggval = this.aggregators[aggname](tempitem);
1530 - const agglabel = arg.label || aggname;
1531 - if (aggval != null) finalitem[agglabel] = aggval;
1532 - aggregated = agglabel;
1533 - finalvalue = dcopy(aggval);
1534 - } else if (!aggregated) {
1535 - let propitems;
1536 - if (op === operations[0] && op.operator === '....' && arg.label && !Array.isArray(arg.value) && !arg.value.startsWith('=')) {
1537 - propitems = [arg.value]
1538 - } else {
1539 - propitems = this.execute(currentlist, Array.isArray(arg.value) ? arg.value.map((x) => structuredClone(x)) : [{operator: '..', args: [{value: arg.value}]}], labeled)
1800 + for (const segment of segments) {
1801 + const tempitem = {of: currentlist.map((item) => dcopy(item))};
1802 + const finalitem = {};
1803 + const postitem = {};
1804 + let aggregated = null;
1805 + let afteraggregated = 0;
1806 + let finalvalue = null;
1807 + let includeof = masterof;
1808 + segment.forEach((arg, propx) => {
1809 + if (labels?.[propx]) {
1810 + arg.label = labels[propx].label;
1811 + arg.subop = labels[propx].subop;
1812 + }
1813 + if (arg.subop == '~' && arg.label == null && arg.value == null) {
1814 + includeof = false
1815 + } else if (!aggregated && arg.subop?.includes('~') && dtype(arg.value, 'string')) {
1816 + tempitem[arg.label || '_' + (propx + 1).toString()] = arg.value;
1817 + } else if (!aggregated && arg.subop?.includes('~') && Array.isArray(arg.value)) {
1818 + const subresult = this.execute(currentlist, arg.value.map(opclone), labeled);
1819 + tempitem[arg.label || '_' + (propx + 1).toString()] = subresult.length > 0 ? getname(subresult[0]) : null;
1820 + } else {
1821 + const prop = arg.label || (typeof arg.value === 'string' ? arg.value : null) || '_' + (propx + 1).toString();
1822 + const aggname = (arg.label == null && typeof arg.value == 'string' && arg.value) || (arg.label && arg.subop == null && arg.value == null);
1823 + const firstitem = currentlist[0];
1824 + const isprop = typeof firstitem == 'object' && prop in firstitem;
1825 + if ((arg.subop == '=' || !isprop) && arg.subop != '~' && aggname in this.annotators) {
1826 + const aggval = this.annotators[aggname](tempitem);
1827 + const agglabel = arg.label || aggname;
1828 + if (aggval != null) finalitem[agglabel] = aggval;
1829 + aggregated = agglabel;
1830 + finalvalue = dcopy(aggval);
1831 + } else if (typeof arg.value == 'string' && arg.value.startsWith('~~')) {
1832 + let template = arg.value.slice(2);
1833 + const extractionfield = arg.label || template;
1834 + const targetfields = this.kkeys(tempitem);
1835 + const fieldmap = {};
1836 + template = this.escapeRegExp(template);
1837 + let tx = 1;
1838 + targetfields.forEach((tf) => {
1839 + const fieldwidth = this.getnumber(tempitem[tf]);
1840 + const innerpattern = fieldwidth == 1 ? '.' : fieldwidth ? `.{${fieldwidth}}` : '.+?';
1841 + if (tf == '_') {
1842 + template = template.replaceAll('_', `(?:${innerpattern})`);
1843 + } else {
1844 + const tkey = 'x' + tx;
1845 + fieldmap[tf] = tkey;
1846 + template = template.replace(this.escapeRegExp(tf), `(?<${tkey}>${innerpattern})`);
1847 + tx += 1;
1848 + }
1849 + });
1850 + template = `^${template}$`;
1851 + const extractor = new RegExp(template);
1852 + finalitem[extractionfield] = [];
1853 + finalitem['_template'] = template;
1854 + finalitem['_field map'] = Object.entries(fieldmap).map(([k, v]) => ({
1855 + field: k,
1856 + code: v
1857 + }));
1858 + tempitem.of.forEach((i) => {
1859 + const extraction = getname(i).match(extractor);
1860 + if (extraction) {
1861 + const extractitem = {};
1862 + targetfields.forEach((k) => {
1863 + extractitem[k.trim()] = extraction.groups[fieldmap[k]];
1864 + });
1865 + finalitem[extractionfield].push(extractitem);
1866 + }
1867 + });
1868 + finalvalue = finalitem[extractionfield];
1869 + aggregated = template;
1870 + } else if (!aggregated) {
1871 + let propitems;
1872 + if (Array.isArray(arg.value) || !arg.label || arg.value?.startsWith('~')) {
1873 + const propres = this.execute(tempitem.of, Array.isArray(arg.value) ? arg.value.map(opclone) : [{
1874 + operator: '..',
1875 + args: [{value: arg.value}]
1876 + }], labeled);
1877 + if (arg.subop?.includes('~') && propres.length > 0) {
1878 + propitems = getname(propres[0]);
1879 + } else {
1880 + propitems = propres;
1881 + }
1882 + } else if (arg.value?.startsWith('=')) {
1883 + const mathres = this.step(tempitem.of, arg.value, arg.subop, labeled);
1884 + if (mathres.length > 0) propitems = mathres[0];
1885 + } else {
1886 + if (arg.subop?.includes('~')) {
1887 + propitems = arg.value;
1888 + } else {
1889 + propitems = [arg.value]
1890 + }
1891 + }
1892 + tempitem[prop] = propitems;
1893 + } else if (aggregated in finalitem && Array.isArray(finalitem[aggregated]) && finalitem[aggregated].length > afteraggregated) {
1894 + const propval = finalitem[aggregated][afteraggregated];
1895 + const mappedval = arg.subop?.endsWith('~') ? propval : [propval];
1896 + tempitem[prop] = mappedval;
1897 + postitem[prop] = mappedval;
1898 + afteraggregated++;
1540 1899 }
1541 - tempitem[prop] = propitems;
1542 - } else if (aggregated in finalitem && Array.isArray(finalitem[aggregated]) && finalitem[aggregated].length > afteraggregated) {
1543 - tempitem[prop] = finalitem[aggregated][afteraggregated];
1544 - afteraggregated++;
1545 1900 }
1546 - }
1547 - });
1548 - if (op.operator === '....' && finalvalue != null) {
1549 - if (this.dtype(finalvalue, 'array')) {
1550 - outputlist = finalvalue;
1551 - } else {
1552 - outputlist = [finalvalue];
1553 - }
1554 - } else {
1555 - for (const prop in tempitem) {
1556 - if (!(prop in finalitem) && prop !== 'of') finalitem[prop] = tempitem[prop];
1557 - }
1558 - if (op.operator === '...') {
1559 - finalitem.of = tempitem.of;
1901 + });
1902 + if (op.operator == '....' && afteraggregated > 0) {
1903 + outputlist.push(postitem);
1904 + } else if (op.operator == '....' && finalvalue != null) {
1905 + if (dtype(finalvalue, 'array')) {
1906 + outputlist = outputlist.concat(finalvalue);
1907 + } else {
1908 + outputlist.push(finalvalue);
1909 + }
1560 1910 } else {
1561 - for (const prop in finalitem) {
1562 - if (Array.isArray(finalitem[prop]) && finalitem[prop].length === 1) finalitem[prop] = finalitem[prop][0];
1911 + for (const prop in tempitem) {
1912 + if (!(prop in finalitem) && prop != 'of') finalitem[prop] = tempitem[prop];
1563 1913 }
1914 + if (includeof) finalitem.of = tempitem.of;
1915 + outputlist.push(finalitem);
1564 1916 }
1565 - outputlist = [finalitem];
1566 1917 }
1567 1918 break;
1568 1919 case '|': // annotate
1569 - case '||': // annotate with values
1570 1920 outputlist = currentlist.slice(0).map((item) => dcopy(item));
1571 - op.args.filter((arg) => arg.subop?.includes('>') && arg.label != null && true && Array.isArray(arg.value))
1921 + op.args.filter((arg) => arg.subop?.includes('>') && arg.label != null && arg.label != undefined && Array.isArray(arg.value))
1572 1922 .forEach((arg) => (labeled['=>'] ??= {})[arg.label] = arg.value);
1573 -
1923 +
1574 1924 const annotatetimer = {};
1575 - for (const i in outputlist) {
1925 + op.args.filter((arg) => arg.subop?.endsWith('@')).forEach((arg) => arg.counter = undefined);
1926 + for (let i = 0; i < outputlist.length; i++) {
1576 1927 this.timecheck(annotatetimer, i, outputlist.length, op);
1577 1928 const baseitem = outputlist[i];
1578 1929 if (typeof baseitem != 'object') {
1579 1930 outputlist[i] = {};
1580 - if (op.args?.[0]?.value !== '_') outputlist[i].name = baseitem;
1931 + if (op.args?.[0]?.value != '_') outputlist[i].name = baseitem;
1581 1932 }
1582 1933 const item = outputlist[i];
1583 - const newprops = op.args.filter((arg) => arg.subop !== '<' && (arg.label || !arg.subop?.includes('-'))).map((arg) => arg.label || arg.value);
1934 + const newprops = op.args.filter((arg) => arg.subop != '<' && (arg.label || !arg.subop?.includes('-'))).map((arg) => arg.label || arg.value);
1584 1935 if (typeof item == 'object') {
1585 1936 let argx = -1;
1586 1937 for (const arg of op.args) {
1587 1938 argx++;
1588 - if (arg?.subop === '<' && dtype(arg.value, 'string') && Array.isArray(item[arg.value]) && argx < op.args.length - 1) {
1589 - const sublabeled = {...labeled};
1590 - if (arg.label) sublabeled[arg.label] = [item];
1591 - item[arg.value] = dactal.execute(item[arg.value], [{operator: op.operator, args: op.args.slice(argx + 1).map((arg) => structuredClone(arg))}], sublabeled);
1939 + if (arg?.subop == '<' && dtype(arg.value, 'string') && argx < op.args.length - 1) {
1940 + if (arg.value in item && (Array.isArray(item[arg.value]) || dtype(item[arg.value], 'object'))) {
1941 + const sublabeled = {...labeled};
1942 + if (arg.label) sublabeled[arg.label] = [item];
1943 + item[arg.value] = this.execute(item[arg.value], [{
1944 + operator: op.operator,
1945 + args: op.args.slice(argx + 1).map(opclone)
1946 + }], sublabeled);
1947 + }
1592 1948 break;
1593 - } else if (arg.separator === ';' && argx === op.args.length - 1 && arg.label == null && arg.value == null && arg.subop == null) {
1949 + } else if (arg.separator == ';' && argx == op.args.length - 1 && arg.label == null && arg.value == null && (arg.subop == null || arg.subop == '~')) {
1594 1950 for (const oldprop in item) {
1595 1951 if (!(newprops.includes(oldprop))) {
1596 1952 const tempval = item[oldprop];
1597 1953 delete item[oldprop];
1598 - item[oldprop] = tempval;
1954 + if (arg.subop != '~') item[oldprop] = tempval;
1599 1955 }
1600 1956 }
1601 1957 } else if (arg.subop?.endsWith('-')) {
@@ -1604,16 +1960,22 @@ export class DACTAL {
1604 1960 delete item[arg.value];
1605 1961 } else if (arg.label && Array.isArray(arg.value)) {
1606 1962 const toremoveids = new Set(step(item, arg.value, null, labeled).map((subitem) => getid(subitem)));
1607 - item[arg.label] = item[arg.label].filter((subitem) => !toremoveids.has(getid(subitem)));
1963 + item[arg.label] = item[arg.label].filter((subitem) => !toremoveids.has(getid(subitem)));
1608 1964 }
1609 - } else if (argx === 0 && arg.label != null && arg.value === '_') {
1610 - item[arg.label] = op.operator === '||' ? baseitem : [baseitem];
1965 + } else if (argx == 0 && arg.label != null && arg.value == '_') {
1966 + item[arg.label] = [baseitem];
1611 1967 } else {
1612 - if (!arg.label && arg.value && typeof arg.value === 'string' && arg.value !== '') {
1613 - const moveprop = arg.value;
1614 - const value = structuredClone(item[moveprop]);
1615 - delete item[moveprop];
1616 - item[moveprop] = value;
1968 + if (!arg.label && arg.value && typeof arg.value === 'string' && arg.value != '') {
1969 + if (arg.subop == '=' && arg.value in this.annotators) {
1970 + item[arg.value] = this.annotators[arg.value](item);
1971 + } else if (arg.value in item) {
1972 + const moveprop = arg.value;
1973 + const value = item[moveprop];
1974 + delete item[moveprop];
1975 + item[moveprop] = value;
1976 + } else if (arg.value in this.annotators) {
1977 + item[arg.value] = this.annotators[arg.value](item);
1978 + }
1617 1979 }
1618 1980 if (arg.label) {
1619 1981 let vals = [];
@@ -1634,20 +1996,33 @@ export class DACTAL {
1634 1996 if (arg.subop.endsWith('@@')) {
1635 1997 vals = [arg.counter];
1636 1998 }
1999 + } else if (arg.subop == '=' && typeof arg.value == 'string' && arg.value in this.annotators) {
2000 + vals = [this.annotators[arg.value](item)];
1637 2001 } else {
1638 2002 vals = step(item, arg.value, arg.subop, labeled);
1639 2003 }
2004 + const scalar = arg.subop?.includes('~') || arg.subop?.endsWith('@') || (arg.label != '_' && arg.subop == '=' && typeof arg.value == 'string' && (arg.value in this.annotators || arg.value.startsWith('=') || arg.value.startsWith('~')));
1640 2005 const base = (arg.subop?.endsWith('+') && arg.label in item && Array.isArray(item[arg.label])) ? item[arg.label] : [];
1641 - if (this.dtype(vals, 'array')) {
1642 - if (op.operator === '|') {
1643 - item[arg.label] = base.concat(vals.map((v) => dcopy(v)));
1644 - } else if (vals.length > 0 ) {
1645 - if (typeof vals[0] === 'object') {
1646 - item[arg.label] = getname(vals[0]) != null ? getname(vals[0]) : vals[0].id != null ? vals[0].id : dcopy(vals[0]);
1647 - } else {
1648 - item[arg.label] = vals[0];
2006 + if (arg.label == '_') {
2007 + const sourceitem = dtype(vals, 'array') ? vals[0] : val;
2008 + if (dtype(sourceitem, 'object')) {
2009 + for (const prop in sourceitem) {
2010 + if (!(prop in item)) {
2011 + if (scalar && Array.isArray(sourceitem[prop])) {
2012 + item[prop] = sourceitem[prop][0];
2013 + } else {
2014 + item[prop] = sourceitem[prop];
2015 + }
2016 + newprops.push(prop);
2017 + }
1649 2018 }
1650 2019 }
2020 + } else if (dtype(vals, 'array')) {
2021 + if (scalar && vals.length > 0) {
2022 + item[arg.label] = getname(vals[0]);
2023 + } else {
2024 + item[arg.label] = base.concat(vals.map((v) => dcopy(v)));
2025 + }
1651 2026 } else if (vals) {
1652 2027 item[arg.label] = base.concat(vals);
1653 2028 }
@@ -1660,90 +2035,106 @@ export class DACTAL {
1660 2035 case '???':
1661 2036 outputlist = currentlist;
1662 2037 const commentval = op.args?.[0]?.value;
1663 - if (commentval in labeled) console.log({[commentval]: labeled[commentval]});
1664 - if (commentval === 'end') return (this.debug && !inputlist) ? operations : outputlist;
2038 + if (commentval in labeled) console.log({[commentval]: labeled[commentval].slice(0)});
2039 + if (commentval?.startsWith('end')) {
2040 + if (this.debug && !inputlist) {
2041 + return operations;
2042 + } else if (commentval.length > 3 && !isNaN(commentval.slice(3))) {
2043 + return outputlist.slice(0, Number(commentval.slice(3)));
2044 + } else {
2045 + return outputlist;
2046 + }
2047 + }
2048 + if (commentval == 'recache') this.recache = new Set();
2049 + if (typeof currentlist[0] == 'object' && op.args?.[0]?.value in currentlist[0] && !op.args?.[0]?.label) console.log({[op.args?.[0]?.value]: currentlist[0][op.args?.[0]?.value]});
1665 2050 break;
1666 2051 default:
1667 2052 outputlist = []
1668 2053 }
1669 2054 currentlist = outputlist.slice(0);
1670 - if (this.debug && !inputlist) op.results = outputlist.slice(0, typeof this.debug == 'number' ? this.debug : outputlist.length);
2055 + if (toplevel) {
2056 + if (!this.adaptive) {
2057 + op.completed = true;
2058 + operations[0].progress = currentlist;
2059 + operations[0].labeled = labeled;
2060 + }
2061 + if (this.debug) {
2062 + op.results = outputlist.slice(0, typeof this.debug == 'number' ? this.debug : outputlist.length);
2063 + op.time = (performance.now() - opstart) / 1000;
2064 + }
2065 + }
1671 2066 }
1672 - return (this.debug && !inputlist) ? operations : outputlist;
2067 + return (this.debug && !inputlistraw) ? operations : outputlist;
1673 2068 }
1674 2069
1675 - gettype(value, allow_relative=true) {
1676 - if (value == null) return null;
2070 + gettype(value) {
2071 + if (value == null || value == undefined) return null;
1677 2072 const trylist = [value];
1678 - if (value.endsWith('s')) {
1679 - trylist.push(value.slice(0, value.length - 1));
1680 - trylist.push(value + 'es');
1681 - } else {
1682 - trylist.push(value + 's');
2073 + if (this.features.plurality) {
2074 + if (value.endsWith('s')) {
2075 + trylist.push(value.slice(0, value.length - 1));
2076 + trylist.push(value + 'es');
2077 + } else {
2078 + trylist.push(value + 's');
2079 + }
1683 2080 }
1684 2081 for (const tryval of trylist) {
1685 2082 if (tryval in this.data) {
1686 2083 return this.data[tryval];
2084 + } else if (this.internal_datasets.includes(tryval)) {
2085 + return [];
1687 2086 }
1688 2087 }
1689 2088 if (this.savedquerynames.has(value)) {
1690 - const savedqueries = this.data.queries.filter((q) => q.name === value);
1691 - if (savedqueries?.length === 1) {
2089 + const savedqueries = this.data.queries.filter((q) => q.name == value && !q.relative);
2090 + if (savedqueries?.length == 1) {
1692 2091 const sq = savedqueries[0];
1693 - if (allow_relative && sq.query.startsWith('?')) {
1694 - return sq;
1695 - } else {
1696 - if (!sq.results) sq.results = this.executeq(sq.query);
1697 - return sq.results.slice(0);
1698 - }
2092 + if (!sq.results) sq.results = this.executeq(sq.query);
2093 + return sq.results.slice(0);
1699 2094 }
1700 2095 }
1701 2096 return null;
1702 2097 }
1703 2098
1704 2099 dcopy = (item) => {
1705 - if (this.dtype(item, 'literal')) return item;
1706 - if (this.dtype(item, 'array')) return item.slice(0);
1707 - return Object.assign({}, item);
2100 + if (item === null || typeof item !== 'object') return item;
2101 + if (Array.isArray(item)) return item.slice();
2102 + return {...item};
1708 2103 }
1709 -
1710 - dtype(item, test=null) {
1711 - let type = null;
1712 - if (item && Array.isArray(item)) {
1713 - type = 'array';
1714 - } else if (item && typeof item === 'object') {
1715 - type = 'object';
1716 - } else if (typeof item === 'string' || typeof item === 'number' || typeof item === 'boolean') {
1717 - type = 'literal';
1718 - }
1719 - if (test) {
1720 - if (test === 'number') {
1721 - return type === 'literal' && !isNaN(item);
1722 - } else if (test === 'string') {
1723 - return type === 'literal' && typeof item === 'string';
1724 - } else if (test === 'boolean') {
1725 - return type === 'literal' && typeof item === 'boolean';
1726 - }
1727 - return type === test;
1728 - }
1729 - return type;
2104 +
2105 + dtype(item, test = null) {
2106 + const t = typeof item;
2107 + const type = t === 'object' ? (item === null ? null : Array.isArray(item) ? 'array' : 'object')
2108 + : (t === 'string' || t === 'number' || t === 'boolean') ? 'literal'
2109 + : null;
2110 + if (!test) return type;
2111 + if (test === 'number') return type === 'literal' && !isNaN(item);
2112 + if (test === 'string') return t === 'string';
2113 + if (test === 'boolean') return t === 'boolean';
2114 + return type === test;
2115 + }
2116 +
2117 + nullish(val) {
2118 + return val === false || val === null || val === undefined || val === '' || (Array.isArray(val) && val.length == 0) || (typeof val == 'object' && Object.keys(val).length == 0);
1730 2119 }
1731 2120
1732 - getname(item) {
2121 + getname = (item) => {
1733 2122 if (typeof item === 'object' && item != null) {
1734 2123 if (item?.name != null) {
1735 2124 return item.name;
1736 2125 } else if (item?.key?.length > 0) {
1737 2126 return item.key.join(' / ');
1738 2127 }
1739 - const {id, ...nonidprops} = item;
1740 - const names = Object.values(nonidprops).filter((val) => typeof val === 'string' || typeof val === 'number');
1741 - if (names.length > 0) {
1742 - return names[0];
1743 - }
1744 - for(const key in nonidprops) {
1745 - if (Array.isArray(item[key]) && item[key].length === 1 && typeof item[key][0] == 'string') {
1746 - return item[key][0];
2128 + if (this.features.guessname) {
2129 + const {id, ...nonidprops} = item;
2130 + const names = Object.values(nonidprops).filter((val) => typeof val === 'string' || typeof val === 'number');
2131 + if (names.length > 0) {
2132 + return names[0];
2133 + }
2134 + for (const key in nonidprops) {
2135 + if (Array.isArray(item[key]) && item[key].length == 1 && typeof item[key][0] == 'string') {
2136 + return item[key][0];
2137 + }
1747 2138 }
1748 2139 }
1749 2140 } else if (typeof item === 'string' || typeof item === 'number') {
@@ -1753,10 +2144,10 @@ export class DACTAL {
1753 2144 }
1754 2145 return '';
1755 2146 }
1756 -
1757 - getid(item) {
2147 +
2148 + getid = (item) => {
1758 2149 if (typeof item === 'object' && item != null) {
1759 - const iditems = (item.id || item.uri || item.name) ? [item] : item.key?.length > 0 ? item.key : Object.keys(item).length === 3 && item.of?.length > 0 ? item.of : [item];
2150 + const iditems = (item.id || item.uri || item.name) ? [item] : (item.key?.length > 0 && !item.keyindex) ? item.key : Object.keys(item).length == 3 && item.of?.length > 0 ? item.of : [item];
1760 2151 return iditems.map((item) => {
1761 2152 if (item.id) {
1762 2153 if (Array.isArray(item.id)) {
@@ -1770,11 +2161,12 @@ export class DACTAL {
1770 2161 } else {
1771 2162 return item.uri;
1772 2163 }
1773 - } else if (item.name) {
2164 + } else if (this.features.guessid && item.name) {
1774 2165 return item.name;
1775 2166 } else {
2167 + const stringifiedid = JSON.stringify(item);
1776 2168 // if (stringifiedid.length > 128) console.warn({idstringify: item, idlength: stringifiedid.length});
1777 - return JSON.stringify(item);
2169 + return stringifiedid;
1778 2170 }
1779 2171 }).join(',');
1780 2172 } else if (typeof item === 'string' || typeof item === 'number') {
@@ -1782,30 +2174,53 @@ export class DACTAL {
1782 2174 }
1783 2175 return null;
1784 2176 }
1785 -
2177 +
2178 + getnumber = (val) => {
2179 + let number = Array.isArray(val) ? val[0] : val;
2180 + number = !isNaN(number) ? Number(val) : null;
2181 + return number;
2182 + }
2183 +
2184 + opclone = (x) => {
2185 + if (x === null || typeof x !== 'object') return x; // primitives
2186 + if (Array.isArray(x)) {
2187 + const a = new Array(x.length);
2188 + for (let i = 0; i < x.length; i++) a[i] = this.opclone(x[i]);
2189 + return a;
2190 + }
2191 + if (x instanceof RegExp) return new RegExp(x.source, x.flags); // preserve the `~` matchers
2192 + const o = {};
2193 + for (const k of Object.keys(x)) o[k] = this.opclone(x[k]);
2194 + return o;
2195 + }
2196 +
1786 2197 step = (item, property, subop, labeled) => {
1787 2198 // (this.data.trace ??= []).push({stepitem: item, property: property, subop: subop, labeled: labeled});
1788 2199 const dtype = this.dtype;
1789 2200 const dcopy = this.dcopy;
1790 2201 const resolve = this.resolve;
1791 2202 const escapeRegExp = this.escapeRegExp;
2203 + const opclone = this.opclone;
1792 2204 const vals = [];
1793 2205 if (subop?.includes('~') && dtype(property, 'literal')) {
1794 2206 vals.push(property);
1795 2207 } else if (Array.isArray(property)) {
1796 - this.execute([item], property.map((x) => structuredClone(x)), labeled).forEach((val) => vals.push(val));
1797 - } else if (property === '_') {
2208 + const res = this.execute([item], property.map(opclone), labeled).flatMap((x) => x);
2209 + if (subop?.includes('~') && res.length > 0) {
2210 + vals.push(this.getname(res[0]));
2211 + } else {
2212 + res.forEach((val) => vals.push(val));
2213 + }
2214 + } else if (property == '_') {
1798 2215 vals.push(dcopy(item));
1799 - } else if (item?.toString()?.match(/^-?\d+$/) && property.match(/^\d+$/)) {
1800 - vals.push((item === 0 && 1 / item === -Infinity ? -1 : 1) * Number(`${item}.${property}`));
1801 2216 } else if (item?.of && dtype(property, 'number')) {
1802 2217 const propval = Number(property);
1803 - (subop === '-' || propval < 0 ? item.of.slice(-1 * Math.abs(Number(property))) : item.of.slice(0, Math.abs(Number(property)))).forEach((val) => vals.push(val));
1804 - } else if (property === 'id' && dtype(item, 'literal')) {
2218 + (subop == '-' || propval < 0 ? item.of.slice(-1 * Math.abs(Number(property))) : item.of.slice(0, Math.abs(Number(property)))).forEach((val) => vals.push(val));
2219 + } else if (property == 'id' && dtype(item, 'literal')) {
1805 2220 vals.push(item);
1806 - } else if (property === 'name') {
2221 + } else if (property == 'name') {
1807 2222 vals.push(this.getname(item));
1808 - } else if ((dtype(item, 'number') || dtype(item, 'object')) && property.startsWith('=')) {
2223 + } else if (this.features.inlinemath && property.startsWith('=')) {
1809 2224 let calculation = property.slice(1);
1810 2225 const mathwords = Object.getOwnPropertyNames(Math).filter((mathword) => mathword.match(/^[a-z0-9]+$/)).sort((a, b) => b.length - a.length || a.localeCompare(b));
1811 2226 const otherwords = ['split'];
@@ -1814,15 +2229,15 @@ export class DACTAL {
1814 2229 .sort((a, b) => b.length - a.length || a.localeCompare(b));
1815 2230 if (dtype(this.getname(item), 'number')) variables.push('_');
1816 2231 const allowedwords = mathwords.concat(otherwords).concat(variables).sort((a, b) => b.length - a.length || a.localeCompare(b));
1817 - const allowed = new RegExp(`^((\b(${allowedwords.map((w) => escapeRegExp(w)).join('|')})\b)|([0-9_\+\/\*\(\)\[\].%=,'" -]*))*$`);
2232 + const allowed = new RegExp(`^((\\b(${allowedwords.map((w) => escapeRegExp(w)).join('|')})\\b)|([0-9_\\+\\/\\*\\(\\)\\[\\]\\.%=,'" -]*))*$`);
1818 2233 let val;
1819 2234 if (calculation.match(allowed)) {
1820 2235 for (const variable of variables) {
1821 - const variableex = new RegExp(`\\b${variable}\\b`, 'g');
2236 + const variableex = new RegExp(`\\b${escapeRegExp(variable)}\\b`, 'g');
1822 2237 if (calculation.match(variableex)) {
1823 - let vval = variable === '_' ? Number(this.getname(item)) : item[variable] ?? labeled[variable];
1824 - if (Array.isArray(vval) && vval.length === 1) vval = vval[0];
1825 - calculation = calculation.replaceAll(variableex, dtype(vval, 'number') ? vval : JSON.stringify(vval));
2238 + let vval = variable == '_' ? Number(this.getname(item)) : item[variable] ?? labeled[variable];
2239 + if (Array.isArray(vval) && vval.length == 1) vval = vval[0];
2240 + calculation = calculation.replaceAll(variableex, ` ${dtype(vval, 'number') ? vval : JSON.stringify(vval)} `);
1826 2241 }
1827 2242 }
1828 2243 if (calculation.match(/[A-Za-z]/)) {
@@ -1842,12 +2257,32 @@ export class DACTAL {
1842 2257 val = calculation;
1843 2258 }
1844 2259 if (val !== false) vals.push(val);
2260 +
2261 + } else if (property.startsWith('~')) {
2262 + let val = property.slice(1);
2263 + const variables = Object.entries(item).concat(Object.entries(labeled))
2264 + .map(([k, v]) => k)
2265 + .sort((a, b) => b.length - a.length || a.localeCompare(b));
2266 + variables.push('_');
2267 + for (const variable of variables) {
2268 + const variableex = new RegExp(`(?<!\w)${escapeRegExp(variable)}(?!\w)`, 'g');
2269 + if (val.match(variableex)) {
2270 + let vval = variable == '_' ? this.getname(item) : item[variable] ?? labeled[variable];
2271 + if (Array.isArray(vval) && vval.length == 1) vval = vval[0];
2272 + if (!dtype(vval, 'string')) vval = this.getname(vval);
2273 + if (vval) val = val.replaceAll(variableex, vval.toString());
2274 + }
2275 + }
2276 + if (val) vals.push(val);
1845 2277 } else if (item) {
1846 2278 let found = false;
1847 2279 if (property != null && !dtype(item, 'literal')) {
1848 2280 const tryvals = [property];
1849 - tryvals.push(property.endsWith('s') ? property.slice(0, -1) : property + 's');
1850 - if (property.match(/ /)) tryvals.push(property.replaceAll(/ /g, '_'));
2281 + if (this.features.plurality) {
2282 + if (!property.endsWith('s') && !this.internal_datasets.includes(property + 's')) tryvals.push(property + 's');
2283 + if (property.endsWith('s') && !this.internal_datasets.includes(property)) tryvals.push(property.replace(/s$/, ''));
2284 + }
2285 + if (this.features.unscore && property.match(/ /)) tryvals.push(property.replaceAll(/ /g, '_'));
1851 2286 for (const tryval of tryvals) {
1852 2287 if (tryval in item) {
1853 2288 const resolved = resolve(tryval, item[tryval], labeled);
@@ -1866,33 +2301,25 @@ export class DACTAL {
1866 2301 found = true;
1867 2302 }
1868 2303 if (!found && (property in this.data || property in this.adapters || this.savedquerynames.has(property) || property in labeled)) {
1869 - let typenav;
1870 - if (!dtype(item, 'literal')) {
1871 - if (this.adapters?.[property]?.annotator) {
1872 - typenav = resolve(property, item, labeled);
1873 - // } else if (property in this.aggregators) {
1874 - // typenav = this.aggregators[property](item);
1875 - } else if ('id' in item || 'uri' in item) {
1876 - typenav = resolve(property, this.getid(item), labeled);
1877 - } else if (this.savedquerynames.has(property)) {
1878 - const relative_query = dactal.data.queries.find((q) => q.relative && q.name === property);
1879 - if (relative_query) {
1880 - typenav = this.step(item, this.parse(relative_query.query), subop, labeled);
1881 - } else {
1882 - typenav = this.getname(item);
1883 - }
1884 - } else {
1885 - typenav = this.getname(item);
1886 - }
1887 - if (typenav) vals.push(typenav);
2304 + const typenav = resolve(property, item, labeled, true);
2305 + if (typenav) {
2306 + (Array.isArray(typenav) ? typenav : [typenav]).forEach((t) => vals.push(t));
2307 + found = true;
2308 + }
2309 + }
2310 + if (!found && property.includes('→')) {
2311 + const parts = property.split(/\s*→\s*/).filter((part) => part?.length > 0);
2312 + if (dtype(item, 'object')) {
2313 + this.step(item, parts.map((part) => ({
2314 + operator: '..',
2315 + args: [{value: part}]
2316 + }))).forEach((val) => vals.push(val));
1888 2317 } else {
1889 - typenav = resolve(property, item, labeled);
1890 - if (typenav) {
1891 - if (Array.isArray(typenav)) {
1892 - typenav.forEach((t) => vals.push(t));
1893 - } else {
1894 - vals.push(typenav);
1895 - }
2318 + const asnum = dtype(item, 'number');
2319 + if (item == (asnum ? Number(parts[0]) : parts[0])) {
2320 + parts.slice(1).forEach((newval) => vals.push(asnum ? Number(newval) : newval));
2321 + } else {
2322 + vals.push(item);
1896 2323 }
1897 2324 }
1898 2325 }
@@ -1900,94 +2327,162 @@ export class DACTAL {
1900 2327 return vals;
1901 2328 }
1902 2329
1903 - resolve = (property, item, labeled) => {
2330 + indexlogit = (property, type) => {
2331 + const today = new Date().toISOString().slice(0, 10);
2332 + ((this.index._ ||= {})[property] ||= {})[today] ||= {read: 0, write: 0};
2333 + this.index._[property][today][type]++;
2334 + this.index_modified.add('_');
2335 + if (type == 'write') this.index_modified.add(property);
2336 + }
2337 +
2338 + lookup = (item, list) => {
2339 + return list.filter((li) => {
2340 + if (item.id) return item.id == li.id;
2341 + if (item.uri) return item.uri == li.uri;
2342 + for (const k of this.kkeys(item)) {
2343 + if (item[k]?.toString() != li?.[k]?.toString()) return false;
2344 + }
2345 + return true;
2346 + });
2347 + }
2348 +
2349 + indexed = (property, key, item, func) => {
2350 + if ((!this.recache || this.recache.has(property)) && this.index[property]?.[key] !== undefined) {
2351 + this.indexlogit(property, 'read');
2352 + return this.index[property][key];
2353 + }
2354 + const res = func(item);
2355 + if (res) {
2356 + (this.index[property] ||= {})[key] = res;
2357 + this.indexlogit(property, 'write');
2358 + }
2359 + return res;
2360 + }
2361 +
2362 + resolve = (property, item, labeled, navigate = false) => {
1904 2363 const dtype = this.dtype;
2364 + const aqueue = this.aqueue;
1905 2365 if (dtype(item, 'object')) {
1906 2366 if (this.adapters?.[property]?.annotator) {
1907 - if ((!this.recache || this.recache.has(property)) && property in this.index && this.getid(item) in this.index[property]) {
1908 - return this.index[property][this.getid(item)];
1909 - } else {
1910 - this.adapters[property].queue.push(this.adapters[property].annotator ? item : this.getid(item));
2367 + return this.indexed(property, this.getid(item), item, () => aqueue(property, item));
2368 + } else if (navigate) {
2369 + const res = this.indexed(property, this.getid(item), item, () => {
2370 + if (property in this.data) {
2371 + return this.lookup(item, this.data[property]);
2372 + } else if (property in labeled) {
2373 + return this.lookup(item, labeled[property]);
2374 + } else if (this.savedquerynames.has(property)) {
2375 + const saved_query = this.data.queries.find((q) => q.name == property);
2376 + if (saved_query.results && !saved_query.relative) {
2377 + return this.lookup(item, saved_query.results);
2378 + } else if (saved_query.relative) {
2379 + return this.execute([item], this.parse(saved_query.query), labeled);
2380 + }
2381 + }
1911 2382 return null;
2383 + });
2384 + if (res) return res;
2385 + if ('id' in item || 'uri' in item) {
2386 + return this.resolve(property, this.getid(item), labeled);
1912 2387 }
2388 + return null;
1913 2389 } else {
1914 2390 return item;
1915 2391 }
1916 - } else if (dtype(item, 'literal') && typeof property == 'string' && (property in labeled || this.destinations.has(property) || this.destinations.has(property + 's'))) {
1917 - if ((!this.recache || this.recache.has(property)) && property in this.index && item in this.index[property]) {
1918 - // this.cachehitcount++;
1919 - return this.index[property][item];
1920 - }
1921 - const typeitems = labeled[property] ?? this.gettype(property);
1922 - if (Array.isArray(typeitems)) {
1923 - for (const lookupkey of ['id', 'uri', 'name', property, property + 's']) {
1924 - const found = typeitems.filter((typeitem) => {
1925 - return dtype(typeitem, 'object') && (lookupkey in typeitem) && (typeitem[lookupkey] === item || (Array.isArray(typeitem[lookupkey]) && typeitem[lookupkey].length === 1 && typeitem[lookupkey][0] === item));
1926 - });
1927 - if (found.length === 1) {
1928 - (this.index[property] ||= {})[item] = found[0];
1929 - return found[0];
2392 + } else if (dtype(item, 'literal') && typeof property == 'string' && (property in labeled || this.destinations.has(property) || (this.features.plurality && !this.internal_datasets.includes(property + 's') && this.destinations.has(property + 's')))) {
2393 + let dataset = false;
2394 + const res = this.indexed(property, item, item, () => {
2395 + if (!(this.features.autoresolve || navigate)) return null;
2396 + const typeitems = labeled[property] ?? this.gettype(property);
2397 + if (Array.isArray(typeitems)) {
2398 + dataset = true;
2399 + const trykeys = ['id', 'uri', 'name'];
2400 + if (this.features.plurality) {
2401 + trykeys.push(property);
2402 + trykeys.push(property.endsWith('s') ? property.slice(0, property.length - 1) : property + 's');
2403 + }
2404 + for (const lookupkey of trykeys) {
2405 + const found = typeitems.filter((typeitem) => {
2406 + return dtype(typeitem, 'object') && (lookupkey in typeitem) && (typeitem[lookupkey].toString() == item || (Array.isArray(typeitem[lookupkey]) && typeitem[lookupkey].length == 1 && typeitem[lookupkey][0].toString() == item));
2407 + });
2408 + if (found.length == 1) {
2409 + return found[0];
2410 + }
2411 + }
2412 + return null;
2413 + } else if (this.savedquerynames.has(property)) {
2414 + const saved_query = this.data.queries.find((q) => q.name == property);
2415 + if (saved_query.results && !saved_query.relative) {
2416 + return this.lookup(item, saved_query.results);
2417 + } else if (saved_query.relative) {
2418 + return this.execute([item], this.parse(saved_query.query), labeled);
1930 2419 }
1931 - }
1932 - if (property in this.adapters && (!this.recache || !this.recache.has(property))) {
1933 - this.adapters[property].queue.push(this.adapters[property].annotator ? item : this.getid(item));
1934 2420 }
1935 2421 return null;
1936 - } else if (typeitems && typeof typeitems == 'object' && 'query' in typeitems) {
1937 - const relative_query = this.assemble(this.tokenize(typeitems.query));
1938 - while (['?', ':'].includes(relative_query[0].operator)) {
1939 - relative_query.shift()
2422 + });
2423 + if (res) return res;
2424 + if (dataset) {
2425 + if (property in this.adapters && (!this.recache || this.recache.has(property))) {
2426 + return aqueue(property, item);
1940 2427 }
1941 - return this.execute([item], relative_query, labeled);
2428 + return null;
1942 2429 }
1943 2430 if (property in this.adapters) {
1944 - this.adapters[property].queue.push(this.adapters[property].annotator ? item : this.getid(item));
1945 - return null;
2431 + return aqueue(property, item);
1946 2432 }
1947 2433 }
1948 2434 return item;
1949 2435 }
1950 -
2436 +
2437 + aqueue = (property, item) => {
2438 + const pending_queues = Object.keys(this.adapters).filter((key) => this.adapters[key].queue.length > 0);
2439 + if (pending_queues.length == 0 || (pending_queues.length == 1 && pending_queues[0] == property)) {
2440 + this.adapters[property].queue.push(this.adapters[property].annotator ? item : this.getid(item));
2441 + }
2442 + this.adaptive = true;
2443 + return undefined;
2444 + }
2445 +
1951 2446 samearray(a, b) {
1952 - if ((a && !b) || (!a && b) || a.length !== b.length) return false;
1953 - for(let x=0; x<a.length; x++) {
1954 - if (this.getid(a[x]) !== this.getid(b[x])) return false;
2447 + if ((a && !b) || (!a && b) || a.length != b.length) return false;
2448 + for (let x = 0; x < a.length; x++) {
2449 + if (this.getid(a[x]) != this.getid(b[x])) return false;
1955 2450 }
1956 2451 return true;
1957 2452 }
1958 -
1959 - async verify(i = null) {
2453 +
2454 + verify = async (i = null) => {
1960 2455 let toverify = this.data.queries;
1961 - if (!isNaN(i)) toverify = toverify.slice(i - 1, i);
2456 + if (i && !isNaN(i)) toverify = toverify.slice(i - 1, i);
1962 2457 let allsame = true;
1963 2458 for (const savedq of toverify) {
1964 2459 console.log('verifying ' + savedq.name);
1965 2460 const testres = await this.query(savedq.query);
1966 - if (testres.length !== savedq.results.length) {
2461 + if (testres.length != savedq.results.length) {
1967 2462 console.log('--x result count changed from ' + savedq.results.length + ' to ' + testres.length);
1968 2463 allsame = false;
1969 2464 } else {
1970 - for (let i=0; i<testres.length; i++) {
2465 + for (let i = 0; i < testres.length; i++) {
1971 2466 const testrow = testres[i];
1972 2467 const savedrow = savedq.results[i];
1973 2468 if (typeof savedrow == 'object') {
1974 2469 for (const prop in savedrow) {
1975 2470 if (!(prop in testrow)) {
1976 - console.log('--x row ' + (i+1) + ': new results missing property ' + prop);
2471 + console.log('--x row ' + (i + 1) + ': new results missing property ' + prop);
1977 2472 allsame = false;
1978 2473 } else {
1979 2474 const testval = JSON.stringify(testrow[prop]);
1980 2475 const savedval = JSON.stringify(savedrow[prop]);
1981 - if (testval !== savedval) {
1982 - console.log('--x row ' + (i+1) + ': different value for property ' + prop);
2476 + if (testval != savedval) {
2477 + console.log('--x row ' + (i + 1) + ': different value for property ' + prop);
1983 2478 console.log({was: savedrow[prop], now: testrow[prop]})
1984 2479 allsame = false;
1985 2480 }
1986 2481 }
1987 2482 }
1988 2483 } else {
1989 - if (testrow !== savedrow) {
1990 - console.log('--x row ' + (i+1) + ': different value');
2484 + if (testrow != savedrow) {
2485 + console.log('--x row ' + (i + 1) + ': different value');
1991 2486 console.log({was: savedrow, now: testrow});
1992 2487 allsame = false;
1993 2488 }
@@ -1997,12 +2492,49 @@ export class DACTAL {
1997 2492 if (allsame) console.log('--- results unchanged');
1998 2493 }
1999 2494 }
2000 -
2495 +
2001 2496 index_check() {
2002 - console.table(Object.entries(this.index).map(([key, vals]) => ({key: key, vals: Object.keys(vals).length, size: Object.keys(vals).length * JSON.stringify(Object.entries(vals).slice(0, 1)).length})).sort((a, b) => b.size - a.size || b.vals - a.vals || a.key.localeCompare(b.key)))
2497 + console.table(Object.entries(this.index).map(([key, vals]) => ({
2498 + key: key,
2499 + vals: Object.keys(vals).length,
2500 + size: Object.keys(vals).length * JSON.stringify(Object.entries(vals).slice(0, 1)).length
2501 + })).sort((a, b) => b.size - a.size || b.vals - a.vals || a.key.localeCompare(b.key)))
2003 2502 }
2004 -
2503 +
2005 2504 index_materialize() {
2006 - this.load(Object.keys(this.index).flatMap((i) => Object.keys(this.index[i]).flatMap((k) => ({index: i, indexed: k, value: this.index[i][k]}))), 'index contents')
2505 + this.load(Object.keys(this.index).flatMap((i) => Object.keys(this.index[i]).flatMap((k) => ({
2506 + index: i,
2507 + indexed: k,
2508 + value: this.index[i][k]
2509 + }))), 'index contents')
2510 + }
2511 +
2512 + indexlog_materialize() {
2513 + this.load(Object.entries(this.index._).map(([k, v]) => ({
2514 + index: k,
2515 + log: Object.entries(v).map(([date, readwrite]) => ({
2516 + date: date,
2517 + read: readwrite.read,
2518 + write: readwrite.write
2519 + }))
2520 + })), 'indexlog');
2521 + }
2522 +
2523 + queries_check() {
2524 + console.table(dactal.data.queries.map((q) => ({
2525 + queryname: q.name,
2526 + results: q.results?.length || 0,
2527 + size: q.results?.length > 0 ? q.results.length * JSON.stringify(q.results[0]).length : 0
2528 + })).sort((a, b) => b.size - a.size || b.results - a.results || a.queryname.localeCompare(b.queryname)))
2529 + }
2530 +
2531 + data_check() {
2532 + console.table(Object.entries(this.data).map(([key, vals]) => ({
2533 + key: key,
2534 + vals: Object.keys(vals).length,
2535 + size: Object.keys(vals).length * JSON.stringify(Object.entries(vals).slice(0, 1)).length
2536 + })).sort((a, b) => b.size - a.size || b.vals - a.vals || a.key.localeCompare(b.key)))
2007 2537 }
2008 2538 }
2539 +
2540 +window.DACTAL = new DACTAL();
No newline at end of file