| @@ -0,0 +1,2008 @@ |
|
1 |
+export class DACTAL { |
|
2 |
+ constructor(data={}) { |
|
3 |
+ this.data = data; |
|
4 |
+ this.index = {}; |
|
5 |
+ this.index_modified = false; |
|
6 |
+ this.adapters = {}; |
|
7 |
+ this.data['query history'] = []; |
|
8 |
+ this.savedquerynames = new Set(); |
|
9 |
+ this.debug = false; |
|
10 |
+ this.recache = false; |
|
11 |
+ this.timelimit = 120000; |
|
12 |
+ this.statusf = (statusmsg) => console.log(statusmsg); |
|
13 |
+ this.aggregators = { |
|
14 |
+ group: (item) => item.of, |
|
15 |
+ count: (item) => this.vals(item).length, |
|
16 |
+ total: (item) => this.numvals(item).reduce((acc, val) => acc + (this.dtype(val, 'number') ? val : 0), 0), |
|
17 |
+ average: (item) => this.numvals(item).reduce((acc, val) => acc + (this.dtype(val, 'number') ? val : 0), 0) / this.numvals(item).length, |
|
18 |
+ min: (item) => { |
|
19 |
+ const nums = this.numvals(item); |
|
20 |
+ return nums.length === 0 ? [] : Math.min(...nums) |
|
21 |
+ }, |
|
22 |
+ max: (item) => { |
|
23 |
+ const nums = this.numvals(item); |
|
24 |
+ return nums.length === 0 ? [] : Math.max(...nums) |
|
25 |
+ }, |
|
26 |
+ product: (item) => this.numvals(item).reduce((acc, val) => acc * (this.dtype(val, 'number') ? val : 1), 1), |
|
27 |
+ difference: (item) => this.numvals(item).reduce((acc, val) => acc - (this.dtype(val, 'number') ? val : 0)), |
|
28 |
+ quotient: (item) => this.numvals(item).reduce((acc, val) => acc / (this.dtype(val, 'number') ? val : 1)), |
|
29 |
+ percent: (item) => Math.round(this.numvals(item).reduce((acc, val) => acc / (this.dtype(val, 'number') ? val : 1)) * 100), |
|
30 |
+ sqrt: (item) => Math.sqrt(this.numvals(item)[0]), |
|
31 |
+ log: (item) => Math.log(this.numvals(item)[0]), |
|
32 |
+ log10: (item) => Math.log10(this.numvals(item)[0]), |
|
33 |
+ abs: (item) => Math.abs(this.numvals(item)[0]), |
|
34 |
+ is: (item) => (item.of?.length > 0) ? 1 : 0, |
|
35 |
+ isnt: (item) => (item.of?.length === 0) ? 1 : 0, |
|
36 |
+ yesno: (item) => (item.of?.length > 0) ? 'yes' : 'no', |
|
37 |
+ missing: (item) => (item.of?.length === 0) ? true : null, |
|
38 |
+ concatenate: (item) => this.vals(item).join(' '), |
|
39 |
+ join: (item) => item.of.map(this.getname).join(this.vals(item)[0]), |
|
40 |
+ str: (item) => item.of.map(this.getname).join(''), |
|
41 |
+ 'to json': (item) => JSON.stringify(item.of), |
|
42 |
+ quote: (item) => `“${this.vals(item)[0]}”`, |
|
43 |
+ url: (item) => { |
|
44 |
+ let u = item.of.map(this.getname).join(''); |
|
45 |
+ if (!u.startsWith('https://')) u = 'https://' + u; |
|
46 |
+ for (const prop of Object.keys(item).filter((key) => key !== 'of')) { |
|
47 |
+ const val = item[prop]; |
|
48 |
+ (Array.isArray(val) ? val : [val]).forEach((vv) => u = u + (u.match(/\?/) ? '&' : '?') + encodeURIComponent(prop) + '=' + encodeURIComponent(vv)); |
|
49 |
+ } |
|
50 |
+ return u; |
|
51 |
+ }, |
|
52 |
+ remove: (item) => this.vals(item).reduce((acc, val) => acc.replaceAll(val, '')), |
|
53 |
+ bmk: (item) => this.vals(item).map((v) => v.toString().toLowerCase().replaceAll(/[$\xA2-\xA5\u058F\u060B\u09F2\u09F3\u09FB\u0AF1\u0BF9\u0E3F\u17DB\u20A0-\u20BD\uA838\uFDFC\uFE69\uFF04\uFFE0\uFFE1\uFFE5\uFFE6,+]/g, '').replaceAll('b', 'kkk').replaceAll('m', 'kk').replaceAll('k', '000')), |
|
54 |
+ sortform: (item) => this.vals(item).map((val) => val.toString().toLowerCase().replace(/^the /, '')), |
|
55 |
+ zip: (item) => { |
|
56 |
+ const itemkeys = Object.keys(item).filter((k) => k !== 'of'); |
|
57 |
+ const zipped = []; |
|
58 |
+ for (let i=0; i<item[itemkeys[0]].length; i++) { |
|
59 |
+ const zipline = {}; |
|
60 |
+ for (const key of itemkeys) { |
|
61 |
+ zipline[key] = item[key][i]; |
|
62 |
+ } |
|
63 |
+ zipped.push(zipline); |
|
64 |
+ } |
|
65 |
+ return zipped; |
|
66 |
+ }, |
|
67 |
+ pairs: (item) => { |
|
68 |
+ return item.of.slice(0, -1).map((val, vx) => ({pair: [val, item.of[vx+1]]})); |
|
69 |
+ }, |
|
70 |
+ triples: (item) => { |
|
71 |
+ return item.of.slice(0, -2).map((val, vx) => ({triple: [val, item.of[vx+1], item.of[vx+2]]})); |
|
72 |
+ }, |
|
73 |
+ quads: (item) => { |
|
74 |
+ return item.of.slice(0, -3).map((val, vx) => ({quad: [val, item.of[vx+1], item.of[vx+2], item.of[vx+3]]})); |
|
75 |
+ }, |
|
76 |
+ sequences: (item) => { |
|
77 |
+ return item.of.map((val, valx, vallist) => ({sequence: vallist.slice(0, valx+1).map((val) => this.dcopy(val))})); |
|
78 |
+ }, |
|
79 |
+ split: (item) => { |
|
80 |
+ const itemvals = this.vals(item); |
|
81 |
+ let splitter; |
|
82 |
+ let tobesplit; |
|
83 |
+ if (Object.keys(item).length === 1) itemvals.push(' '); |
|
84 |
+ if (itemvals.length === 1) { |
|
85 |
+ splitter = itemvals[0]; |
|
86 |
+ tobesplit = [this.getname(item)]; |
|
87 |
+ } else { |
|
88 |
+ splitter = itemvals.pop(); |
|
89 |
+ tobesplit = itemvals.slice(0); |
|
90 |
+ } |
|
91 |
+ if (splitter.startsWith('~')) splitter = new RegExp(splitter.replace(/^~*/, ''), splitter.startsWith('~~') ? 'i' : ''); |
|
92 |
+ return tobesplit.flatMap((v) => v.toString().split(splitter)); |
|
93 |
+ }, |
|
94 |
+ extract: (item) => { |
|
95 |
+ const itemvals = this.vals(item); |
|
96 |
+ const delimiters = itemvals.pop(); |
|
97 |
+ const res = []; |
|
98 |
+ for (const itemval of itemvals) { |
|
99 |
+ for(let i=0; i<delimiters.length; i+=2) { |
|
100 |
+ const d1 = delimiters[i]; |
|
101 |
+ const d2 = delimiters[i+1]; |
|
102 |
+ const d1x = itemval.indexOf(d1); |
|
103 |
+ const d2x = itemval.indexOf(d2); |
|
104 |
+ if (d1x > -1 && d2x > d1x) { |
|
105 |
+ res.push(itemval.slice(d1x+1, d2x).trim()); |
|
106 |
+ break; |
|
107 |
+ } |
|
108 |
+ } |
|
109 |
+ } |
|
110 |
+ return res; |
|
111 |
+ }, |
|
112 |
+ unchain: (item) => { |
|
113 |
+ const chainprops = Object.keys(item).filter((key) => key !== 'of'); |
|
114 |
+ const unchained = []; |
|
115 |
+ const queue = item.of.slice(0); |
|
116 |
+ while (queue.length > 0) { |
|
117 |
+ const thisitem = queue.shift(); |
|
118 |
+ if (!unchained.includes(thisitem)) { |
|
119 |
+ unchained.push(thisitem); |
|
120 |
+ for (const chainprop of chainprops) { |
|
121 |
+ if (this.dtype(thisitem, 'object') && chainprop in thisitem) { |
|
122 |
+ if (this.dtype(thisitem[chainprop], 'array')) { |
|
123 |
+ for (const x of thisitem[chainprop].slice(0).reverse()) queue.unshift(x); |
|
124 |
+ } else if (thisitem[chainprop]) { |
|
125 |
+ queue.unshift(thisitem[chainprop]) |
|
126 |
+ } |
|
127 |
+ } |
|
128 |
+ } |
|
129 |
+ } |
|
130 |
+ } |
|
131 |
+ return unchained; |
|
132 |
+ }, |
|
133 |
+ itemize: (item) => { |
|
134 |
+ const propname = item?.property ?? 'property'; |
|
135 |
+ const valname = item?.value ?? 'value'; |
|
136 |
+ return item.of.flatMap((subitem) => Object.entries(subitem).map(([key, val]) => ({[propname]: key, [valname]: val}))); |
|
137 |
+ }, |
|
138 |
+ schematize: (item) => { |
|
139 |
+ const itemkeys = Object.keys(item).filter((k) => k !== 'of'); |
|
140 |
+ const schematized = {}; |
|
141 |
+ item[itemkeys[0]].forEach((subitem) => { |
|
142 |
+ let subkey; |
|
143 |
+ let subval; |
|
144 |
+ if (this.dtype(subitem, 'array')) { |
|
145 |
+ const [subkey, subval] = subitem; |
|
146 |
+ } else { |
|
147 |
+ [subkey, subval] = Object.values(subitem); |
|
148 |
+ } |
|
149 |
+ schematized[subkey] = subval; |
|
150 |
+ }); |
|
151 |
+ return schematized; |
|
152 |
+ }, |
|
153 |
+ index: (item) => { |
|
154 |
+ return Object.entries(item).filter(([k, v]) => k !== 'of').map(([k, v]) => ({id: k, name: !isNaN(v) ? Number(v) : v})); |
|
155 |
+ }, |
|
156 |
+ unflatten: (item) => { |
|
157 |
+ const itemkeys = Object.keys(item).filter((k) => k !== 'of'); |
|
158 |
+ if (itemkeys.length === 0) itemkeys.push(''); |
|
159 |
+ const newindex = {}; |
|
160 |
+ const neworder = []; |
|
161 |
+ item.of.forEach((subitem) => { |
|
162 |
+ Object.keys(subitem).forEach((field) => { |
|
163 |
+ itemkeys.forEach((key) => { |
|
164 |
+ if (field.startsWith(key)) { |
|
165 |
+ const subid = field.replace(key, ''); |
|
166 |
+ if (subid.length > 0) { |
|
167 |
+ if (!(subid in newindex)) { |
|
168 |
+ newindex[subid] = {}; |
|
169 |
+ neworder.push(subid); |
|
170 |
+ } |
|
171 |
+ newindex[subid][key] = subitem[field]; |
|
172 |
+ } |
|
173 |
+ } |
|
174 |
+ }); |
|
175 |
+ }); |
|
176 |
+ }); |
|
177 |
+ itemkeys.forEach((key) => delete item[key]); |
|
178 |
+ return neworder.map((k) => { |
|
179 |
+ const subitem = {}; |
|
180 |
+ subitem.subid = k; |
|
181 |
+ Object.assign(subitem, newindex[k]); |
|
182 |
+ return subitem; |
|
183 |
+ }); |
|
184 |
+ }, |
|
185 |
+ detupled: (item) => { |
|
186 |
+ const newobj = {}; |
|
187 |
+ item.of.forEach((subitem) => { |
|
188 |
+ if (Array.isArray(subitem) && subitem.length === 2) { |
|
189 |
+ newobj[subitem[0]] = subitem[1]; |
|
190 |
+ } |
|
191 |
+ }) |
|
192 |
+ return [newobj]; |
|
193 |
+ }, |
|
194 |
+ csv: (item) => { |
|
195 |
+ const keys = Object.entries(item).find(([k, v]) => k !== 'of')[1]; |
|
196 |
+ const res = []; |
|
197 |
+ const vals = item.of.flatMap((val) => this.dtype(val, 'string') ? val.split('\n').map((val) => val.trim()) : val); |
|
198 |
+ for (let i=0; i<vals.length; i+=keys.length) { |
|
199 |
+ const newitem = {}; |
|
200 |
+ for (let k=0; k<keys.length; k++) { |
|
201 |
+ newitem[keys[k]] = vals[i+k]; |
|
202 |
+ } |
|
203 |
+ res.push(newitem); |
|
204 |
+ } |
|
205 |
+ return res; |
|
206 |
+ }, |
|
207 |
+ tsv: (item) => { |
|
208 |
+ const text = this.getname(item); |
|
209 |
+ const lines = text.split('\n').filter((line) => line !== '').map((line) => line.split('\t').map((val) => val.trim())); |
|
210 |
+ const keys = lines[0]; |
|
211 |
+ return lines.slice(1).map((vals) => Object.fromEntries(vals.map((val, vi) => [keys[vi], val]))); |
|
212 |
+ }, |
|
213 |
+ ssv: (item) => { |
|
214 |
+ const text = this.getname(item); |
|
215 |
+ const lines = text.split('\n').filter((line) => line !== '').map((line) => line.split(/ +/).map((val) => val.trim())); |
|
216 |
+ const keys = lines[0]; |
|
217 |
+ return lines.slice(1).map((vals) => Object.fromEntries(vals.map((val, vi) => [keys[vi], val]))); |
|
218 |
+ }, |
|
219 |
+ json: (item) => this.vals(item).flatMap((v) => JSON.parse(v)), |
|
220 |
+ year: (item) => { |
|
221 |
+ if (item.date?.length > 0) return item.date[0].match(/\d\d\d\d/)[0]; |
|
222 |
+ let yearmatch = this.getname(item).match(/\d\d\d\d/); |
|
223 |
+ if (yearmatch) { |
|
224 |
+ return yearmatch[0]; |
|
225 |
+ } else { |
|
226 |
+ yearmatch = this.getid(item.of[0] || '').toString().match(/\d\d\d\d/); |
|
227 |
+ if (yearmatch) { |
|
228 |
+ return yearmatch[0]; |
|
229 |
+ } |
|
230 |
+ return null; |
|
231 |
+ } |
|
232 |
+ }, |
|
233 |
+ month: (item) => { |
|
234 |
+ if (item.date?.length > 0) return item.date[0].match(/\d\d\d\d-(\d\d)-\d\d/)[1]; |
|
235 |
+ let monthmatch = this.getname(item).match(/\d\d\d\d-(\d\d)-\d\d/); |
|
236 |
+ if (monthmatch) { |
|
237 |
+ return monthmatch[1]; |
|
238 |
+ } else { |
|
239 |
+ monthmatch = this.getid(item.of[0] || '').toString().match(/\d\d\d\d-(\d\d)-\d\d/); |
|
240 |
+ if (monthmatch) { |
|
241 |
+ return monthmatch[1]; |
|
242 |
+ } |
|
243 |
+ return null; |
|
244 |
+ } |
|
245 |
+ }, |
|
246 |
+ date: (item) => { |
|
247 |
+ let itemvals = this.vals(item); |
|
248 |
+ if (itemvals) { |
|
249 |
+ let datematch = itemvals[0].match(/\d\d\d\d-\d\d-\d\d/); |
|
250 |
+ if (datematch) { |
|
251 |
+ return datematch[0]; |
|
252 |
+ } else { |
|
253 |
+ datematch = this.getid(item.of[0] || '').toString().match(/\d\d\d\d-\d\d-\d\d/); |
|
254 |
+ if (datematch) { |
|
255 |
+ return datematch[0]; |
|
256 |
+ } |
|
257 |
+ } |
|
258 |
+ } |
|
259 |
+ return null; |
|
260 |
+ }, |
|
261 |
+ weekday: (item) => { |
|
262 |
+ const days = ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun']; |
|
263 |
+ if (item.date?.length > 0) return days[new Date(item.date).getDay()]; |
|
264 |
+ }, |
|
265 |
+ timeshift: (item) => { |
|
266 |
+ const vals = this.vals(item); |
|
267 |
+ let tsx = new Date(vals[0]); |
|
268 |
+ let adjust = Number(vals[1]) * 60*60*1000; |
|
269 |
+ tsx.setTime(tsx.getTime() + adjust); |
|
270 |
+ return tsx.toISOString(); |
|
271 |
+ }, |
|
272 |
+ hour: (item) => this.vals(item).map((v) => v.split('T')[1].split(':')[0]), |
|
273 |
+ datediff: (item) => { |
|
274 |
+ const [d1, d2] = this.vals(item); |
|
275 |
+ return (new Date(d2) - new Date(d1)) / (24*60*60*1000); |
|
276 |
+ }, |
|
277 |
+ timediff: (item) => { |
|
278 |
+ const [d1, d2] = this.vals(item); |
|
279 |
+ return (new Date(d2) - new Date(d1)); |
|
280 |
+ }, |
|
281 |
+ dateforms: (item) => { |
|
282 |
+ const basedate = this.vals(item)[0]; |
|
283 |
+ const [baseyear, basemonth, baseday] = basedate.split('-'); |
|
284 |
+ return [ |
|
285 |
+ basedate, |
|
286 |
+ `${basemonth}/${baseday}/${baseyear}`, |
|
287 |
+ basemonth.startsWith('0') ? `${basemonth.replace(/^0/,'')}/${baseday}/${baseyear}` : null, |
|
288 |
+ basemonth.startsWith('0') && baseday.startsWith('0') ? `${basemonth.replace(/^0/,'')}/${baseday.replace(/^0/,'')}/${baseyear}` : null, |
|
289 |
+ `${basemonth}/${baseday}/${baseyear.slice(2)}`, |
|
290 |
+ `${basemonth.replace(/^0/,'')}/${baseday.replace(/^0/,'')}/${baseyear.slice(2)}`, |
|
291 |
+ `${['', 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'][Number(basemonth)]} ${baseday.replace(/^0/,'')}, ${baseyear}` |
|
292 |
+ ].filter(x => x); |
|
293 |
+ }, |
|
294 |
+ now: (item) => performance.now(), |
|
295 |
+ round: (item) => this.numvals(item).map((v) => Math.round(v)), |
|
296 |
+ roundaway: (item) => this.numvals(item).map((v) => Math.sign(v) * Math.round(Math.abs(v))), |
|
297 |
+ roundm: (item) => { |
|
298 |
+ const vals = this.numvals(item); |
|
299 |
+ const multiple = vals.pop(); |
|
300 |
+ return vals.map((v) => Math.round(v / multiple) * multiple); |
|
301 |
+ }, |
|
302 |
+ roundd: (item) => { |
|
303 |
+ const vals = this.numvals(item); |
|
304 |
+ const digits = vals.pop(); |
|
305 |
+ return vals.map((v) => { |
|
306 |
+ const factor = 10 ** (Math.floor(Math.log10(v)) - digits + 1); |
|
307 |
+ return Math.round(v / factor) * factor; |
|
308 |
+ }); |
|
309 |
+ }, |
|
310 |
+ floor: (item) => this.numvals(item).map((v) => Math.floor(v)), |
|
311 |
+ ceil: (item) => this.numvals(item).map((v) => Math.ceil(v)), |
|
312 |
+ n: (item) => { |
|
313 |
+ let i = 0; |
|
314 |
+ const test = this.getname(item.of[0]); |
|
315 |
+ for (const key in item) { |
|
316 |
+ if (key !== 'of') { |
|
317 |
+ i--; |
|
318 |
+ if (key === test) return i; |
|
319 |
+ } |
|
320 |
+ } |
|
321 |
+ return i - 1; |
|
322 |
+ }, |
|
323 |
+ numbers: (item) => { |
|
324 |
+ const res = []; |
|
325 |
+ const firstnum = item.from !== undefined && item.from != null ? Number(item.from) : 1; |
|
326 |
+ const lastnum = item.to !== undefined && item.to != null ? Number(item.to) : Number(this.getname(item.of[0])); |
|
327 |
+ for(let x=firstnum; x<=lastnum; x++) { |
|
328 |
+ res.push(x) |
|
329 |
+ } |
|
330 |
+ return res; |
|
331 |
+ }, |
|
332 |
+ dehyphenate: (item) => this.vals(item).flatMap((v) => v.replaceAll(/(?<=\w)-\n(?=\w)/g, '')), |
|
333 |
+ sentences: (item) => this.vals(item).flatMap((v) => v.split(/[.?!…]['"’”»]?\s+/)), |
|
334 |
+ allwords: (item) => this.vals(item).flatMap((v) => Array.from(v.toString().toLowerCase().matchAll(/[\p{Letter}\p{Number}]+(?:['‘’][\p{Letter}\p{Number}]+)?/gu).map((m) => m[0]))), |
|
335 |
+ words: (item) => this.vals(item).flatMap((v) => Array.from(v.toString().toLowerCase().matchAll(/[\p{Letter}\p{Number}]+(?:['‘’][\p{Letter}\p{Number}]+)?/gu).map((m) => m[0])).filter((w) => w.length >= 4)), |
|
336 |
+ characters: (item) => this.vals(item).flatMap((v) => Array.from(v)), |
|
337 |
+ 'character count': (item) => this.vals(item).flatMap((v) => v.length), |
|
338 |
+ case: (item) => this.vals(item).map((v) => { |
|
339 |
+ const hasupper = v.match(/[A-Z]/); |
|
340 |
+ const haslower = v.match(/[a-z]/); |
|
341 |
+ if (!hasupper && !haslower) { |
|
342 |
+ return 'none'; |
|
343 |
+ } else if (hasupper && !haslower) { |
|
344 |
+ return 'upper'; |
|
345 |
+ } else if (haslower && !hasupper) { |
|
346 |
+ return 'lower'; |
|
347 |
+ } else { |
|
348 |
+ if (v[0].match(/[A-Z]/) && !(v.slice(1).match(/[A-Z]/))) { |
|
349 |
+ return 'initial'; |
|
350 |
+ } else { |
|
351 |
+ return 'mixed'; |
|
352 |
+ } |
|
353 |
+ } |
|
354 |
+ }), |
|
355 |
+ uppercase: (item) => this.vals(item).flatMap((v) => v.toString().toUpperCase()), |
|
356 |
+ lowercase: (item) => this.vals(item).flatMap((v) => v.toString().toLowerCase().replaceAll(/[‘’`]/g, "'").replaceAll(/[“”]/g, '"')), |
|
357 |
+ list: (item) => Object.keys(item).filter((k) => k !== 'of'), |
|
358 |
+ items: (item) => item?.of?.flatMap((typename) => this.data?.[this.getname(typename)]), |
|
359 |
+ random: (item) => Math.random(), |
|
360 |
+ shuffle: (item) => { |
|
361 |
+ const newArray = [].concat(item?.of || []); |
|
362 |
+ for (let i = newArray.length - 1; i > 0; i--) { |
|
363 |
+ const j = Math.floor(Math.random() * (i + 1)); |
|
364 |
+ [newArray[i], newArray[j]] = [newArray[j], newArray[i]]; |
|
365 |
+ } |
|
366 |
+ return newArray; |
|
367 |
+ }, |
|
368 |
+ 'weighted shuffle': (item) => { |
|
369 |
+ return item.of.map((subitem) => ({weight: Number(subitem.weight || 1) * Math.random(), subitem: subitem})).sort((a, b) => b.weight - a.weight).map((ws) => ws.subitem); |
|
370 |
+ }, |
|
371 |
+ pick: (item) => item?.of?.[~~(Math.random() * item?.of?.length)], |
|
372 |
+ link: (item) => `<a href="${item.url}"${item.target ? ' target="' + item.target + '"' : ''}>${item.text}</a>`, |
|
373 |
+ img: (item) => { |
|
374 |
+ if (item.uri) { |
|
375 |
+ return `<a href="${item.uri}"><img src="${item.src}" height=${item.height}px width=${item.width}px></a>`; |
|
376 |
+ } else { |
|
377 |
+ return `<img src="${item.src}" height=${item.height}px width=${item.width}px>`; |
|
378 |
+ } |
|
379 |
+ }, |
|
380 |
+ sign: (item) => item.of.map((subitem) => { |
|
381 |
+ if (subitem.id) { |
|
382 |
+ return subitem; |
|
383 |
+ } else { |
|
384 |
+ const str = JSON.stringify(subitem); |
|
385 |
+ let hash = 0; |
|
386 |
+ for (let i = 0; i < str.length; i++) { |
|
387 |
+ const char = str.charCodeAt(i); |
|
388 |
+ hash = (hash << 5) - hash + char; |
|
389 |
+ } |
|
390 |
+ // Convert to 32bit unsigned integer in base 36 and pad with "0" to ensure length is 7. |
|
391 |
+ return Object.assign({id: (hash >>> 0).toString(36).padStart(7, '0')}, subitem); |
|
392 |
+ } |
|
393 |
+ }), |
|
394 |
+ 'parse query': (item) => item.of.map((subitem) => ({name: subitem.name, query: subitem.query, assembly: this.parse(subitem.query)})) |
|
395 |
+ } |
|
396 |
+ this.data.aggregators = Object.keys(this.aggregators); |
|
397 |
+ // this.cachehitcount = 0; |
|
398 |
+ // this.cachemisscount = {}; |
|
399 |
+ this.destinations = new Set(); |
|
400 |
+ } |
|
401 |
+ |
|
402 |
+ survey() { |
|
403 |
+ this.destinations = new Set(Object.keys(this.data).concat(this.data?.queries?.map((q) => q.name) || []).concat(Object.keys(this.adapters))); |
|
404 |
+ } |
|
405 |
+ |
|
406 |
+ vals(item) { |
|
407 |
+ const getname = this.getname; |
|
408 |
+ const itemvals = []; |
|
409 |
+ if (Object.keys(item).length > 1) { |
|
410 |
+ Object.entries(item).filter(([k, v]) => k !== 'of').flatMap(([k, v]) => Array.isArray(v) ? v : [v]).map(getname).forEach((v) => itemvals.push(v)); |
|
411 |
+ } else { |
|
412 |
+ item.of.map(getname).forEach((v) => itemvals.push(v)); |
|
413 |
+ } |
|
414 |
+ return itemvals; |
|
415 |
+ } |
|
416 |
+ |
|
417 |
+ numvals(item) { |
|
418 |
+ return this.vals(item).filter((val) => this.dtype(val, 'number')).map((val) => Number(val)); |
|
419 |
+ } |
|
420 |
+ |
|
421 |
+ async querylive(query, inputlist=null) { |
|
422 |
+ this.recache = new Set(); |
|
423 |
+ return await this.query(query, inputlist); |
|
424 |
+ } |
|
425 |
+ |
|
426 |
+ async query(query, inputlist=null, loop=50) { |
|
427 |
+ this.survey(); |
|
428 |
+ const operations = Array.isArray(query) ? query : this.assemble(this.tokenize(query)); |
|
429 |
+ const result = this.execute(inputlist, operations); |
|
430 |
+ this.data['current results'] = result; |
|
431 |
+ const queued = Object.keys(this.adapters).filter((key) => this.adapters[key].queue.length > 0); |
|
432 |
+ if (loop > 0 && queued.length > 0) { |
|
433 |
+ await this.adapt(); |
|
434 |
+ if (this.recache) queued.forEach((q) => this.recache.add(q)); |
|
435 |
+ return await this.query(operations, inputlist, loop - 1); |
|
436 |
+ } else if (loop === 0) { |
|
437 |
+ this.recache = false; |
|
438 |
+ Object.keys(this.adapters).forEach((key) => { |
|
439 |
+ if (!this.adapters[key].annotator) { |
|
440 |
+ this.adapters[key].queue.forEach((id) => { |
|
441 |
+ (this.index[key] ||= {})[id] = null; |
|
442 |
+ }) |
|
443 |
+ } |
|
444 |
+ }) |
|
445 |
+ } |
|
446 |
+ return result; |
|
447 |
+ } |
|
448 |
+ |
|
449 |
+ async adapt() { |
|
450 |
+ for (const key in this.adapters) { |
|
451 |
+ const adapter = this.adapters[key]; |
|
452 |
+ if (adapter.queue.length > 0) { |
|
453 |
+ if (adapter.annotator) { |
|
454 |
+ while (adapter.queue.length > 0) { |
|
455 |
+ const item_to_annotate = adapter.queue[0]; |
|
456 |
+ try { |
|
457 |
+ (this.index[key] ||= {})[this.getid(item_to_annotate)] = await adapter.f(item_to_annotate); |
|
458 |
+ this.index_modified = true; |
|
459 |
+ adapter.queue.shift(); |
|
460 |
+ } catch (e) {} |
|
461 |
+ } |
|
462 |
+ } else { |
|
463 |
+ const newids = adapter.queue.filter((id) => !adapter.pending.has(id)); |
|
464 |
+ newids.forEach((id) => adapter.pending.add(id)); |
|
465 |
+ const res = await adapter.f(newids); |
|
466 |
+ newids.forEach((id) => adapter.pending.delete(id)); |
|
467 |
+ for (let i=0; i<adapter.queue.length; i++) { |
|
468 |
+ const item = res[i]; |
|
469 |
+ const id = this.getid(item) ?? adapter.queue[i]; |
|
470 |
+ (this.index[key] ||= {})[id] = item; |
|
471 |
+ this.index_modified = true; |
|
472 |
+ if (key in this.data) { |
|
473 |
+ this.data[key].push(item); |
|
474 |
+ } |
|
475 |
+ } |
|
476 |
+ this.adapters[key].queue = []; |
|
477 |
+ } |
|
478 |
+ } |
|
479 |
+ } |
|
480 |
+ } |
|
481 |
+ |
|
482 |
+ async rehope() { |
|
483 |
+ Object.keys(this.index).forEach((k) => { |
|
484 |
+ const todelete = Object.keys(this.index[k]).filter((dk) => this.index[k][dk] == null || this.index[k][dk] === undefined); |
|
485 |
+ todelete.forEach((dk) => delete(this.index[k][dk])); |
|
486 |
+ }); |
|
487 |
+ } |
|
488 |
+ |
|
489 |
+ load(something, named, append=false) { |
|
490 |
+ if (!named) return |
|
491 |
+ if (!append || !(named in this.data)) this.data[named] = []; |
|
492 |
+ if (Array.isArray(something)) { |
|
493 |
+ something.forEach((somethingx) => this.data[named].push(somethingx)); |
|
494 |
+ } else if (typeof something == 'object') { |
|
495 |
+ const firstkey = Object.keys(something)[0]; |
|
496 |
+ const firstobj = something[firstkey]; |
|
497 |
+ if (typeof firstobj === 'object' && !Array.isArray(firstobj) && (firstkey in Object.values(firstobj) || 'id' in firstobj)) { |
|
498 |
+ this.data[named].push(...Object.values(something)); |
|
499 |
+ } else if (typeof firstobj === 'object' && !Array.isArray(firstobj)) { |
|
500 |
+ this.data[named].push(...Object.entries(firstobj).map(([key, val]) => ({id: key, ...val}))); |
|
501 |
+ } else if (typeof firstobj === 'string') { |
|
502 |
+ this.data[named].push(...Object.entries(something).map(([key, val]) => ({id: key, name: val}))); |
|
503 |
+ } else { |
|
504 |
+ this.data[named].push(...Object.entries(something).map(([key, val]) => ({id: key, value: val}))); |
|
505 |
+ } |
|
506 |
+ } else { |
|
507 |
+ this.data[named].push(something); |
|
508 |
+ } |
|
509 |
+ return this.data[named]; |
|
510 |
+ } |
|
511 |
+ |
|
512 |
+ async loadjsonl(something, named, append=false) { |
|
513 |
+ if (!named) return |
|
514 |
+ if (!append || !(named in this.data)) this.data[named] = []; |
|
515 |
+ if (typeof something == 'string' && (something.startsWith('http') || something.startsWith('file://'))) { |
|
516 |
+ const fetchres = await fetch(something); |
|
517 |
+ something = await fetchres.text(); |
|
518 |
+ } |
|
519 |
+ const rows = something.trim().split(/[\n\r]+/); |
|
520 |
+ for (const row of rows) { |
|
521 |
+ const rowdata = JSON.parse(row); |
|
522 |
+ if (rowdata) this.data[named].push(rowdata); |
|
523 |
+ } |
|
524 |
+ } |
|
525 |
+ |
|
526 |
+ async loadcsv(something, named, quoteChar = '"', delimiter = ',', headerrows=1) { |
|
527 |
+ if (typeof something == 'string' && (something.startsWith('http') || something.startsWith('file://'))) { |
|
528 |
+ const fetchres = await fetch(something); |
|
529 |
+ something = await fetchres.text(); |
|
530 |
+ } |
|
531 |
+ const rows = something.split(/[\n\r]+/); |
|
532 |
+ |
|
533 |
+ const regex = new RegExp(`\\s*(${quoteChar})?(.*?)\\1\\s*(?:${delimiter}|$)`, 'gs'); |
|
534 |
+ |
|
535 |
+ const match = (line) => Array.from(line.matchAll(regex), (m) => m[2]); |
|
536 |
+ |
|
537 |
+ const headers = []; |
|
538 |
+ for (let hrowx=0; hrowx<headerrows; hrowx++) { |
|
539 |
+ const hrow = rows.shift(); |
|
540 |
+ match(hrow).forEach((h, hx) => { |
|
541 |
+ if (hrowx === 0) { |
|
542 |
+ headers.push(h); |
|
543 |
+ } else { |
|
544 |
+ headers[hx] = headers[hx] + ' ' + h; |
|
545 |
+ } |
|
546 |
+ }); |
|
547 |
+ } |
|
548 |
+ const heads = headers ?? match(rows.shift()); |
|
549 |
+ const lines = rows.slice(0).filter((line) => line); |
|
550 |
+ const parsed = lines.map((line) => { |
|
551 |
+ return match(line).reduce((acc, cur, i) => { |
|
552 |
+ // replace blank matches with `null` |
|
553 |
+ const val = cur.length <= 0 ? null : (!isNaN(cur) ? Number(cur) : cur); |
|
554 |
+ const key = heads[i] ?? `{i}`; |
|
555 |
+ if (key === '') { |
|
556 |
+ return { ...acc}; |
|
557 |
+ } else { |
|
558 |
+ return { ...acc, [key]: val }; |
|
559 |
+ } |
|
560 |
+ }, {}); |
|
561 |
+ }); |
|
562 |
+ this.load(parsed, named); |
|
563 |
+ } |
|
564 |
+ |
|
565 |
+ apacheLogToDate(apacheTimestamp) { |
|
566 |
+ // Apache log format: [10/Oct/2000:13:55:36 -0700] |
|
567 |
+ // Remove brackets if present |
|
568 |
+ const cleanTimestamp = apacheTimestamp.replace(/^\[|]$/g, ''); |
|
569 |
+ |
|
570 |
+ // Split into date/time and timezone parts |
|
571 |
+ const [dateTimePart, timezone] = cleanTimestamp.split(' '); |
|
572 |
+ |
|
573 |
+ // Parse the date/time part: dd/MMM/yyyy:HH:mm:ss |
|
574 |
+ const [datePart, hour, minute, second] = dateTimePart.split(':'); |
|
575 |
+ const [day, month, year] = datePart.split('/'); |
|
576 |
+ |
|
577 |
+ // Month mapping |
|
578 |
+ const months = { |
|
579 |
+ 'Jan': 0, 'Feb': 1, 'Mar': 2, 'Apr': 3, 'May': 4, 'Jun': 5, |
|
580 |
+ 'Jul': 6, 'Aug': 7, 'Sep': 8, 'Oct': 9, 'Nov': 10, 'Dec': 11 |
|
581 |
+ }; |
|
582 |
+ |
|
583 |
+ // Create Date object (months are 0-indexed in JS) |
|
584 |
+ const date = new Date( |
|
585 |
+ parseInt(year), |
|
586 |
+ months[month], |
|
587 |
+ parseInt(day), |
|
588 |
+ parseInt(hour), |
|
589 |
+ parseInt(minute), |
|
590 |
+ parseInt(second) |
|
591 |
+ ); |
|
592 |
+ |
|
593 |
+ // Handle timezone offset if present |
|
594 |
+ if (timezone) { |
|
595 |
+ const sign = timezone[0] === '+' ? 1 : -1; |
|
596 |
+ const tzHours = parseInt(timezone.slice(1, 3)); |
|
597 |
+ const tzMinutes = parseInt(timezone.slice(3, 5)); |
|
598 |
+ const offsetMs = sign * (tzHours * 60 + tzMinutes) * 60 * 1000; |
|
599 |
+ |
|
600 |
+ // Adjust for timezone (Apache logs are in local time, JS Date assumes UTC) |
|
601 |
+ date.setTime(date.getTime() - offsetMs); |
|
602 |
+ } |
|
603 |
+ |
|
604 |
+ return date; |
|
605 |
+ } |
|
606 |
+ |
|
607 |
+ async loadclf(loglines, named, apiroutes=[]) { |
|
608 |
+ const lines = loglines.trim().split('\n'); |
|
609 |
+ const cols9 = ['ip', 'name', 'username', 'timestamp', 'requestraw', 'status', 'bytes', 'referrer', 'useragent']; |
|
610 |
+ const cols10 = ['host'].concat(cols9); |
|
611 |
+ const parsed = lines.map((line) => { |
|
612 |
+ const vals = Array.from(line.matchAll(/"(?:\\"|.)*?"|\[.*?]|\S+/g)).map((m) => m[0]); |
|
613 |
+ const cols = vals.length === 10 ? cols10 : cols9; |
|
614 |
+ const obj = Object.fromEntries(vals.map((v, vx) => ([cols[vx], v]))); |
|
615 |
+ const date = this.apacheLogToDate(obj.timestamp); |
|
616 |
+ obj.timestamp = date.toISOString(); |
|
617 |
+ [obj.date, obj.time] = obj.timestamp.slice(0, -1).split('T'); |
|
618 |
+ if (obj.requestraw.match(/ [^ ]+ /) && !obj.requestraw.startsWith('"{')) { |
|
619 |
+ [obj.method, obj.request, obj.protocol] = obj.requestraw.slice(1, -1).split(' '); |
|
620 |
+ if (this.data['.clf API routes']) { |
|
621 |
+ for (const apiroute of this.data['.clf API routes'].sort((a, b) => b.length - a.length)) { |
|
622 |
+ if (obj.request.startsWith(apiroute)) { |
|
623 |
+ obj.page = apiroute; |
|
624 |
+ break; |
|
625 |
+ } |
|
626 |
+ } |
|
627 |
+ } |
|
628 |
+ obj.page ??= obj.request.split('?')[0]; |
|
629 |
+ } |
|
630 |
+ obj.referrer = obj.referrer.slice(1, -1); |
|
631 |
+ obj.useragent = obj.useragent.slice(1, -1); |
|
632 |
+ obj.logline = line; |
|
633 |
+ return obj; |
|
634 |
+ }); |
|
635 |
+ this.load(parsed, named); |
|
636 |
+ } |
|
637 |
+ |
|
638 |
+ async loadrss(rsstext, named) { |
|
639 |
+ const rssval = (k, rawval) => { |
|
640 |
+ if (k.match(/date/i)) { |
|
641 |
+ return new Date(rawval).toISOString(); |
|
642 |
+ } else if (!isNaN(rawval)) { |
|
643 |
+ return Number(rawval); |
|
644 |
+ } else { |
|
645 |
+ return rawval; |
|
646 |
+ } |
|
647 |
+ } |
|
648 |
+ const rssdom = new window.DOMParser().parseFromString(rsstext, "text/xml"); |
|
649 |
+ const items = Array.from(rssdom.querySelectorAll('item')).map((i) => { |
|
650 |
+ const obj = {}; |
|
651 |
+ Array.from(i.children).forEach((c) => { |
|
652 |
+ const k = c.tagName; |
|
653 |
+ const rawval = c.textContent; |
|
654 |
+ obj[k] = rssval(k, rawval); |
|
655 |
+ }); |
|
656 |
+ return obj; |
|
657 |
+ }); |
|
658 |
+ this.load(items, named); |
|
659 |
+ } |
|
660 |
+ |
|
661 |
+ connect(key, adapter, annotator=false) { |
|
662 |
+ this.adapters[key] = {queue: [], pending: new Set(), f: adapter, annotator: annotator}; |
|
663 |
+ this.data.adapters ??= []; |
|
664 |
+ if (!this.data.adapters.includes(key)) this.data.adapters.push(key); |
|
665 |
+ } |
|
666 |
+ |
|
667 |
+ connect_annotator(key, adapter) { |
|
668 |
+ this.connect(key, adapter, true); |
|
669 |
+ } |
|
670 |
+ |
|
671 |
+ register(key, aggregator) { |
|
672 |
+ this.aggregators[key] = aggregator; |
|
673 |
+ this.data.aggregators ??= []; |
|
674 |
+ if (!this.data.aggregators.includes(key)) this.data.aggregators.push(key); |
|
675 |
+ } |
|
676 |
+ |
|
677 |
+ unbracket(token) { |
|
678 |
+ if (!(typeof token == 'string')) token = token.toString(); |
|
679 |
+ if (token.startsWith('[') && token.endsWith(']')) { |
|
680 |
+ return token.slice(1, -1).replaceAll(']]', ']'); |
|
681 |
+ } |
|
682 |
+ return token; |
|
683 |
+ } |
|
684 |
+ |
|
685 |
+ bracket(token) { |
|
686 |
+ if (!(typeof token == 'string')) token = token?.toString() ?? ''; |
|
687 |
+ if (token.match(/[?.:#\/|!<>=~@\[\](),;+-]/) || token.startsWith(' ') || token.endsWith(' ')) { |
|
688 |
+ return '[' + token.replaceAll(']', ']]') + ']'; |
|
689 |
+ } |
|
690 |
+ return token; |
|
691 |
+ } |
|
692 |
+ |
|
693 |
+ escapeRegExp(string) { |
|
694 |
+ return string.replace(/[.*+?${}()|[\]\\]/g, "\\$&"); |
|
695 |
+ } |
|
696 |
+ |
|
697 |
+ tokenize(text) { |
|
698 |
+ if (typeof text != 'string') text = String(text); |
|
699 |
+ const matches = text.matchAll(/(\[(?:]]|[^\]])*])|(\?{1,3})|(\.{1,4})|(\|{1,2})|(\/{1,2})|([:#!])|(\()|(\))|([@~=<>+-]+)|(,)|(;)|([^\[\]().?:\/#|!~=<>@,;+-]+)|([\[\]])/gms); |
|
700 |
+ return Array.from(matches, m => m[0].trim()).filter((token) => token.length > 0); |
|
701 |
+ } |
|
702 |
+ |
|
703 |
+ assemble(tokenized) { |
|
704 |
+ const isOperator = (token) => '???....://#||!'.includes(token); |
|
705 |
+ const tokenlist = '= ~ =~ ~< ~> > < <> >< >= <= - =- ~- + =+ @ @@ @- @= @@= =@ =@@ @< @@< @<= @@<= @> @@> @>= @@>= >> >>= << <<= ~~ =>'.split(' '); |
|
706 |
+ const isSubop = (token) => tokenlist.includes(token) || (token?.startsWith('-') && (token.length === 1 || tokenlist.includes(token.slice(1)))); |
|
707 |
+ const isSeparator = (token) => ',;'.includes(token); |
|
708 |
+ const isValue = (token) => !isOperator(token) && !isSubop(token) && !isSeparator(token); |
|
709 |
+ |
|
710 |
+ let tokens = tokenized.slice(); |
|
711 |
+ let level = 0; |
|
712 |
+ const operations = []; |
|
713 |
+ while (tokens.length > 0) { |
|
714 |
+ const op = {operator: null, args: []}; |
|
715 |
+ let token = tokens.shift(); |
|
716 |
+ if (isOperator(token) || ((isValue(token) || isSubop(token)) && operations.length === 0)) { |
|
717 |
+ if ((isValue(token) || isSubop(token)) && operations.length === 0) { |
|
718 |
+ op.operator = '?'; |
|
719 |
+ tokens.unshift(token); |
|
720 |
+ } else { |
|
721 |
+ op.operator = token; |
|
722 |
+ } |
|
723 |
+ while (tokens.length > 0 && !isOperator(tokens[0])) { |
|
724 |
+ const arg = {separator: null, label: null, subop: null, value: null}; |
|
725 |
+ if (isSeparator(tokens[0])) { |
|
726 |
+ arg.separator = tokens.shift(); |
|
727 |
+ } |
|
728 |
+ while (tokens.length > 0 && !isOperator(tokens[0]) && !isSeparator(tokens[0])) { |
|
729 |
+ const frag = tokens.shift(); |
|
730 |
+ if (frag !== '(' && isValue(frag) && isSubop(tokens[0])) { |
|
731 |
+ arg.label = this.unbracket(frag); |
|
732 |
+ arg.subop = tokens.shift(); |
|
733 |
+ } else if (isSubop(frag)) { |
|
734 |
+ arg.subop = frag; |
|
735 |
+ } else if (frag === '(') { |
|
736 |
+ const subquery = []; |
|
737 |
+ level++; |
|
738 |
+ while (tokens.length > 0 && level > 0) { |
|
739 |
+ const sub = tokens.shift(); |
|
740 |
+ if (sub === '(') { |
|
741 |
+ level++; |
|
742 |
+ if (level > 0) { |
|
743 |
+ subquery.push(sub); |
|
744 |
+ } |
|
745 |
+ } else if (sub === ')') { |
|
746 |
+ level--; |
|
747 |
+ if (level > 0) { |
|
748 |
+ subquery.push(sub); |
|
749 |
+ } |
|
750 |
+ } else if (level > 0) { |
|
751 |
+ subquery.push(sub); |
|
752 |
+ } |
|
753 |
+ } |
|
754 |
+ arg.value = this.assemble(subquery); |
|
755 |
+ } else if (!arg.value) { |
|
756 |
+ arg.value = this.unbracket(frag); |
|
757 |
+ } else { |
|
758 |
+ const failure = {tokens: tokenized, assembled: operations.slice(0), assembling: {op: op, arg: arg, unexpected: frag}, unassembled: tokens.slice(0)} |
|
759 |
+ throw new Error("Unexpected token", {cause: failure}); |
|
760 |
+ } |
|
761 |
+ } |
|
762 |
+ op.args.push(arg); |
|
763 |
+ } |
|
764 |
+ } |
|
765 |
+ operations.push(op); |
|
766 |
+ |
|
767 |
+ } |
|
768 |
+ if (level > 0) console.warn({parentropy: level, operations: operations}); |
|
769 |
+ return operations; |
|
770 |
+ } |
|
771 |
+ |
|
772 |
+ parse(querystr) { |
|
773 |
+ return this.assemble(this.tokenize(querystr)); |
|
774 |
+ } |
|
775 |
+ |
|
776 |
+ disassemble(assembly) { |
|
777 |
+ return assembly.map((operation, ox) => operation.operator + ((operation.args.length === 0 && ox < assembly.length - 1 && operation.operator[0] === assembly[ox + 1].operator[0]) ? ' ' : operation.args.map((arg) => (arg.separator ?? '') + (arg.label ? this.bracket(arg.label) : '') + (arg.subop ?? '') + (Array.isArray(arg.value) ? ('(' + this.disassemble(arg.value) + ')') : (arg.value ? this.bracket(arg.value) : ''))).join(''))).join(''); |
|
778 |
+ } |
|
779 |
+ |
|
780 |
+ compact(querystr) { |
|
781 |
+ return this.disassemble(this.parse(querystr)); |
|
782 |
+ } |
|
783 |
+ |
|
784 |
+ executeq(querystr) { |
|
785 |
+ return this.execute([], this.assemble(this.tokenize(querystr))); |
|
786 |
+ } |
|
787 |
+ |
|
788 |
+ timecheck(timer, i, count, op) { |
|
789 |
+ if (i === 1) timer.loopstart = new Date(); |
|
790 |
+ if (i >= 10) { |
|
791 |
+ const taken = new Date() - timer.loopstart; |
|
792 |
+ const projected = count * taken / i; |
|
793 |
+ if (projected > this.timelimit) throw new Error('Query overrun.', {cause: {operation: dactal.disassemble([op]), items: count, done: i, elapsed: taken + 'ms', projected: Math.round(projected / 60000) + ' minutes', timelimit: Math.round(this.timelimit / 60000) + ' minutes'}}); |
|
794 |
+ } |
|
795 |
+ } |
|
796 |
+ |
|
797 |
+ execute(inputlist, operations, labeled=null, level=null) { |
|
798 |
+ let currentlist = inputlist?.slice(0) || []; |
|
799 |
+ const getname = this.getname; |
|
800 |
+ const getid = this.getid; |
|
801 |
+ const dtype = this.dtype; |
|
802 |
+ const dcopy = this.dcopy; |
|
803 |
+ const step = this.step; |
|
804 |
+ const escapeRegExp = this.escapeRegExp; |
|
805 |
+ const unmmss = (mmssstr) => { |
|
806 |
+ const parts = mmssstr.split(':'); |
|
807 |
+ let s = 0; |
|
808 |
+ const seconds = parts.pop(); |
|
809 |
+ if (seconds) s += Number(seconds); |
|
810 |
+ const minutes = parts.pop(); |
|
811 |
+ if (minutes) s += 60 * Number(minutes); |
|
812 |
+ const hours = parts.pop(); |
|
813 |
+ if (hours) s += 60 * 60 * Number(hours); |
|
814 |
+ return s; |
|
815 |
+ }; |
|
816 |
+ const compvals = (araw, braw) => { |
|
817 |
+ const a = araw.toString(); |
|
818 |
+ const b = braw.toString(); |
|
819 |
+ const atime = a.match(/^\d+(?::\d{2,})+$/); |
|
820 |
+ const btime = b.match(/^\d+(?::\d{2,})+$/); |
|
821 |
+ if (atime && btime) { |
|
822 |
+ return unmmss(btime[0]) - unmmss(atime[0]); |
|
823 |
+ } |
|
824 |
+ return dethe(a).localeCompare(dethe(b)); |
|
825 |
+ } |
|
826 |
+ |
|
827 |
+ const dethe = (value) => value.toLowerCase().replace(/^the /, ''); |
|
828 |
+ |
|
829 |
+ labeled ||= {}; |
|
830 |
+ let outputlist = []; |
|
831 |
+ for (let opx=0; opx<operations.length; opx++) { |
|
832 |
+ const op = operations[opx]; |
|
833 |
+ outputlist = []; |
|
834 |
+ switch (op.operator) { |
|
835 |
+ case '?': // start |
|
836 |
+ if (op.args.length === 0) { |
|
837 |
+ outputlist = Object.keys(this.data).sort((a, b) => dethe(a).localeCompare(dethe(b))); |
|
838 |
+ break; |
|
839 |
+ } |
|
840 |
+ outputlist = []; |
|
841 |
+ op.args.forEach((arg) => { |
|
842 |
+ let startitems; |
|
843 |
+ if (arg.label && arg.subop?.match(/\+/)) labeled[arg.label] ||= []; |
|
844 |
+ if (arg.separator !== ';' || outputlist.length === 0) { |
|
845 |
+ if (Array.isArray(arg.value)) { |
|
846 |
+ startitems = this.execute([], arg.value, labeled); |
|
847 |
+ if (arg.label) { |
|
848 |
+ if (arg.subop?.match(/\+/)) { |
|
849 |
+ startitems.forEach((i) => labeled[arg.label].push(i)); |
|
850 |
+ } else { |
|
851 |
+ labeled[arg.label] = startitems; |
|
852 |
+ this.index[arg.label] = {}; |
|
853 |
+ } |
|
854 |
+ labeled[arg.label].forEach((i) => outputlist.push(i)); |
|
855 |
+ } else { |
|
856 |
+ startitems.forEach((i) => outputlist.push(i)); |
|
857 |
+ } |
|
858 |
+ } else if (arg.subop === '~') { |
|
859 |
+ outputlist.push(arg.value); |
|
860 |
+ } else if (arg.value in labeled) { |
|
861 |
+ labeled[arg.value].forEach((i) => outputlist.push(i)); |
|
862 |
+ } else if (startitems = this.gettype(arg.value, false)) { |
|
863 |
+ startitems.forEach((i) => outputlist.push(i)); |
|
864 |
+ } else if (arg.value?.trim().length > 0 && this.dtype(arg.value, 'number')) { |
|
865 |
+ if (arg.subop === '-' && arg.value === '0') { |
|
866 |
+ outputlist.push(-0); |
|
867 |
+ } else { |
|
868 |
+ outputlist.push((arg.subop === '-' ? -1 : 1) * Number(arg.value)); |
|
869 |
+ } |
|
870 |
+ } else if (arg.value && arg.subop !== '=') { |
|
871 |
+ outputlist.push(arg.value); |
|
872 |
+ } |
|
873 |
+ } |
|
874 |
+ }); |
|
875 |
+ break; |
|
876 |
+ case '??': // label |
|
877 |
+ outputlist = currentlist; |
|
878 |
+ let ended = false; |
|
879 |
+ op.args.forEach((arg) => { |
|
880 |
+ if (!ended) { |
|
881 |
+ if (arg.label === '_timelimit' && !isNaN(arg.value)) { |
|
882 |
+ this.timelimit = Number(arg.value) * 60000; |
|
883 |
+ } else if (arg.label) { |
|
884 |
+ if (arg.subop.includes('~') && dtype(arg.value, 'string')) { |
|
885 |
+ labeled[arg.label] = [arg.value]; |
|
886 |
+ } else if (arg.labeled) { |
|
887 |
+ labeled[arg.label] = arg.labeled; |
|
888 |
+ } else { |
|
889 |
+ if (arg.subop?.match(/\+/)) labeled[arg.label] ||= []; |
|
890 |
+ const newvals = this.execute(currentlist, Array.isArray(arg.value) ? arg.value : '.' + arg.value, labeled, level); |
|
891 |
+ if (arg.subop?.match(/\+/)) { |
|
892 |
+ newvals.forEach((nv) => labeled[arg.label].push(nv)); |
|
893 |
+ } else { |
|
894 |
+ labeled[arg.label] = newvals; |
|
895 |
+ } |
|
896 |
+ if (!level && opx === 0) arg.labeled = newvals; |
|
897 |
+ } |
|
898 |
+ this.index[arg.label] = {}; |
|
899 |
+ } else if (dtype(arg.value, 'string')) { |
|
900 |
+ if (arg.subop?.match(/\+/)) { |
|
901 |
+ labeled[arg.value] ??= []; |
|
902 |
+ currentlist.forEach((i) => labeled[arg.value].push(i)); |
|
903 |
+ } else { |
|
904 |
+ labeled[arg.value] = currentlist.slice(0); |
|
905 |
+ } |
|
906 |
+ this.index[arg.value] = {}; |
|
907 |
+ if (arg.value === 'end') { |
|
908 |
+ ended = true; |
|
909 |
+ } |
|
910 |
+ } |
|
911 |
+ } |
|
912 |
+ }); |
|
913 |
+ if (ended) return (this.debug && !inputlist) ? operations : outputlist; |
|
914 |
+ break; |
|
915 |
+ case '!': // repeat |
|
916 |
+ if (opx > 0) { |
|
917 |
+ const repeat_ops = operations.slice(opx - 1, opx + 1); |
|
918 |
+ outputlist = currentlist.slice(0); |
|
919 |
+ level ??= 0; |
|
920 |
+ level += 1; |
|
921 |
+ const maxrecursion = ((op.args.length > 0 && op.args[0].value) || 1000); |
|
922 |
+ if (outputlist.length > 0 && level < maxrecursion && (!this.samearray(inputlist, outputlist) || level === 1)) { |
|
923 |
+ const recursed = this.execute(outputlist, repeat_ops, labeled, level); |
|
924 |
+ if (recursed.length > 0 && !this.samearray(recursed, outputlist)) outputlist = recursed.slice(0); |
|
925 |
+ } |
|
926 |
+ } |
|
927 |
+ break; |
|
928 |
+ case '.': // traverse |
|
929 |
+ case '..': // traverse with duplicates |
|
930 |
+ const seen = new Set(); |
|
931 |
+ const sofar = Array.isArray(op.args?.[0]?.value) && op.args?.[0]?.label; |
|
932 |
+ let transq = null; |
|
933 |
+ if (sofar) labeled[sofar] = []; |
|
934 |
+ if (op.operator === '.' && op.args?.length === 1) { |
|
935 |
+ const firstarg = op.args[0]; |
|
936 |
+ const firstval = firstarg?.value; |
|
937 |
+ if (dtype(firstval, 'string')) { |
|
938 |
+ transq = this.data.queries?.find((q) => q.relative === true && q.name === firstval); |
|
939 |
+ } |
|
940 |
+ } |
|
941 |
+ if (transq) { |
|
942 |
+ outputlist = this.execute(currentlist, this.parse(transq.query), labeled, level); |
|
943 |
+ } else { |
|
944 |
+ const traversetimer = {}; |
|
945 |
+ outputlist = currentlist.reduce((acc, item, i) => { |
|
946 |
+ this.timecheck(traversetimer, i, currentlist.length, op); |
|
947 |
+ if (op.args.length === 0) { |
|
948 |
+ const itemkey = getid(item); |
|
949 |
+ if (op.operator === '..' || !seen.has(itemkey)) { |
|
950 |
+ acc.push(item); |
|
951 |
+ seen.add(itemkey); |
|
952 |
+ } |
|
953 |
+ return acc; |
|
954 |
+ } |
|
955 |
+ if (op.args[0].subop?.includes('<') && acc.length > 0) return acc; |
|
956 |
+ let itemvals = []; |
|
957 |
+ let toremoveids = {}; |
|
958 |
+ for (const arg of op.args.filter((arg) => arg.value != null)) { |
|
959 |
+ let passdown = null; |
|
960 |
+ if (arg.subop?.includes('>') && typeof arg.value == 'string') { |
|
961 |
+ if (arg?.label in item) { |
|
962 |
+ passdown = item[arg.label]; |
|
963 |
+ } else { |
|
964 |
+ passdown = JSON.parse(JSON.stringify(item, Object.keys(item).filter((k) => k !== arg.value))); |
|
965 |
+ } |
|
966 |
+ } |
|
967 |
+ if (arg.separator !== ';' || itemvals.length === 0) { |
|
968 |
+ const itemval = step(item, arg.value, arg.subop, labeled); |
|
969 |
+ if (arg.subop?.includes('-') && isNaN(arg.value)) { |
|
970 |
+ itemval.forEach((subitem) => { |
|
971 |
+ const subid = getid(subitem); |
|
972 |
+ toremoveids[subid] ||= 0; |
|
973 |
+ toremoveids[subid] += 1; |
|
974 |
+ }); |
|
975 |
+ } else { |
|
976 |
+ itemval.forEach((subitem) => { |
|
977 |
+ if (passdown) { |
|
978 |
+ if (!dtype(subitem, 'object')) subitem = {value: subitem}; |
|
979 |
+ if (arg.label) { |
|
980 |
+ subitem[arg.label] = [passdown]; |
|
981 |
+ } else { |
|
982 |
+ Object.assign(subitem, passdown); |
|
983 |
+ } |
|
984 |
+ } |
|
985 |
+ const subitemkey = getid(subitem); |
|
986 |
+ if (op.operator === '..' || !seen.has(subitemkey)) { |
|
987 |
+ seen.add(subitemkey); |
|
988 |
+ itemvals.push(subitem); |
|
989 |
+ if (sofar) labeled[sofar].push(subitem); |
|
990 |
+ } |
|
991 |
+ }) |
|
992 |
+ } |
|
993 |
+ } |
|
994 |
+ if (Object.keys(toremoveids).length > 0) { |
|
995 |
+ if (op.operator === '.') { |
|
996 |
+ itemvals = itemvals.filter((subitem) => !(getid(subitem) in toremoveids)); |
|
997 |
+ } else { |
|
998 |
+ itemvals = itemvals.filter((subitem) => { |
|
999 |
+ const subid = getid(subitem); |
|
1000 |
+ if (toremoveids[subid] > 0) { |
|
1001 |
+ toremoveids[subid] -= 1; |
|
1002 |
+ return false; |
|
1003 |
+ } else { |
|
1004 |
+ return true; |
|
1005 |
+ } |
|
1006 |
+ }) |
|
1007 |
+ } |
|
1008 |
+ } |
|
1009 |
+ } |
|
1010 |
+ itemvals.forEach((x) => acc.push(x)); |
|
1011 |
+ return acc; |
|
1012 |
+ }, []); |
|
1013 |
+ } |
|
1014 |
+ outputlist = outputlist.filter((item) => item != null); |
|
1015 |
+ break; |
|
1016 |
+ case ':': // filter |
|
1017 |
+ if (!op?.args?.length > 0) { |
|
1018 |
+ outputlist = currentlist; |
|
1019 |
+ break; |
|
1020 |
+ } |
|
1021 |
+ for(const arg of op.args) { |
|
1022 |
+ if (arg.subop && ['>>', '>>=', '<<', '<<=', '~~'].includes(arg.subop.replace(/^-/, ''))) { |
|
1023 |
+ arg.tests = {id: new Set(), name: new Set()}; |
|
1024 |
+ if (Array.isArray(arg.value)) { |
|
1025 |
+ const testitems = this.execute(currentlist, arg.value, labeled); |
|
1026 |
+ testitems.forEach((testitem) => arg.tests.id.add(getid(testitem))); |
|
1027 |
+ testitems.forEach((testitem) => arg.tests.name.add(getname(testitem))); |
|
1028 |
+ } |
|
1029 |
+ |
|
1030 |
+ if (['>>', '>>=', '<<', '<<='].includes(arg.subop)) { |
|
1031 |
+ for (const [tryval, tryf] of [['id', getid], ['name', getname]]) { |
|
1032 |
+ for (let j=0; j < currentlist.length; j++) { |
|
1033 |
+ if (arg.tests[tryval].has(tryf(currentlist[j]))) { |
|
1034 |
+ arg.matchpos = j; |
|
1035 |
+ break; |
|
1036 |
+ } |
|
1037 |
+ } |
|
1038 |
+ if (arg.matchpos) break |
|
1039 |
+ } |
|
1040 |
+ } |
|
1041 |
+ } else if ([null, '', '=', '~'].includes((arg.subop || '').replace(/-/, '')) && dtype(arg.value, 'string') && arg.value.startsWith('~')) { |
|
1042 |
+ const flags = arg.value.startsWith('~~') ? 'i' : ''; |
|
1043 |
+ const pattern = arg.value.replace(/^~*/, ''); |
|
1044 |
+ const fullpattern = (arg.subop || '').replace(/-/, '') === '=' ? ((pattern.startsWith('^') ? '' : '^') + pattern + (pattern.endsWith('$') ? '' : '$')) : pattern; |
|
1045 |
+ arg.re = new RegExp(fullpattern, flags); |
|
1046 |
+ } |
|
1047 |
+ } |
|
1048 |
+ |
|
1049 |
+ const ands = [[]]; |
|
1050 |
+ for (const arg of op.args) { |
|
1051 |
+ if (arg.separator === ';') { |
|
1052 |
+ ands.push([arg]); |
|
1053 |
+ } else { |
|
1054 |
+ ands[ands.length - 1].push(arg) |
|
1055 |
+ } |
|
1056 |
+ } |
|
1057 |
+ const filtertimer = {}; |
|
1058 |
+ outputlist = currentlist.filter((item, i) => { |
|
1059 |
+ this.timecheck(filtertimer, i, currentlist.length, op); |
|
1060 |
+ return ands.filter((and) => { |
|
1061 |
+ return and.filter((arg) => { |
|
1062 |
+ let comparator = arg.subop; |
|
1063 |
+ let polarize = (x) => x; |
|
1064 |
+ if (comparator?.startsWith('-') || comparator?.endsWith('-')) { |
|
1065 |
+ polarize = (x) => !x; |
|
1066 |
+ comparator = comparator.replace(/-|-$/, ''); |
|
1067 |
+ } |
|
1068 |
+ if (!comparator && arg.label == null && Array.isArray(arg.value)) { |
|
1069 |
+ return polarize(this.execute([item], arg.value, labeled).length > 0); |
|
1070 |
+ } else if (comparator === '~~') { |
|
1071 |
+ if (arg.label == null && dtype(arg.value, 'number')) { |
|
1072 |
+ return polarize(currentlist.length === Number(arg.value)); |
|
1073 |
+ } else { |
|
1074 |
+ if (arg.label) { |
|
1075 |
+ const argitems = step(item, arg.label, null, labeled); |
|
1076 |
+ return polarize(argitems.find((argitem) => arg.tests.id.has(getid(argitem)) || arg.tests.name.has(getname(argitem)))); |
|
1077 |
+ } else { |
|
1078 |
+ return polarize(arg.tests.id.has(getid(item)) || (item?.id === undefined && arg.tests.name.has(getname(item)))); |
|
1079 |
+ } |
|
1080 |
+ } |
|
1081 |
+ } else if (['~~', '<<', '<<=', '>>', '>>='].includes(comparator) && dtype(arg.value, 'number')) { |
|
1082 |
+ const argval = Number(arg.value); |
|
1083 |
+ let testlength; |
|
1084 |
+ if (typeof arg.label == 'string') { |
|
1085 |
+ testlength = step(item, arg.label, null, labeled).length; |
|
1086 |
+ } else { |
|
1087 |
+ testlength = currentlist.length; |
|
1088 |
+ } |
|
1089 |
+ switch (comparator) { |
|
1090 |
+ case '~~': return polarize(testlength === argval); |
|
1091 |
+ case '<<': return polarize(testlength < argval); |
|
1092 |
+ case '<<=': return polarize(testlength <= argval); |
|
1093 |
+ case '>>': return polarize(testlength > argval); |
|
1094 |
+ case '>>=': return polarize(testlength >= argval); |
|
1095 |
+ } |
|
1096 |
+ } else if (['+', ''].includes(comparator) && arg.label && !arg.value && dtype(arg.label, 'string') && dtype(item, 'object')) { |
|
1097 |
+ const propval = item[arg.label]; |
|
1098 |
+ // console.log({plusminus: comparator, label: arg.label, propval: propval}) |
|
1099 |
+ return polarize(Array.isArray(propval) ? propval.length > 0 : propval); |
|
1100 |
+ } |
|
1101 |
+ |
|
1102 |
+ let testitems = [item]; |
|
1103 |
+ if (arg.label && dtype(item, 'object')) { |
|
1104 |
+ let found = false; |
|
1105 |
+ for (const tryval of [arg.label, arg.label + 's']) { |
|
1106 |
+ if (item?.[tryval] != null) { |
|
1107 |
+ const propitems = item[tryval]; |
|
1108 |
+ if (propitems != null) { |
|
1109 |
+ found = true; |
|
1110 |
+ testitems = Array.isArray(propitems) ? propitems : [propitems]; |
|
1111 |
+ break; |
|
1112 |
+ } |
|
1113 |
+ } |
|
1114 |
+ } |
|
1115 |
+ if (!found) { |
|
1116 |
+ if (dactal.savedquerynames.has(arg.label) || (labeled['=>'] && arg.label in labeled['=>'])) { |
|
1117 |
+ testitems = step(item, arg.label, null, labeled); |
|
1118 |
+ } else { |
|
1119 |
+ testitems = []; |
|
1120 |
+ } |
|
1121 |
+ } |
|
1122 |
+ } |
|
1123 |
+ |
|
1124 |
+ const testvals = testitems.map((testitem) => { |
|
1125 |
+ if (comparator || arg.re) { |
|
1126 |
+ return getname(testitem); |
|
1127 |
+ } else if (this.dtype(testitem, 'literal')) { |
|
1128 |
+ return testitem; |
|
1129 |
+ } else if ('id' in testitem) { |
|
1130 |
+ return testitem.id; |
|
1131 |
+ } else { |
|
1132 |
+ return getname(testitem) |
|
1133 |
+ } |
|
1134 |
+ }); |
|
1135 |
+ |
|
1136 |
+ let argvals; |
|
1137 |
+ if (Array.isArray(arg.value)) { |
|
1138 |
+ const argvalitems = this.execute([item], arg.value, labeled); |
|
1139 |
+ argvals = argvalitems.map((argvalitem) => { |
|
1140 |
+ let argval; |
|
1141 |
+ if (dtype(argvalitem, 'literal')) { |
|
1142 |
+ argval = argvalitem; |
|
1143 |
+ } else { |
|
1144 |
+ argval = getname(argvalitem); |
|
1145 |
+ } |
|
1146 |
+ if (argval == null) { |
|
1147 |
+ const argvalitemkeys = Object.keys(argvalitem).filter((key) => key !== 'id'); |
|
1148 |
+ if (argvalitemkeys.length === 1) { |
|
1149 |
+ argval = argvalitem[argvalitemkeys[0]]; |
|
1150 |
+ } |
|
1151 |
+ } |
|
1152 |
+ return argval; |
|
1153 |
+ }); |
|
1154 |
+ } |
|
1155 |
+ |
|
1156 |
+ return testvals.find((testval) => { |
|
1157 |
+ if (!argvals) { |
|
1158 |
+ if (dtype(testval, 'number') && dtype(arg.value, 'number')) { |
|
1159 |
+ testval = Number(testval); |
|
1160 |
+ argvals = [Number(arg.value)]; |
|
1161 |
+ } else { |
|
1162 |
+ argvals = [arg.value]; |
|
1163 |
+ } |
|
1164 |
+ } |
|
1165 |
+ if (comparator?.startsWith('@')) { |
|
1166 |
+ if (dtype(arg.value, 'number')) { |
|
1167 |
+ argvals = [Number(arg.value)]; |
|
1168 |
+ } else if (arg.value in labeled) { |
|
1169 |
+ argvals = Number(labeled[arg.value]); |
|
1170 |
+ if (!Array.isArray(argvals)) argvals = [argvals]; |
|
1171 |
+ } |
|
1172 |
+ if (comparator.startsWith('@@')) { |
|
1173 |
+ testval = currentlist.length - i; |
|
1174 |
+ comparator = comparator.slice(2); |
|
1175 |
+ } else { |
|
1176 |
+ testval = i + 1; |
|
1177 |
+ comparator = comparator.slice(1); |
|
1178 |
+ } |
|
1179 |
+ } else if (['<<', '<<=', '>>', '>>='].includes(comparator) && dtype(arg.value, 'number')) { |
|
1180 |
+ const argval = Number(arg.value); |
|
1181 |
+ switch (comparator) { |
|
1182 |
+ case '<<': return polarize(currentlist.length < argval); |
|
1183 |
+ case '<<=': return polarize(currentlist.length <= argval); |
|
1184 |
+ case '>>': return polarize(currentlist.length > argval); |
|
1185 |
+ case '>>=': return polarize(currentlist.length >= argval); |
|
1186 |
+ } |
|
1187 |
+ } |
|
1188 |
+ if (testval == null || argvals == null || argvals.length === 0) return false; |
|
1189 |
+ return argvals.find((argval) => { |
|
1190 |
+ if (arg.re) { |
|
1191 |
+ return polarize(testval.match(arg.re)); |
|
1192 |
+ } else { |
|
1193 |
+ switch (comparator) { |
|
1194 |
+ case null: return polarize(testval === argval); |
|
1195 |
+ case '': return polarize(testval === argval); |
|
1196 |
+ case '=': return polarize(testval === argval); |
|
1197 |
+ case '>=': return polarize(testval >= argval); |
|
1198 |
+ case '<=': return polarize(testval <= argval); |
|
1199 |
+ case '>': return polarize(testval > argval); |
|
1200 |
+ case '<': return polarize(testval < argval); |
|
1201 |
+ case '~': return polarize(testval?.toString().toLowerCase().includes(argval?.toString().toLowerCase())); |
|
1202 |
+ case '~<': return polarize(testval?.toString().toLowerCase().startsWith(argval?.toString().toLowerCase())); |
|
1203 |
+ case '~>': return polarize(testval?.toString().toLowerCase().endsWith(argval?.toString().toLowerCase())); |
|
1204 |
+ case '>>': return polarize(i > arg.matchpos); |
|
1205 |
+ case '>>=': return polarize(i >= arg.matchpos); |
|
1206 |
+ case '<<': return polarize(i < arg.matchpos); |
|
1207 |
+ case '<<=': return polarize(i <= arg.matchpos); |
|
1208 |
+ } |
|
1209 |
+ } |
|
1210 |
+ }) != null; |
|
1211 |
+ }) != null; |
|
1212 |
+ }).length > 0; |
|
1213 |
+ }).length === ands.length; |
|
1214 |
+ }); |
|
1215 |
+ break; |
|
1216 |
+ case '#': // sort |
|
1217 |
+ const sortargs = op.args.slice(0); |
|
1218 |
+ const lastarg = sortargs[sortargs.length - 1]; |
|
1219 |
+ let temped = false; |
|
1220 |
+ if (!lastarg || ![';', '=;'].includes(Object.values(lastarg).join(''))) sortargs.push(...[{subop: null, value: 'name'}, {subop: null, value: 'id'}]); |
|
1221 |
+ if (!dtype(currentlist[0], 'object') && lastarg && lastarg.separator === ';') { |
|
1222 |
+ currentlist = currentlist.map((v) => ({_value: v})); |
|
1223 |
+ temped = true; |
|
1224 |
+ } else if (sortargs[0]?.label) { |
|
1225 |
+ currentlist = currentlist.map((i) => dcopy(i)); |
|
1226 |
+ } |
|
1227 |
+ sortargs.forEach((arg) => { |
|
1228 |
+ arg.extraindex = {}; |
|
1229 |
+ const stablesort = (arg.label == null && arg.value == null && arg.separator === ';' ) ? (arg.subop === '-' ? -1 : 1) : null; |
|
1230 |
+ if (arg.subop?.includes('~')) { |
|
1231 |
+ arg.sortmode = 'literal'; |
|
1232 |
+ } else if (arg.subop?.endsWith('-')) { |
|
1233 |
+ arg.sortmode = 'numeric'; |
|
1234 |
+ } else if (arg.subop?.endsWith('+')) { |
|
1235 |
+ arg.sortmode = 'rank'; |
|
1236 |
+ } else { |
|
1237 |
+ arg.sortmode = stablesort || ['rank', 'index', 'number', 'id'].includes(arg.value) ? 'rank' : 'numeric'; |
|
1238 |
+ } |
|
1239 |
+ const vals = new Set(); |
|
1240 |
+ const extradone = new Set(); |
|
1241 |
+ for (let ix=0; ix<currentlist.length; ix++) { |
|
1242 |
+ const item = currentlist[ix]; |
|
1243 |
+ const vallist = stablesort ? [ix] : (arg.value ? step(item, arg.value, null, labeled) : (Array.isArray(item) ? item : [item])); |
|
1244 |
+ if (vallist && !stablesort) vallist.forEach((val) => vals.add(val)); |
|
1245 |
+ if (typeof item == 'object') { |
|
1246 |
+ (item._sortindex ||= []).push(vallist); |
|
1247 |
+ } else if (!dtype(item, 'number') && !extradone.has(item)) { |
|
1248 |
+ arg.extraindex[item] = vallist; |
|
1249 |
+ extradone.add(item); |
|
1250 |
+ } |
|
1251 |
+ } |
|
1252 |
+ if (arg.sortmode !== 'literal') { |
|
1253 |
+ for (const val of vals) { |
|
1254 |
+ if (val != null && !dtype(val, 'number') && !(dtype(val, 'object') && dtype(getname(val), 'number'))) { |
|
1255 |
+ arg.sortmode = null; |
|
1256 |
+ break; |
|
1257 |
+ } |
|
1258 |
+ } |
|
1259 |
+ } |
|
1260 |
+ }) |
|
1261 |
+ |
|
1262 |
+ outputlist = currentlist.sort((a, b) => { |
|
1263 |
+ let comp = 0; |
|
1264 |
+ let ii = 0; |
|
1265 |
+ for (const arg of sortargs) { |
|
1266 |
+ const alist = typeof a == 'object' ? a._sortindex[ii] : dtype(a, 'number') ? [Number(a)] : arg.extraindex[a]; |
|
1267 |
+ const blist = typeof b == 'object' ? b._sortindex[ii] : dtype(b, 'number') ? [Number(b)] : arg.extraindex[b]; |
|
1268 |
+ ii++; |
|
1269 |
+ if (alist && blist) { |
|
1270 |
+ if (arg?.subop?.includes('@')) { |
|
1271 |
+ const alookup = alist.indexOf(getname(a)); |
|
1272 |
+ const blookup = blist.indexOf(getname(b)); |
|
1273 |
+ if (alookup >-1 && blookup === -1) { |
|
1274 |
+ comp = -1; |
|
1275 |
+ } else if (alookup === -1 && blookup > -1) { |
|
1276 |
+ comp = 1; |
|
1277 |
+ } else { |
|
1278 |
+ comp = alist.indexOf(getname(a)) - alist.indexOf(getname(b)) |
|
1279 |
+ } |
|
1280 |
+ } else { |
|
1281 |
+ for (let i=0; i < Math.min(alist.length, blist.length); i++) { |
|
1282 |
+ let aitem = alist[i]; |
|
1283 |
+ let bitem = blist[i]; |
|
1284 |
+ if (aitem != null && bitem == null) { |
|
1285 |
+ comp = -1; |
|
1286 |
+ } else if (aitem == null && bitem != null) { |
|
1287 |
+ comp = 1; |
|
1288 |
+ } else { |
|
1289 |
+ switch (arg.sortmode) { |
|
1290 |
+ case 'literal': |
|
1291 |
+ comp = aitem < bitem ? -1 : (bitem < aitem ? 1 : 0); |
|
1292 |
+ break; |
|
1293 |
+ case 'numeric': |
|
1294 |
+ let bnum = Number(bitem); |
|
1295 |
+ let anum = Number(aitem); |
|
1296 |
+ if (isNaN(bnum) || isNaN(anum)) { |
|
1297 |
+ bnum = Number(getname(bitem)); |
|
1298 |
+ anum = Number(getname(aitem)); |
|
1299 |
+ } |
|
1300 |
+ if (isNaN(bnum) || isNaN(anum)) { |
|
1301 |
+ comp = compvals(aitem, bitem) |
|
1302 |
+ } else { |
|
1303 |
+ comp = bnum - anum; |
|
1304 |
+ } |
|
1305 |
+ break; |
|
1306 |
+ case 'rank': |
|
1307 |
+ comp = Number(aitem) - Number(bitem); |
|
1308 |
+ break; |
|
1309 |
+ default: |
|
1310 |
+ if (dtype(aitem, 'literal') && dtype(bitem, 'literal')) { |
|
1311 |
+ comp = compvals(aitem, bitem); |
|
1312 |
+ } else if (typeof aitem == 'object' && typeof bitem == 'object') { |
|
1313 |
+ let aval = getname(aitem); |
|
1314 |
+ let bval = getname(bitem); |
|
1315 |
+ if (aval != null && bval != null) { |
|
1316 |
+ comp = compvals(aval, bval); |
|
1317 |
+ } else { |
|
1318 |
+ aval = getid(aitem); |
|
1319 |
+ bval = getid(bitem); |
|
1320 |
+ if (aval != null && bval != null) { |
|
1321 |
+ comp = compvals(aval, bval); |
|
1322 |
+ } else { |
|
1323 |
+ comp = 0; |
|
1324 |
+ } |
|
1325 |
+ } |
|
1326 |
+ } else if (aitem != null && bitem == null) { |
|
1327 |
+ comp = -1; |
|
1328 |
+ } else if (aitem == null && bitem != null) { |
|
1329 |
+ comp = 1; |
|
1330 |
+ } |
|
1331 |
+ break; |
|
1332 |
+ } |
|
1333 |
+ } |
|
1334 |
+ if (comp !== 0) break; |
|
1335 |
+ } |
|
1336 |
+ } |
|
1337 |
+ } |
|
1338 |
+ if (comp === 0) { |
|
1339 |
+ if (alist && blist) { |
|
1340 |
+ comp = blist.length - alist.length; |
|
1341 |
+ } else if (alist) { |
|
1342 |
+ comp = -1; |
|
1343 |
+ } else if (blist) { |
|
1344 |
+ comp = 1; |
|
1345 |
+ } |
|
1346 |
+ } |
|
1347 |
+ if (arg?.subop?.includes('-') && (arg.sortmode === 'literal' || !arg.sortmode)) { |
|
1348 |
+ comp = -comp; |
|
1349 |
+ } |
|
1350 |
+ if (comp !== 0) break; |
|
1351 |
+ } |
|
1352 |
+ return comp; |
|
1353 |
+ }) |
|
1354 |
+ if (temped) outputlist = outputlist.map((vt) => vt._value); |
|
1355 |
+ outputlist.forEach((item, i) => { |
|
1356 |
+ delete item._sortindex; |
|
1357 |
+ if (op.args.length > 0 && op.args[0].label) item[op.args[0].label] = i + 1; |
|
1358 |
+ }); |
|
1359 |
+ break; |
|
1360 |
+ case '/': // group |
|
1361 |
+ case '//': // merge |
|
1362 |
+ const groupindex = {}; |
|
1363 |
+ let keylists = {}; |
|
1364 |
+ let ofname = 'of'; |
|
1365 |
+ let countname = 'count'; |
|
1366 |
+ let sortgroups = true; |
|
1367 |
+ const groupargs = []; |
|
1368 |
+ const merge_ands = []; |
|
1369 |
+ for (const arg of op.args) { |
|
1370 |
+ if (arg.separator === ';' && arg.label == null && arg.value == null) { |
|
1371 |
+ sortgroups = false; |
|
1372 |
+ } else if (op.operator === '//' && (arg.separator === ';' || merge_ands.length > 0)) { |
|
1373 |
+ if (arg.separator === ';') { |
|
1374 |
+ merge_ands.push([arg.value]); |
|
1375 |
+ } else { |
|
1376 |
+ merge_ands[merge_ands.length - 1].push(arg.value) |
|
1377 |
+ } |
|
1378 |
+ } else { |
|
1379 |
+ groupargs.push(arg); |
|
1380 |
+ } |
|
1381 |
+ } |
|
1382 |
+ if (groupargs.length === 0) groupargs.push({value: null}); |
|
1383 |
+ |
|
1384 |
+ groupargs.forEach((arg) => arg.groupcounter = 0); |
|
1385 |
+ |
|
1386 |
+ const grouptimer = {}; |
|
1387 |
+ for (let ix=0; ix<currentlist.length; ix++) { |
|
1388 |
+ this.timecheck(grouptimer, ix, currentlist.length, op); |
|
1389 |
+ const item = currentlist[ix]; |
|
1390 |
+ let keys = null; |
|
1391 |
+ let keyi = 0; |
|
1392 |
+ for (const arg of groupargs) { |
|
1393 |
+ if (arg.label === 'of') { |
|
1394 |
+ ofname = arg.value; |
|
1395 |
+ continue; |
|
1396 |
+ } else if (arg.label === 'count') { |
|
1397 |
+ countname = arg.value; |
|
1398 |
+ continue; |
|
1399 |
+ } |
|
1400 |
+ keyi++; |
|
1401 |
+ let groupnumber = null; |
|
1402 |
+ if (op.operator === '/' && dtype(arg.value, 'number')) { |
|
1403 |
+ if (arg.subop?.endsWith('@')) { |
|
1404 |
+ arg.divisor = Number(arg.value); |
|
1405 |
+ } else { |
|
1406 |
+ arg.divisor = currentlist.length / Number(arg.value); |
|
1407 |
+ } |
|
1408 |
+ groupnumber = Math.floor(ix / arg.divisor) + 1; |
|
1409 |
+ } else if (arg.value != null && arg.subop?.endsWith('@@')) { |
|
1410 |
+ const groupval = step(item, arg.value, null, labeled); |
|
1411 |
+ if (ix === 0 || groupval?.length > 0) arg.groupcounter += 1; |
|
1412 |
+ groupnumber = arg.groupcounter; |
|
1413 |
+ } |
|
1414 |
+ const label = arg.label ?? (typeof arg.value === 'string' ? arg.value : null) ?? keyi; |
|
1415 |
+ const newkeyitems = groupnumber != null ? [groupnumber] : arg.value ? step(item, arg.value, arg.subop, labeled) : [null]; |
|
1416 |
+ const newkeys = newkeyitems.map((newkey) => ([{arglabel: arg.label, label: label, keyitem: this.resolve(arg.value, newkey, labeled)}])); |
|
1417 |
+ if (keys) { |
|
1418 |
+ keys = keys.flatMap((oldkeys) => newkeys.filter((newkey) => arg.separator === ',' || oldkeys.filter((oldkey) => compvals(getname(oldkey.keyitem) || oldkey.keyitem, getname(newkey[0].keyitem) || newkey[0].keyitem) >= 0).length === 0).map((newkey) => oldkeys.concat(newkey))); |
|
1419 |
+ } else { |
|
1420 |
+ keys = newkeys; |
|
1421 |
+ } |
|
1422 |
+ if (arg.subop?.endsWith('@')) { |
|
1423 |
+ keys.forEach((key) => { |
|
1424 |
+ const testkey = JSON.stringify(key.slice(0, -1)); |
|
1425 |
+ const newkeytest = JSON.stringify(key[key.length - 1]); |
|
1426 |
+ if (testkey in keylists) { |
|
1427 |
+ const lastkey = keylists[testkey][keylists[testkey].length - 1]; |
|
1428 |
+ if (newkeytest !== lastkey) keylists[testkey].push(newkeytest); |
|
1429 |
+ } else { |
|
1430 |
+ keylists[testkey] = [newkeytest]; |
|
1431 |
+ } |
|
1432 |
+ key[key.length - 1].keyindex = keylists[testkey].length; |
|
1433 |
+ }); |
|
1434 |
+ } |
|
1435 |
+ } |
|
1436 |
+ if (keys) { |
|
1437 |
+ for (const key of keys) { |
|
1438 |
+ const keystr = JSON.stringify(key); |
|
1439 |
+ (groupindex[keystr] ||= []).push(item); |
|
1440 |
+ } |
|
1441 |
+ } |
|
1442 |
+ } |
|
1443 |
+ for (const [keystr, items] of Object.entries(groupindex)) { |
|
1444 |
+ if (op.operator === '/') { |
|
1445 |
+ const newgroup = {}; |
|
1446 |
+ const keydata = JSON.parse(keystr); |
|
1447 |
+ if (keydata.length === 1 && keydata[0].keyitem != null) { |
|
1448 |
+ let keyitemname = keydata[0].keyindex || getname(keydata[0].keyitem); |
|
1449 |
+ if (keyitemname != null) { |
|
1450 |
+ newgroup.name = keyitemname; |
|
1451 |
+ } |
|
1452 |
+ } |
|
1453 |
+ let skip = false; |
|
1454 |
+ let keys = []; |
|
1455 |
+ for (const {arglabel, label, keyitem, keyindex} of keydata) { |
|
1456 |
+ if (keyitem != null) { |
|
1457 |
+ const keyobj = (dtype(keyitem, 'object') || keyindex == null || false) ? keyitem : {name: keyitem}; |
|
1458 |
+ if (keyindex != null) { |
|
1459 |
+ keyobj.index = keyindex; |
|
1460 |
+ } |
|
1461 |
+ keys.push(keyobj) |
|
1462 |
+ if (label && dtype(label, 'string') && isNaN(label) && label !== '_') { |
|
1463 |
+ newgroup[label] = [keyobj]; |
|
1464 |
+ } |
|
1465 |
+ } |
|
1466 |
+ } |
|
1467 |
+ newgroup[countname] = items.length; |
|
1468 |
+ if (keys) newgroup.key = keys; |
|
1469 |
+ newgroup[ofname] = items; |
|
1470 |
+ outputlist.push(newgroup); |
|
1471 |
+ } else if (op.operator === '//') { |
|
1472 |
+ if (merge_ands.length === 0 || !merge_ands.find((mand) => !mand.find((mr) => items.find((item) => mr in item)))) { |
|
1473 |
+ const newgroup = items.reduce((acc, item) => { |
|
1474 |
+ for(const prop in item) { |
|
1475 |
+ if (prop in acc) { |
|
1476 |
+ if (Array.isArray(acc[prop])) { |
|
1477 |
+ (Array.isArray(item[prop]) ? item[prop] : [item[prop]]).filter((val) => !acc[prop].includes(val)).forEach((val) => acc[prop].push(val)); |
|
1478 |
+ } |
|
1479 |
+ } else { |
|
1480 |
+ acc[prop] = item[prop]; |
|
1481 |
+ } |
|
1482 |
+ } |
|
1483 |
+ return acc; |
|
1484 |
+ }, {}); |
|
1485 |
+ const keydata = JSON.parse(keystr); |
|
1486 |
+ for (const {arglabel, label, keyitem, keyindex} of keydata) { |
|
1487 |
+ if (arglabel && typeof arglabel === 'string' && arglabel !== '_') { |
|
1488 |
+ newgroup[arglabel] = [keyitem]; |
|
1489 |
+ } |
|
1490 |
+ } |
|
1491 |
+ outputlist.push(newgroup); |
|
1492 |
+ } |
|
1493 |
+ } |
|
1494 |
+ } |
|
1495 |
+ if (sortgroups) { |
|
1496 |
+ const sortquery = '#' + groupargs.map((arg, argx) => (arg.subop?.endsWith('@') || arg.divisor ? '+' : '') + '(..key:@' + (argx + 1) + (arg.subop?.endsWith('@') ? '.index;_' : '') + ')').join(','); |
|
1497 |
+ outputlist = this.execute(outputlist, this.assemble(this.tokenize(sortquery)), labeled); |
|
1498 |
+ } |
|
1499 |
+ break; |
|
1500 |
+ case '...': // aggregate |
|
1501 |
+ case '....': // aggregate to value |
|
1502 |
+ if (!(op?.args?.length > 0)) { |
|
1503 |
+ if (op.operator === '...') { |
|
1504 |
+ outputlist = [{of: currentlist.map((item) => dcopy(item))}]; |
|
1505 |
+ } else { |
|
1506 |
+ outputlist = [currentlist.length] |
|
1507 |
+ } |
|
1508 |
+ break; |
|
1509 |
+ } |
|
1510 |
+ const tempitem = {}; |
|
1511 |
+ if (op.operator === '...') { |
|
1512 |
+ tempitem.of = currentlist.map((item) => dcopy(item)); |
|
1513 |
+ } else { |
|
1514 |
+ tempitem.of = currentlist; |
|
1515 |
+ } |
|
1516 |
+ const finalitem = {}; |
|
1517 |
+ let aggregated = null; |
|
1518 |
+ let afteraggregated = 0; |
|
1519 |
+ let finalvalue = null; |
|
1520 |
+ op.args.forEach((arg, propx) => { |
|
1521 |
+ if (dtype(arg.value, 'string') && arg.subop === '~') { |
|
1522 |
+ tempitem[arg.label || '_' + (propx + 1).toString()] = [arg.value]; |
|
1523 |
+ } else { |
|
1524 |
+ const prop = arg.label ?? (typeof arg.value === 'string' ? arg.value : null) ?? '_' + (propx + 1).toString(); |
|
1525 |
+ const aggname = (arg.label == null && typeof arg.value == 'string' && arg.value) || (arg.value == null && arg.label); |
|
1526 |
+ const firstitem = currentlist[0]; |
|
1527 |
+ const isprop = typeof firstitem == 'object' && prop in firstitem; |
|
1528 |
+ if ((arg.separator === ';' || !isprop) && aggname in this.aggregators) { |
|
1529 |
+ const aggval = this.aggregators[aggname](tempitem); |
|
1530 |
+ const agglabel = arg.label || aggname; |
|
1531 |
+ if (aggval != null) finalitem[agglabel] = aggval; |
|
1532 |
+ aggregated = agglabel; |
|
1533 |
+ finalvalue = dcopy(aggval); |
|
1534 |
+ } else if (!aggregated) { |
|
1535 |
+ let propitems; |
|
1536 |
+ if (op === operations[0] && op.operator === '....' && arg.label && !Array.isArray(arg.value) && !arg.value.startsWith('=')) { |
|
1537 |
+ propitems = [arg.value] |
|
1538 |
+ } else { |
|
1539 |
+ propitems = this.execute(currentlist, Array.isArray(arg.value) ? arg.value.map((x) => structuredClone(x)) : [{operator: '..', args: [{value: arg.value}]}], labeled) |
|
1540 |
+ } |
|
1541 |
+ tempitem[prop] = propitems; |
|
1542 |
+ } else if (aggregated in finalitem && Array.isArray(finalitem[aggregated]) && finalitem[aggregated].length > afteraggregated) { |
|
1543 |
+ tempitem[prop] = finalitem[aggregated][afteraggregated]; |
|
1544 |
+ afteraggregated++; |
|
1545 |
+ } |
|
1546 |
+ } |
|
1547 |
+ }); |
|
1548 |
+ if (op.operator === '....' && finalvalue != null) { |
|
1549 |
+ if (this.dtype(finalvalue, 'array')) { |
|
1550 |
+ outputlist = finalvalue; |
|
1551 |
+ } else { |
|
1552 |
+ outputlist = [finalvalue]; |
|
1553 |
+ } |
|
1554 |
+ } else { |
|
1555 |
+ for (const prop in tempitem) { |
|
1556 |
+ if (!(prop in finalitem) && prop !== 'of') finalitem[prop] = tempitem[prop]; |
|
1557 |
+ } |
|
1558 |
+ if (op.operator === '...') { |
|
1559 |
+ finalitem.of = tempitem.of; |
|
1560 |
+ } else { |
|
1561 |
+ for (const prop in finalitem) { |
|
1562 |
+ if (Array.isArray(finalitem[prop]) && finalitem[prop].length === 1) finalitem[prop] = finalitem[prop][0]; |
|
1563 |
+ } |
|
1564 |
+ } |
|
1565 |
+ outputlist = [finalitem]; |
|
1566 |
+ } |
|
1567 |
+ break; |
|
1568 |
+ case '|': // annotate |
|
1569 |
+ case '||': // annotate with values |
|
1570 |
+ outputlist = currentlist.slice(0).map((item) => dcopy(item)); |
|
1571 |
+ op.args.filter((arg) => arg.subop?.includes('>') && arg.label != null && true && Array.isArray(arg.value)) |
|
1572 |
+ .forEach((arg) => (labeled['=>'] ??= {})[arg.label] = arg.value); |
|
1573 |
+ |
|
1574 |
+ const annotatetimer = {}; |
|
1575 |
+ for (const i in outputlist) { |
|
1576 |
+ this.timecheck(annotatetimer, i, outputlist.length, op); |
|
1577 |
+ const baseitem = outputlist[i]; |
|
1578 |
+ if (typeof baseitem != 'object') { |
|
1579 |
+ outputlist[i] = {}; |
|
1580 |
+ if (op.args?.[0]?.value !== '_') outputlist[i].name = baseitem; |
|
1581 |
+ } |
|
1582 |
+ const item = outputlist[i]; |
|
1583 |
+ const newprops = op.args.filter((arg) => arg.subop !== '<' && (arg.label || !arg.subop?.includes('-'))).map((arg) => arg.label || arg.value); |
|
1584 |
+ if (typeof item == 'object') { |
|
1585 |
+ let argx = -1; |
|
1586 |
+ for (const arg of op.args) { |
|
1587 |
+ argx++; |
|
1588 |
+ if (arg?.subop === '<' && dtype(arg.value, 'string') && Array.isArray(item[arg.value]) && argx < op.args.length - 1) { |
|
1589 |
+ const sublabeled = {...labeled}; |
|
1590 |
+ if (arg.label) sublabeled[arg.label] = [item]; |
|
1591 |
+ item[arg.value] = dactal.execute(item[arg.value], [{operator: op.operator, args: op.args.slice(argx + 1).map((arg) => structuredClone(arg))}], sublabeled); |
|
1592 |
+ break; |
|
1593 |
+ } else if (arg.separator === ';' && argx === op.args.length - 1 && arg.label == null && arg.value == null && arg.subop == null) { |
|
1594 |
+ for (const oldprop in item) { |
|
1595 |
+ if (!(newprops.includes(oldprop))) { |
|
1596 |
+ const tempval = item[oldprop]; |
|
1597 |
+ delete item[oldprop]; |
|
1598 |
+ item[oldprop] = tempval; |
|
1599 |
+ } |
|
1600 |
+ } |
|
1601 |
+ } else if (arg.subop?.endsWith('-')) { |
|
1602 |
+ if (arg.value && typeof arg.value == 'string' && arg.value in item) { |
|
1603 |
+ if (arg.label && typeof arg.label == 'string') item[arg.label] = item[arg.value]; |
|
1604 |
+ delete item[arg.value]; |
|
1605 |
+ } else if (arg.label && Array.isArray(arg.value)) { |
|
1606 |
+ const toremoveids = new Set(step(item, arg.value, null, labeled).map((subitem) => getid(subitem))); |
|
1607 |
+ item[arg.label] = item[arg.label].filter((subitem) => !toremoveids.has(getid(subitem))); |
|
1608 |
+ } |
|
1609 |
+ } else if (argx === 0 && arg.label != null && arg.value === '_') { |
|
1610 |
+ item[arg.label] = op.operator === '||' ? baseitem : [baseitem]; |
|
1611 |
+ } else { |
|
1612 |
+ if (!arg.label && arg.value && typeof arg.value === 'string' && arg.value !== '') { |
|
1613 |
+ const moveprop = arg.value; |
|
1614 |
+ const value = structuredClone(item[moveprop]); |
|
1615 |
+ delete item[moveprop]; |
|
1616 |
+ item[moveprop] = value; |
|
1617 |
+ } |
|
1618 |
+ if (arg.label) { |
|
1619 |
+ let vals = []; |
|
1620 |
+ if (arg.subop.endsWith('@')) { |
|
1621 |
+ if (arg.subop.endsWith('@@')) { |
|
1622 |
+ arg.counter ??= arg.value == null ? outputlist.length + 1 : 0; |
|
1623 |
+ } else { |
|
1624 |
+ arg.counter ??= 1; |
|
1625 |
+ vals = [arg.counter]; |
|
1626 |
+ } |
|
1627 |
+ if (Array.isArray(arg.value)) { |
|
1628 |
+ arg.counter += step(item, arg.value, null, labeled).length; |
|
1629 |
+ } else if (dtype(arg.value, 'string') && dtype(item[arg.value], 'number')) { |
|
1630 |
+ arg.counter += Number(item[arg.value]); |
|
1631 |
+ } else { |
|
1632 |
+ arg.counter += arg.subop.endsWith('@@') ? -1 : 1; |
|
1633 |
+ } |
|
1634 |
+ if (arg.subop.endsWith('@@')) { |
|
1635 |
+ vals = [arg.counter]; |
|
1636 |
+ } |
|
1637 |
+ } else { |
|
1638 |
+ vals = step(item, arg.value, arg.subop, labeled); |
|
1639 |
+ } |
|
1640 |
+ const base = (arg.subop?.endsWith('+') && arg.label in item && Array.isArray(item[arg.label])) ? item[arg.label] : []; |
|
1641 |
+ if (this.dtype(vals, 'array')) { |
|
1642 |
+ if (op.operator === '|') { |
|
1643 |
+ item[arg.label] = base.concat(vals.map((v) => dcopy(v))); |
|
1644 |
+ } else if (vals.length > 0 ) { |
|
1645 |
+ if (typeof vals[0] === 'object') { |
|
1646 |
+ item[arg.label] = getname(vals[0]) != null ? getname(vals[0]) : vals[0].id != null ? vals[0].id : dcopy(vals[0]); |
|
1647 |
+ } else { |
|
1648 |
+ item[arg.label] = vals[0]; |
|
1649 |
+ } |
|
1650 |
+ } |
|
1651 |
+ } else if (vals) { |
|
1652 |
+ item[arg.label] = base.concat(vals); |
|
1653 |
+ } |
|
1654 |
+ } |
|
1655 |
+ } |
|
1656 |
+ } |
|
1657 |
+ } |
|
1658 |
+ } |
|
1659 |
+ break; |
|
1660 |
+ case '???': |
|
1661 |
+ outputlist = currentlist; |
|
1662 |
+ const commentval = op.args?.[0]?.value; |
|
1663 |
+ if (commentval in labeled) console.log({[commentval]: labeled[commentval]}); |
|
1664 |
+ if (commentval === 'end') return (this.debug && !inputlist) ? operations : outputlist; |
|
1665 |
+ break; |
|
1666 |
+ default: |
|
1667 |
+ outputlist = [] |
|
1668 |
+ } |
|
1669 |
+ currentlist = outputlist.slice(0); |
|
1670 |
+ if (this.debug && !inputlist) op.results = outputlist.slice(0, typeof this.debug == 'number' ? this.debug : outputlist.length); |
|
1671 |
+ } |
|
1672 |
+ return (this.debug && !inputlist) ? operations : outputlist; |
|
1673 |
+ } |
|
1674 |
+ |
|
1675 |
+ gettype(value, allow_relative=true) { |
|
1676 |
+ if (value == null) return null; |
|
1677 |
+ const trylist = [value]; |
|
1678 |
+ if (value.endsWith('s')) { |
|
1679 |
+ trylist.push(value.slice(0, value.length - 1)); |
|
1680 |
+ trylist.push(value + 'es'); |
|
1681 |
+ } else { |
|
1682 |
+ trylist.push(value + 's'); |
|
1683 |
+ } |
|
1684 |
+ for (const tryval of trylist) { |
|
1685 |
+ if (tryval in this.data) { |
|
1686 |
+ return this.data[tryval]; |
|
1687 |
+ } |
|
1688 |
+ } |
|
1689 |
+ if (this.savedquerynames.has(value)) { |
|
1690 |
+ const savedqueries = this.data.queries.filter((q) => q.name === value); |
|
1691 |
+ if (savedqueries?.length === 1) { |
|
1692 |
+ const sq = savedqueries[0]; |
|
1693 |
+ if (allow_relative && sq.query.startsWith('?')) { |
|
1694 |
+ return sq; |
|
1695 |
+ } else { |
|
1696 |
+ if (!sq.results) sq.results = this.executeq(sq.query); |
|
1697 |
+ return sq.results.slice(0); |
|
1698 |
+ } |
|
1699 |
+ } |
|
1700 |
+ } |
|
1701 |
+ return null; |
|
1702 |
+ } |
|
1703 |
+ |
|
1704 |
+ dcopy = (item) => { |
|
1705 |
+ if (this.dtype(item, 'literal')) return item; |
|
1706 |
+ if (this.dtype(item, 'array')) return item.slice(0); |
|
1707 |
+ return Object.assign({}, item); |
|
1708 |
+ } |
|
1709 |
+ |
|
1710 |
+ dtype(item, test=null) { |
|
1711 |
+ let type = null; |
|
1712 |
+ if (item && Array.isArray(item)) { |
|
1713 |
+ type = 'array'; |
|
1714 |
+ } else if (item && typeof item === 'object') { |
|
1715 |
+ type = 'object'; |
|
1716 |
+ } else if (typeof item === 'string' || typeof item === 'number' || typeof item === 'boolean') { |
|
1717 |
+ type = 'literal'; |
|
1718 |
+ } |
|
1719 |
+ if (test) { |
|
1720 |
+ if (test === 'number') { |
|
1721 |
+ return type === 'literal' && !isNaN(item); |
|
1722 |
+ } else if (test === 'string') { |
|
1723 |
+ return type === 'literal' && typeof item === 'string'; |
|
1724 |
+ } else if (test === 'boolean') { |
|
1725 |
+ return type === 'literal' && typeof item === 'boolean'; |
|
1726 |
+ } |
|
1727 |
+ return type === test; |
|
1728 |
+ } |
|
1729 |
+ return type; |
|
1730 |
+ } |
|
1731 |
+ |
|
1732 |
+ getname(item) { |
|
1733 |
+ if (typeof item === 'object' && item != null) { |
|
1734 |
+ if (item?.name != null) { |
|
1735 |
+ return item.name; |
|
1736 |
+ } else if (item?.key?.length > 0) { |
|
1737 |
+ return item.key.join(' / '); |
|
1738 |
+ } |
|
1739 |
+ const {id, ...nonidprops} = item; |
|
1740 |
+ const names = Object.values(nonidprops).filter((val) => typeof val === 'string' || typeof val === 'number'); |
|
1741 |
+ if (names.length > 0) { |
|
1742 |
+ return names[0]; |
|
1743 |
+ } |
|
1744 |
+ for(const key in nonidprops) { |
|
1745 |
+ if (Array.isArray(item[key]) && item[key].length === 1 && typeof item[key][0] == 'string') { |
|
1746 |
+ return item[key][0]; |
|
1747 |
+ } |
|
1748 |
+ } |
|
1749 |
+ } else if (typeof item === 'string' || typeof item === 'number') { |
|
1750 |
+ return item; |
|
1751 |
+ } else if (typeof item === 'boolean') { |
|
1752 |
+ return item.toString(); |
|
1753 |
+ } |
|
1754 |
+ return ''; |
|
1755 |
+ } |
|
1756 |
+ |
|
1757 |
+ getid(item) { |
|
1758 |
+ if (typeof item === 'object' && item != null) { |
|
1759 |
+ const iditems = (item.id || item.uri || item.name) ? [item] : item.key?.length > 0 ? item.key : Object.keys(item).length === 3 && item.of?.length > 0 ? item.of : [item]; |
|
1760 |
+ return iditems.map((item) => { |
|
1761 |
+ if (item.id) { |
|
1762 |
+ if (Array.isArray(item.id)) { |
|
1763 |
+ return item.id[0]; |
|
1764 |
+ } else { |
|
1765 |
+ return item.id; |
|
1766 |
+ } |
|
1767 |
+ } else if (item.uri) { |
|
1768 |
+ if (Array.isArray(item.uri)) { |
|
1769 |
+ return item.uri[0]; |
|
1770 |
+ } else { |
|
1771 |
+ return item.uri; |
|
1772 |
+ } |
|
1773 |
+ } else if (item.name) { |
|
1774 |
+ return item.name; |
|
1775 |
+ } else { |
|
1776 |
+ // if (stringifiedid.length > 128) console.warn({idstringify: item, idlength: stringifiedid.length}); |
|
1777 |
+ return JSON.stringify(item); |
|
1778 |
+ } |
|
1779 |
+ }).join(','); |
|
1780 |
+ } else if (typeof item === 'string' || typeof item === 'number') { |
|
1781 |
+ return item; |
|
1782 |
+ } |
|
1783 |
+ return null; |
|
1784 |
+ } |
|
1785 |
+ |
|
1786 |
+ step = (item, property, subop, labeled) => { |
|
1787 |
+ // (this.data.trace ??= []).push({stepitem: item, property: property, subop: subop, labeled: labeled}); |
|
1788 |
+ const dtype = this.dtype; |
|
1789 |
+ const dcopy = this.dcopy; |
|
1790 |
+ const resolve = this.resolve; |
|
1791 |
+ const escapeRegExp = this.escapeRegExp; |
|
1792 |
+ const vals = []; |
|
1793 |
+ if (subop?.includes('~') && dtype(property, 'literal')) { |
|
1794 |
+ vals.push(property); |
|
1795 |
+ } else if (Array.isArray(property)) { |
|
1796 |
+ this.execute([item], property.map((x) => structuredClone(x)), labeled).forEach((val) => vals.push(val)); |
|
1797 |
+ } else if (property === '_') { |
|
1798 |
+ vals.push(dcopy(item)); |
|
1799 |
+ } else if (item?.toString()?.match(/^-?\d+$/) && property.match(/^\d+$/)) { |
|
1800 |
+ vals.push((item === 0 && 1 / item === -Infinity ? -1 : 1) * Number(`${item}.${property}`)); |
|
1801 |
+ } else if (item?.of && dtype(property, 'number')) { |
|
1802 |
+ const propval = Number(property); |
|
1803 |
+ (subop === '-' || propval < 0 ? item.of.slice(-1 * Math.abs(Number(property))) : item.of.slice(0, Math.abs(Number(property)))).forEach((val) => vals.push(val)); |
|
1804 |
+ } else if (property === 'id' && dtype(item, 'literal')) { |
|
1805 |
+ vals.push(item); |
|
1806 |
+ } else if (property === 'name') { |
|
1807 |
+ vals.push(this.getname(item)); |
|
1808 |
+ } else if ((dtype(item, 'number') || dtype(item, 'object')) && property.startsWith('=')) { |
|
1809 |
+ let calculation = property.slice(1); |
|
1810 |
+ const mathwords = Object.getOwnPropertyNames(Math).filter((mathword) => mathword.match(/^[a-z0-9]+$/)).sort((a, b) => b.length - a.length || a.localeCompare(b)); |
|
1811 |
+ const otherwords = ['split']; |
|
1812 |
+ const variables = Object.entries(item).concat(Object.entries(labeled)) |
|
1813 |
+ .map(([k, v]) => k) |
|
1814 |
+ .sort((a, b) => b.length - a.length || a.localeCompare(b)); |
|
1815 |
+ if (dtype(this.getname(item), 'number')) variables.push('_'); |
|
1816 |
+ const allowedwords = mathwords.concat(otherwords).concat(variables).sort((a, b) => b.length - a.length || a.localeCompare(b)); |
|
1817 |
+ const allowed = new RegExp(`^((\b(${allowedwords.map((w) => escapeRegExp(w)).join('|')})\b)|([0-9_\+\/\*\(\)\[\].%=,'" -]*))*$`); |
|
1818 |
+ let val; |
|
1819 |
+ if (calculation.match(allowed)) { |
|
1820 |
+ for (const variable of variables) { |
|
1821 |
+ const variableex = new RegExp(`\\b${variable}\\b`, 'g'); |
|
1822 |
+ if (calculation.match(variableex)) { |
|
1823 |
+ let vval = variable === '_' ? Number(this.getname(item)) : item[variable] ?? labeled[variable]; |
|
1824 |
+ if (Array.isArray(vval) && vval.length === 1) vval = vval[0]; |
|
1825 |
+ calculation = calculation.replaceAll(variableex, dtype(vval, 'number') ? vval : JSON.stringify(vval)); |
|
1826 |
+ } |
|
1827 |
+ } |
|
1828 |
+ if (calculation.match(/[A-Za-z]/)) { |
|
1829 |
+ for (const mathword of mathwords) { |
|
1830 |
+ const mathwordex = new RegExp(`\\b${mathword}\\b`, 'g'); |
|
1831 |
+ calculation = calculation.replaceAll(mathwordex, `Math.${mathword}`); |
|
1832 |
+ if (!calculation.match(/[A-Za-z]/)) break; |
|
1833 |
+ } |
|
1834 |
+ } |
|
1835 |
+ calculation = calculation.replaceAll(/\b=\b/g, '=='); |
|
1836 |
+ try { |
|
1837 |
+ val = eval(calculation); |
|
1838 |
+ } catch (error) { |
|
1839 |
+ val = calculation; |
|
1840 |
+ } |
|
1841 |
+ } else { |
|
1842 |
+ val = calculation; |
|
1843 |
+ } |
|
1844 |
+ if (val !== false) vals.push(val); |
|
1845 |
+ } else if (item) { |
|
1846 |
+ let found = false; |
|
1847 |
+ if (property != null && !dtype(item, 'literal')) { |
|
1848 |
+ const tryvals = [property]; |
|
1849 |
+ tryvals.push(property.endsWith('s') ? property.slice(0, -1) : property + 's'); |
|
1850 |
+ if (property.match(/ /)) tryvals.push(property.replaceAll(/ /g, '_')); |
|
1851 |
+ for (const tryval of tryvals) { |
|
1852 |
+ if (tryval in item) { |
|
1853 |
+ const resolved = resolve(tryval, item[tryval], labeled); |
|
1854 |
+ if (Array.isArray(resolved)) { |
|
1855 |
+ resolved.forEach((x) => vals.push(x)); |
|
1856 |
+ } else { |
|
1857 |
+ vals.push(resolved); |
|
1858 |
+ } |
|
1859 |
+ found = true; |
|
1860 |
+ break; |
|
1861 |
+ } |
|
1862 |
+ } |
|
1863 |
+ } |
|
1864 |
+ if (!found && labeled?.['=>']?.[property]) { |
|
1865 |
+ this.execute([item], labeled['=>'][property], labeled).forEach((val) => vals.push(val)); |
|
1866 |
+ found = true; |
|
1867 |
+ } |
|
1868 |
+ if (!found && (property in this.data || property in this.adapters || this.savedquerynames.has(property) || property in labeled)) { |
|
1869 |
+ let typenav; |
|
1870 |
+ if (!dtype(item, 'literal')) { |
|
1871 |
+ if (this.adapters?.[property]?.annotator) { |
|
1872 |
+ typenav = resolve(property, item, labeled); |
|
1873 |
+ // } else if (property in this.aggregators) { |
|
1874 |
+ // typenav = this.aggregators[property](item); |
|
1875 |
+ } else if ('id' in item || 'uri' in item) { |
|
1876 |
+ typenav = resolve(property, this.getid(item), labeled); |
|
1877 |
+ } else if (this.savedquerynames.has(property)) { |
|
1878 |
+ const relative_query = dactal.data.queries.find((q) => q.relative && q.name === property); |
|
1879 |
+ if (relative_query) { |
|
1880 |
+ typenav = this.step(item, this.parse(relative_query.query), subop, labeled); |
|
1881 |
+ } else { |
|
1882 |
+ typenav = this.getname(item); |
|
1883 |
+ } |
|
1884 |
+ } else { |
|
1885 |
+ typenav = this.getname(item); |
|
1886 |
+ } |
|
1887 |
+ if (typenav) vals.push(typenav); |
|
1888 |
+ } else { |
|
1889 |
+ typenav = resolve(property, item, labeled); |
|
1890 |
+ if (typenav) { |
|
1891 |
+ if (Array.isArray(typenav)) { |
|
1892 |
+ typenav.forEach((t) => vals.push(t)); |
|
1893 |
+ } else { |
|
1894 |
+ vals.push(typenav); |
|
1895 |
+ } |
|
1896 |
+ } |
|
1897 |
+ } |
|
1898 |
+ } |
|
1899 |
+ } |
|
1900 |
+ return vals; |
|
1901 |
+ } |
|
1902 |
+ |
|
1903 |
+ resolve = (property, item, labeled) => { |
|
1904 |
+ const dtype = this.dtype; |
|
1905 |
+ if (dtype(item, 'object')) { |
|
1906 |
+ if (this.adapters?.[property]?.annotator) { |
|
1907 |
+ if ((!this.recache || this.recache.has(property)) && property in this.index && this.getid(item) in this.index[property]) { |
|
1908 |
+ return this.index[property][this.getid(item)]; |
|
1909 |
+ } else { |
|
1910 |
+ this.adapters[property].queue.push(this.adapters[property].annotator ? item : this.getid(item)); |
|
1911 |
+ return null; |
|
1912 |
+ } |
|
1913 |
+ } else { |
|
1914 |
+ return item; |
|
1915 |
+ } |
|
1916 |
+ } else if (dtype(item, 'literal') && typeof property == 'string' && (property in labeled || this.destinations.has(property) || this.destinations.has(property + 's'))) { |
|
1917 |
+ if ((!this.recache || this.recache.has(property)) && property in this.index && item in this.index[property]) { |
|
1918 |
+ // this.cachehitcount++; |
|
1919 |
+ return this.index[property][item]; |
|
1920 |
+ } |
|
1921 |
+ const typeitems = labeled[property] ?? this.gettype(property); |
|
1922 |
+ if (Array.isArray(typeitems)) { |
|
1923 |
+ for (const lookupkey of ['id', 'uri', 'name', property, property + 's']) { |
|
1924 |
+ const found = typeitems.filter((typeitem) => { |
|
1925 |
+ return dtype(typeitem, 'object') && (lookupkey in typeitem) && (typeitem[lookupkey] === item || (Array.isArray(typeitem[lookupkey]) && typeitem[lookupkey].length === 1 && typeitem[lookupkey][0] === item)); |
|
1926 |
+ }); |
|
1927 |
+ if (found.length === 1) { |
|
1928 |
+ (this.index[property] ||= {})[item] = found[0]; |
|
1929 |
+ return found[0]; |
|
1930 |
+ } |
|
1931 |
+ } |
|
1932 |
+ if (property in this.adapters && (!this.recache || !this.recache.has(property))) { |
|
1933 |
+ this.adapters[property].queue.push(this.adapters[property].annotator ? item : this.getid(item)); |
|
1934 |
+ } |
|
1935 |
+ return null; |
|
1936 |
+ } else if (typeitems && typeof typeitems == 'object' && 'query' in typeitems) { |
|
1937 |
+ const relative_query = this.assemble(this.tokenize(typeitems.query)); |
|
1938 |
+ while (['?', ':'].includes(relative_query[0].operator)) { |
|
1939 |
+ relative_query.shift() |
|
1940 |
+ } |
|
1941 |
+ return this.execute([item], relative_query, labeled); |
|
1942 |
+ } |
|
1943 |
+ if (property in this.adapters) { |
|
1944 |
+ this.adapters[property].queue.push(this.adapters[property].annotator ? item : this.getid(item)); |
|
1945 |
+ return null; |
|
1946 |
+ } |
|
1947 |
+ } |
|
1948 |
+ return item; |
|
1949 |
+ } |
|
1950 |
+ |
|
1951 |
+ samearray(a, b) { |
|
1952 |
+ if ((a && !b) || (!a && b) || a.length !== b.length) return false; |
|
1953 |
+ for(let x=0; x<a.length; x++) { |
|
1954 |
+ if (this.getid(a[x]) !== this.getid(b[x])) return false; |
|
1955 |
+ } |
|
1956 |
+ return true; |
|
1957 |
+ } |
|
1958 |
+ |
|
1959 |
+ async verify(i = null) { |
|
1960 |
+ let toverify = this.data.queries; |
|
1961 |
+ if (!isNaN(i)) toverify = toverify.slice(i - 1, i); |
|
1962 |
+ let allsame = true; |
|
1963 |
+ for (const savedq of toverify) { |
|
1964 |
+ console.log('verifying ' + savedq.name); |
|
1965 |
+ const testres = await this.query(savedq.query); |
|
1966 |
+ if (testres.length !== savedq.results.length) { |
|
1967 |
+ console.log('--x result count changed from ' + savedq.results.length + ' to ' + testres.length); |
|
1968 |
+ allsame = false; |
|
1969 |
+ } else { |
|
1970 |
+ for (let i=0; i<testres.length; i++) { |
|
1971 |
+ const testrow = testres[i]; |
|
1972 |
+ const savedrow = savedq.results[i]; |
|
1973 |
+ if (typeof savedrow == 'object') { |
|
1974 |
+ for (const prop in savedrow) { |
|
1975 |
+ if (!(prop in testrow)) { |
|
1976 |
+ console.log('--x row ' + (i+1) + ': new results missing property ' + prop); |
|
1977 |
+ allsame = false; |
|
1978 |
+ } else { |
|
1979 |
+ const testval = JSON.stringify(testrow[prop]); |
|
1980 |
+ const savedval = JSON.stringify(savedrow[prop]); |
|
1981 |
+ if (testval !== savedval) { |
|
1982 |
+ console.log('--x row ' + (i+1) + ': different value for property ' + prop); |
|
1983 |
+ console.log({was: savedrow[prop], now: testrow[prop]}) |
|
1984 |
+ allsame = false; |
|
1985 |
+ } |
|
1986 |
+ } |
|
1987 |
+ } |
|
1988 |
+ } else { |
|
1989 |
+ if (testrow !== savedrow) { |
|
1990 |
+ console.log('--x row ' + (i+1) + ': different value'); |
|
1991 |
+ console.log({was: savedrow, now: testrow}); |
|
1992 |
+ allsame = false; |
|
1993 |
+ } |
|
1994 |
+ } |
|
1995 |
+ } |
|
1996 |
+ } |
|
1997 |
+ if (allsame) console.log('--- results unchanged'); |
|
1998 |
+ } |
|
1999 |
+ } |
|
2000 |
+ |
|
2001 |
+ index_check() { |
|
2002 |
+ console.table(Object.entries(this.index).map(([key, vals]) => ({key: key, vals: Object.keys(vals).length, size: Object.keys(vals).length * JSON.stringify(Object.entries(vals).slice(0, 1)).length})).sort((a, b) => b.size - a.size || b.vals - a.vals || a.key.localeCompare(b.key))) |
|
2003 |
+ } |
|
2004 |
+ |
|
2005 |
+ index_materialize() { |
|
2006 |
+ this.load(Object.keys(this.index).flatMap((i) => Object.keys(this.index[i]).flatMap((k) => ({index: i, indexed: k, value: this.index[i][k]}))), 'index contents') |
|
2007 |
+ } |
|
2008 |
+} |