| 1 |
export class DACTAL { |
| 2 |
constructor(data={}) { |
| 3 |
this.data = data; |
| 4 |
this.index = {}; |
| 5 |
this.index_modified = false; |
| 6 |
this.adapters = {}; |
| 7 |
this.data['query history'] = []; |
| 8 |
this.savedquerynames = new Set(); |
| 9 |
this.debug = false; |
| 10 |
this.recache = false; |
| 11 |
this.timelimit = 120000; |
| 12 |
this.statusf = (statusmsg) => console.log(statusmsg); |
| 13 |
this.aggregators = { |
| 14 |
group: (item) => item.of, |
| 15 |
count: (item) => this.vals(item).length, |
| 16 |
total: (item) => this.numvals(item).reduce((acc, val) => acc + (this.dtype(val, 'number') ? val : 0), 0), |
| 17 |
average: (item) => this.numvals(item).reduce((acc, val) => acc + (this.dtype(val, 'number') ? val : 0), 0) / this.numvals(item).length, |
| 18 |
min: (item) => { |
| 19 |
const nums = this.numvals(item); |
| 20 |
return nums.length === 0 ? [] : Math.min(...nums) |
| 21 |
}, |
| 22 |
max: (item) => { |
| 23 |
const nums = this.numvals(item); |
| 24 |
return nums.length === 0 ? [] : Math.max(...nums) |
| 25 |
}, |
| 26 |
product: (item) => this.numvals(item).reduce((acc, val) => acc * (this.dtype(val, 'number') ? val : 1), 1), |
| 27 |
difference: (item) => this.numvals(item).reduce((acc, val) => acc - (this.dtype(val, 'number') ? val : 0)), |
| 28 |
quotient: (item) => this.numvals(item).reduce((acc, val) => acc / (this.dtype(val, 'number') ? val : 1)), |
| 29 |
percent: (item) => Math.round(this.numvals(item).reduce((acc, val) => acc / (this.dtype(val, 'number') ? val : 1)) * 100), |
| 30 |
sqrt: (item) => Math.sqrt(this.numvals(item)[0]), |
| 31 |
log: (item) => Math.log(this.numvals(item)[0]), |
| 32 |
log10: (item) => Math.log10(this.numvals(item)[0]), |
| 33 |
abs: (item) => Math.abs(this.numvals(item)[0]), |
| 34 |
is: (item) => (item.of?.length > 0) ? 1 : 0, |
| 35 |
isnt: (item) => (item.of?.length === 0) ? 1 : 0, |
| 36 |
yesno: (item) => (item.of?.length > 0) ? 'yes' : 'no', |
| 37 |
missing: (item) => (item.of?.length === 0) ? true : null, |
| 38 |
concatenate: (item) => this.vals(item).join(' '), |
| 39 |
join: (item) => item.of.map(this.getname).join(this.vals(item)[0]), |
| 40 |
str: (item) => item.of.map(this.getname).join(''), |
| 41 |
'to json': (item) => JSON.stringify(item.of), |
| 42 |
quote: (item) => `“${this.vals(item)[0]}”`, |
| 43 |
url: (item) => { |
| 44 |
let u = item.of.map(this.getname).join(''); |
| 45 |
if (!u.startsWith('https://')) u = 'https://' + u; |
| 46 |
for (const prop of Object.keys(item).filter((key) => key !== 'of')) { |
| 47 |
const val = item[prop]; |
| 48 |
(Array.isArray(val) ? val : [val]).forEach((vv) => u = u + (u.match(/\?/) ? '&' : '?') + encodeURIComponent(prop) + '=' + encodeURIComponent(vv)); |
| 49 |
} |
| 50 |
return u; |
| 51 |
}, |
| 52 |
remove: (item) => this.vals(item).reduce((acc, val) => acc.replaceAll(val, '')), |
| 53 |
bmk: (item) => this.vals(item).map((v) => v.toString().toLowerCase().replaceAll(/[$\xA2-\xA5\u058F\u060B\u09F2\u09F3\u09FB\u0AF1\u0BF9\u0E3F\u17DB\u20A0-\u20BD\uA838\uFDFC\uFE69\uFF04\uFFE0\uFFE1\uFFE5\uFFE6,+]/g, '').replaceAll('b', 'kkk').replaceAll('m', 'kk').replaceAll('k', '000')), |
| 54 |
sortform: (item) => this.vals(item).map((val) => val.toString().toLowerCase().replace(/^the /, '')), |
| 55 |
zip: (item) => { |
| 56 |
const itemkeys = Object.keys(item).filter((k) => k !== 'of'); |
| 57 |
const zipped = []; |
| 58 |
for (let i=0; i<item[itemkeys[0]].length; i++) { |
| 59 |
const zipline = {}; |
| 60 |
for (const key of itemkeys) { |
| 61 |
zipline[key] = item[key][i]; |
| 62 |
} |
| 63 |
zipped.push(zipline); |
| 64 |
} |
| 65 |
return zipped; |
| 66 |
}, |
| 67 |
pairs: (item) => { |
| 68 |
return item.of.slice(0, -1).map((val, vx) => ({pair: [val, item.of[vx+1]]})); |
| 69 |
}, |
| 70 |
triples: (item) => { |
| 71 |
return item.of.slice(0, -2).map((val, vx) => ({triple: [val, item.of[vx+1], item.of[vx+2]]})); |
| 72 |
}, |
| 73 |
quads: (item) => { |
| 74 |
return item.of.slice(0, -3).map((val, vx) => ({quad: [val, item.of[vx+1], item.of[vx+2], item.of[vx+3]]})); |
| 75 |
}, |
| 76 |
sequences: (item) => { |
| 77 |
return item.of.map((val, valx, vallist) => ({sequence: vallist.slice(0, valx+1).map((val) => this.dcopy(val))})); |
| 78 |
}, |
| 79 |
split: (item) => { |
| 80 |
const itemvals = this.vals(item); |
| 81 |
let splitter; |
| 82 |
let tobesplit; |
| 83 |
if (Object.keys(item).length === 1) itemvals.push(' '); |
| 84 |
if (itemvals.length === 1) { |
| 85 |
splitter = itemvals[0]; |
| 86 |
tobesplit = [this.getname(item)]; |
| 87 |
} else { |
| 88 |
splitter = itemvals.pop(); |
| 89 |
tobesplit = itemvals.slice(0); |
| 90 |
} |
| 91 |
if (splitter.startsWith('~')) splitter = new RegExp(splitter.replace(/^~*/, ''), splitter.startsWith('~~') ? 'i' : ''); |
| 92 |
return tobesplit.flatMap((v) => v.toString().split(splitter)); |
| 93 |
}, |
| 94 |
extract: (item) => { |
| 95 |
const itemvals = this.vals(item); |
| 96 |
const delimiters = itemvals.pop(); |
| 97 |
const res = []; |
| 98 |
for (const itemval of itemvals) { |
| 99 |
for(let i=0; i<delimiters.length; i+=2) { |
| 100 |
const d1 = delimiters[i]; |
| 101 |
const d2 = delimiters[i+1]; |
| 102 |
const d1x = itemval.indexOf(d1); |
| 103 |
const d2x = itemval.indexOf(d2); |
| 104 |
if (d1x > -1 && d2x > d1x) { |
| 105 |
res.push(itemval.slice(d1x+1, d2x).trim()); |
| 106 |
break; |
| 107 |
} |
| 108 |
} |
| 109 |
} |
| 110 |
return res; |
| 111 |
}, |
| 112 |
unchain: (item) => { |
| 113 |
const chainprops = Object.keys(item).filter((key) => key !== 'of'); |
| 114 |
const unchained = []; |
| 115 |
const queue = item.of.slice(0); |
| 116 |
while (queue.length > 0) { |
| 117 |
const thisitem = queue.shift(); |
| 118 |
if (!unchained.includes(thisitem)) { |
| 119 |
unchained.push(thisitem); |
| 120 |
for (const chainprop of chainprops) { |
| 121 |
if (this.dtype(thisitem, 'object') && chainprop in thisitem) { |
| 122 |
if (this.dtype(thisitem[chainprop], 'array')) { |
| 123 |
for (const x of thisitem[chainprop].slice(0).reverse()) queue.unshift(x); |
| 124 |
} else if (thisitem[chainprop]) { |
| 125 |
queue.unshift(thisitem[chainprop]) |
| 126 |
} |
| 127 |
} |
| 128 |
} |
| 129 |
} |
| 130 |
} |
| 131 |
return unchained; |
| 132 |
}, |
| 133 |
itemize: (item) => { |
| 134 |
const propname = item?.property ?? 'property'; |
| 135 |
const valname = item?.value ?? 'value'; |
| 136 |
return item.of.flatMap((subitem) => Object.entries(subitem).map(([key, val]) => ({[propname]: key, [valname]: val}))); |
| 137 |
}, |
| 138 |
schematize: (item) => { |
| 139 |
const itemkeys = Object.keys(item).filter((k) => k !== 'of'); |
| 140 |
const schematized = {}; |
| 141 |
item[itemkeys[0]].forEach((subitem) => { |
| 142 |
let subkey; |
| 143 |
let subval; |
| 144 |
if (this.dtype(subitem, 'array')) { |
| 145 |
const [subkey, subval] = subitem; |
| 146 |
} else { |
| 147 |
[subkey, subval] = Object.values(subitem); |
| 148 |
} |
| 149 |
schematized[subkey] = subval; |
| 150 |
}); |
| 151 |
return schematized; |
| 152 |
}, |
| 153 |
index: (item) => { |
| 154 |
return Object.entries(item).filter(([k, v]) => k !== 'of').map(([k, v]) => ({id: k, name: !isNaN(v) ? Number(v) : v})); |
| 155 |
}, |
| 156 |
unflatten: (item) => { |
| 157 |
const itemkeys = Object.keys(item).filter((k) => k !== 'of'); |
| 158 |
if (itemkeys.length === 0) itemkeys.push(''); |
| 159 |
const newindex = {}; |
| 160 |
const neworder = []; |
| 161 |
item.of.forEach((subitem) => { |
| 162 |
Object.keys(subitem).forEach((field) => { |
| 163 |
itemkeys.forEach((key) => { |
| 164 |
if (field.startsWith(key)) { |
| 165 |
const subid = field.replace(key, ''); |
| 166 |
if (subid.length > 0) { |
| 167 |
if (!(subid in newindex)) { |
| 168 |
newindex[subid] = {}; |
| 169 |
neworder.push(subid); |
| 170 |
} |
| 171 |
newindex[subid][key] = subitem[field]; |
| 172 |
} |
| 173 |
} |
| 174 |
}); |
| 175 |
}); |
| 176 |
}); |
| 177 |
itemkeys.forEach((key) => delete item[key]); |
| 178 |
return neworder.map((k) => { |
| 179 |
const subitem = {}; |
| 180 |
subitem.subid = k; |
| 181 |
Object.assign(subitem, newindex[k]); |
| 182 |
return subitem; |
| 183 |
}); |
| 184 |
}, |
| 185 |
detupled: (item) => { |
| 186 |
const newobj = {}; |
| 187 |
item.of.forEach((subitem) => { |
| 188 |
if (Array.isArray(subitem) && subitem.length === 2) { |
| 189 |
newobj[subitem[0]] = subitem[1]; |
| 190 |
} |
| 191 |
}) |
| 192 |
return [newobj]; |
| 193 |
}, |
| 194 |
csv: (item) => { |
| 195 |
const keys = Object.entries(item).find(([k, v]) => k !== 'of')[1]; |
| 196 |
const res = []; |
| 197 |
const vals = item.of.flatMap((val) => this.dtype(val, 'string') ? val.split('\n').map((val) => val.trim()) : val); |
| 198 |
for (let i=0; i<vals.length; i+=keys.length) { |
| 199 |
const newitem = {}; |
| 200 |
for (let k=0; k<keys.length; k++) { |
| 201 |
newitem[keys[k]] = vals[i+k]; |
| 202 |
} |
| 203 |
res.push(newitem); |
| 204 |
} |
| 205 |
return res; |
| 206 |
}, |
| 207 |
tsv: (item) => { |
| 208 |
const text = this.getname(item); |
| 209 |
const lines = text.split('\n').filter((line) => line !== '').map((line) => line.split('\t').map((val) => val.trim())); |
| 210 |
const keys = lines[0]; |
| 211 |
return lines.slice(1).map((vals) => Object.fromEntries(vals.map((val, vi) => [keys[vi], val]))); |
| 212 |
}, |
| 213 |
ssv: (item) => { |
| 214 |
const text = this.getname(item); |
| 215 |
const lines = text.split('\n').filter((line) => line !== '').map((line) => line.split(/ +/).map((val) => val.trim())); |
| 216 |
const keys = lines[0]; |
| 217 |
return lines.slice(1).map((vals) => Object.fromEntries(vals.map((val, vi) => [keys[vi], val]))); |
| 218 |
}, |
| 219 |
json: (item) => this.vals(item).flatMap((v) => JSON.parse(v)), |
| 220 |
year: (item) => { |
| 221 |
if (item.date?.length > 0) return item.date[0].match(/\d\d\d\d/)[0]; |
| 222 |
let yearmatch = this.getname(item).match(/\d\d\d\d/); |
| 223 |
if (yearmatch) { |
| 224 |
return yearmatch[0]; |
| 225 |
} else { |
| 226 |
yearmatch = this.getid(item.of[0] || '').toString().match(/\d\d\d\d/); |
| 227 |
if (yearmatch) { |
| 228 |
return yearmatch[0]; |
| 229 |
} |
| 230 |
return null; |
| 231 |
} |
| 232 |
}, |
| 233 |
month: (item) => { |
| 234 |
if (item.date?.length > 0) return item.date[0].match(/\d\d\d\d-(\d\d)-\d\d/)[1]; |
| 235 |
let monthmatch = this.getname(item).match(/\d\d\d\d-(\d\d)-\d\d/); |
| 236 |
if (monthmatch) { |
| 237 |
return monthmatch[1]; |
| 238 |
} else { |
| 239 |
monthmatch = this.getid(item.of[0] || '').toString().match(/\d\d\d\d-(\d\d)-\d\d/); |
| 240 |
if (monthmatch) { |
| 241 |
return monthmatch[1]; |
| 242 |
} |
| 243 |
return null; |
| 244 |
} |
| 245 |
}, |
| 246 |
date: (item) => { |
| 247 |
let itemvals = this.vals(item); |
| 248 |
if (itemvals) { |
| 249 |
let datematch = itemvals[0].match(/\d\d\d\d-\d\d-\d\d/); |
| 250 |
if (datematch) { |
| 251 |
return datematch[0]; |
| 252 |
} else { |
| 253 |
datematch = this.getid(item.of[0] || '').toString().match(/\d\d\d\d-\d\d-\d\d/); |
| 254 |
if (datematch) { |
| 255 |
return datematch[0]; |
| 256 |
} |
| 257 |
} |
| 258 |
} |
| 259 |
return null; |
| 260 |
}, |
| 261 |
weekday: (item) => { |
| 262 |
const days = ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun']; |
| 263 |
if (item.date?.length > 0) return days[new Date(item.date).getDay()]; |
| 264 |
}, |
| 265 |
timeshift: (item) => { |
| 266 |
const vals = this.vals(item); |
| 267 |
let tsx = new Date(vals[0]); |
| 268 |
let adjust = Number(vals[1]) * 60*60*1000; |
| 269 |
tsx.setTime(tsx.getTime() + adjust); |
| 270 |
return tsx.toISOString(); |
| 271 |
}, |
| 272 |
hour: (item) => this.vals(item).map((v) => v.split('T')[1].split(':')[0]), |
| 273 |
datediff: (item) => { |
| 274 |
const [d1, d2] = this.vals(item); |
| 275 |
return (new Date(d2) - new Date(d1)) / (24*60*60*1000); |
| 276 |
}, |
| 277 |
timediff: (item) => { |
| 278 |
const [d1, d2] = this.vals(item); |
| 279 |
return (new Date(d2) - new Date(d1)); |
| 280 |
}, |
| 281 |
dateforms: (item) => { |
| 282 |
const basedate = this.vals(item)[0]; |
| 283 |
const [baseyear, basemonth, baseday] = basedate.split('-'); |
| 284 |
return [ |
| 285 |
basedate, |
| 286 |
`${basemonth}/${baseday}/${baseyear}`, |
| 287 |
basemonth.startsWith('0') ? `${basemonth.replace(/^0/,'')}/${baseday}/${baseyear}` : null, |
| 288 |
basemonth.startsWith('0') && baseday.startsWith('0') ? `${basemonth.replace(/^0/,'')}/${baseday.replace(/^0/,'')}/${baseyear}` : null, |
| 289 |
`${basemonth}/${baseday}/${baseyear.slice(2)}`, |
| 290 |
`${basemonth.replace(/^0/,'')}/${baseday.replace(/^0/,'')}/${baseyear.slice(2)}`, |
| 291 |
`${['', 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'][Number(basemonth)]} ${baseday.replace(/^0/,'')}, ${baseyear}` |
| 292 |
].filter(x => x); |
| 293 |
}, |
| 294 |
now: (item) => performance.now(), |
| 295 |
round: (item) => this.numvals(item).map((v) => Math.round(v)), |
| 296 |
roundaway: (item) => this.numvals(item).map((v) => Math.sign(v) * Math.round(Math.abs(v))), |
| 297 |
roundm: (item) => { |
| 298 |
const vals = this.numvals(item); |
| 299 |
const multiple = vals.pop(); |
| 300 |
return vals.map((v) => Math.round(v / multiple) * multiple); |
| 301 |
}, |
| 302 |
roundd: (item) => { |
| 303 |
const vals = this.numvals(item); |
| 304 |
const digits = vals.pop(); |
| 305 |
return vals.map((v) => { |
| 306 |
const factor = 10 ** (Math.floor(Math.log10(v)) - digits + 1); |
| 307 |
return Math.round(v / factor) * factor; |
| 308 |
}); |
| 309 |
}, |
| 310 |
floor: (item) => this.numvals(item).map((v) => Math.floor(v)), |
| 311 |
ceil: (item) => this.numvals(item).map((v) => Math.ceil(v)), |
| 312 |
n: (item) => { |
| 313 |
let i = 0; |
| 314 |
const test = this.getname(item.of[0]); |
| 315 |
for (const key in item) { |
| 316 |
if (key !== 'of') { |
| 317 |
i--; |
| 318 |
if (key === test) return i; |
| 319 |
} |
| 320 |
} |
| 321 |
return i - 1; |
| 322 |
}, |
| 323 |
numbers: (item) => { |
| 324 |
const res = []; |
| 325 |
const firstnum = item.from !== undefined && item.from != null ? Number(item.from) : 1; |
| 326 |
const lastnum = item.to !== undefined && item.to != null ? Number(item.to) : Number(this.getname(item.of[0])); |
| 327 |
for(let x=firstnum; x<=lastnum; x++) { |
| 328 |
res.push(x) |
| 329 |
} |
| 330 |
return res; |
| 331 |
}, |
| 332 |
dehyphenate: (item) => this.vals(item).flatMap((v) => v.replaceAll(/(?<=\w)-\n(?=\w)/g, '')), |
| 333 |
sentences: (item) => this.vals(item).flatMap((v) => v.split(/[.?!…]['"’”»]?\s+/)), |
| 334 |
allwords: (item) => this.vals(item).flatMap((v) => Array.from(v.toString().toLowerCase().matchAll(/[\p{Letter}\p{Number}]+(?:['‘’][\p{Letter}\p{Number}]+)?/gu).map((m) => m[0]))), |
| 335 |
words: (item) => this.vals(item).flatMap((v) => Array.from(v.toString().toLowerCase().matchAll(/[\p{Letter}\p{Number}]+(?:['‘’][\p{Letter}\p{Number}]+)?/gu).map((m) => m[0])).filter((w) => w.length >= 4)), |
| 336 |
characters: (item) => this.vals(item).flatMap((v) => Array.from(v)), |
| 337 |
'character count': (item) => this.vals(item).flatMap((v) => v.length), |
| 338 |
case: (item) => this.vals(item).map((v) => { |
| 339 |
const hasupper = v.match(/[A-Z]/); |
| 340 |
const haslower = v.match(/[a-z]/); |
| 341 |
if (!hasupper && !haslower) { |
| 342 |
return 'none'; |
| 343 |
} else if (hasupper && !haslower) { |
| 344 |
return 'upper'; |
| 345 |
} else if (haslower && !hasupper) { |
| 346 |
return 'lower'; |
| 347 |
} else { |
| 348 |
if (v[0].match(/[A-Z]/) && !(v.slice(1).match(/[A-Z]/))) { |
| 349 |
return 'initial'; |
| 350 |
} else { |
| 351 |
return 'mixed'; |
| 352 |
} |
| 353 |
} |
| 354 |
}), |
| 355 |
uppercase: (item) => this.vals(item).flatMap((v) => v.toString().toUpperCase()), |
| 356 |
lowercase: (item) => this.vals(item).flatMap((v) => v.toString().toLowerCase().replaceAll(/[‘’`]/g, "'").replaceAll(/[“”]/g, '"')), |
| 357 |
list: (item) => Object.keys(item).filter((k) => k !== 'of'), |
| 358 |
items: (item) => item?.of?.flatMap((typename) => this.data?.[this.getname(typename)]), |
| 359 |
random: (item) => Math.random(), |
| 360 |
shuffle: (item) => { |
| 361 |
const newArray = [].concat(item?.of || []); |
| 362 |
for (let i = newArray.length - 1; i > 0; i--) { |
| 363 |
const j = Math.floor(Math.random() * (i + 1)); |
| 364 |
[newArray[i], newArray[j]] = [newArray[j], newArray[i]]; |
| 365 |
} |
| 366 |
return newArray; |
| 367 |
}, |
| 368 |
'weighted shuffle': (item) => { |
| 369 |
return item.of.map((subitem) => ({weight: Number(subitem.weight || 1) * Math.random(), subitem: subitem})).sort((a, b) => b.weight - a.weight).map((ws) => ws.subitem); |
| 370 |
}, |
| 371 |
pick: (item) => item?.of?.[~~(Math.random() * item?.of?.length)], |
| 372 |
link: (item) => `<a href="${item.url}"${item.target ? ' target="' + item.target + '"' : ''}>${item.text}</a>`, |
| 373 |
img: (item) => { |
| 374 |
if (item.uri) { |
| 375 |
return `<a href="${item.uri}"><img src="${item.src}" height=${item.height}px width=${item.width}px></a>`; |
| 376 |
} else { |
| 377 |
return `<img src="${item.src}" height=${item.height}px width=${item.width}px>`; |
| 378 |
} |
| 379 |
}, |
| 380 |
sign: (item) => item.of.map((subitem) => { |
| 381 |
if (subitem.id) { |
| 382 |
return subitem; |
| 383 |
} else { |
| 384 |
const str = JSON.stringify(subitem); |
| 385 |
let hash = 0; |
| 386 |
for (let i = 0; i < str.length; i++) { |
| 387 |
const char = str.charCodeAt(i); |
| 388 |
hash = (hash << 5) - hash + char; |
| 389 |
} |
| 390 |
// Convert to 32bit unsigned integer in base 36 and pad with "0" to ensure length is 7. |
| 391 |
return Object.assign({id: (hash >>> 0).toString(36).padStart(7, '0')}, subitem); |
| 392 |
} |
| 393 |
}), |
| 394 |
'parse query': (item) => item.of.map((subitem) => ({name: subitem.name, query: subitem.query, assembly: this.parse(subitem.query)})) |
| 395 |
} |
| 396 |
this.data.aggregators = Object.keys(this.aggregators); |
| 397 |
// this.cachehitcount = 0; |
| 398 |
// this.cachemisscount = {}; |
| 399 |
this.destinations = new Set(); |
| 400 |
} |
| 401 |
|
| 402 |
survey() { |
| 403 |
this.destinations = new Set(Object.keys(this.data).concat(this.data?.queries?.map((q) => q.name) || []).concat(Object.keys(this.adapters))); |
| 404 |
} |
| 405 |
|
| 406 |
vals(item) { |
| 407 |
const getname = this.getname; |
| 408 |
const itemvals = []; |
| 409 |
if (Object.keys(item).length > 1) { |
| 410 |
Object.entries(item).filter(([k, v]) => k !== 'of').flatMap(([k, v]) => Array.isArray(v) ? v : [v]).map(getname).forEach((v) => itemvals.push(v)); |
| 411 |
} else { |
| 412 |
item.of.map(getname).forEach((v) => itemvals.push(v)); |
| 413 |
} |
| 414 |
return itemvals; |
| 415 |
} |
| 416 |
|
| 417 |
numvals(item) { |
| 418 |
return this.vals(item).filter((val) => this.dtype(val, 'number')).map((val) => Number(val)); |
| 419 |
} |
| 420 |
|
| 421 |
async querylive(query, inputlist=null) { |
| 422 |
this.recache = new Set(); |
| 423 |
return await this.query(query, inputlist); |
| 424 |
} |
| 425 |
|
| 426 |
async query(query, inputlist=null, loop=50) { |
| 427 |
this.survey(); |
| 428 |
const operations = Array.isArray(query) ? query : this.assemble(this.tokenize(query)); |
| 429 |
const result = this.execute(inputlist, operations); |
| 430 |
this.data['current results'] = result; |
| 431 |
const queued = Object.keys(this.adapters).filter((key) => this.adapters[key].queue.length > 0); |
| 432 |
if (loop > 0 && queued.length > 0) { |
| 433 |
await this.adapt(); |
| 434 |
if (this.recache) queued.forEach((q) => this.recache.add(q)); |
| 435 |
return await this.query(operations, inputlist, loop - 1); |
| 436 |
} else if (loop === 0) { |
| 437 |
this.recache = false; |
| 438 |
Object.keys(this.adapters).forEach((key) => { |
| 439 |
if (!this.adapters[key].annotator) { |
| 440 |
this.adapters[key].queue.forEach((id) => { |
| 441 |
(this.index[key] ||= {})[id] = null; |
| 442 |
}) |
| 443 |
} |
| 444 |
}) |
| 445 |
} |
| 446 |
return result; |
| 447 |
} |
| 448 |
|
| 449 |
async adapt() { |
| 450 |
for (const key in this.adapters) { |
| 451 |
const adapter = this.adapters[key]; |
| 452 |
if (adapter.queue.length > 0) { |
| 453 |
if (adapter.annotator) { |
| 454 |
while (adapter.queue.length > 0) { |
| 455 |
const item_to_annotate = adapter.queue[0]; |
| 456 |
try { |
| 457 |
(this.index[key] ||= {})[this.getid(item_to_annotate)] = await adapter.f(item_to_annotate); |
| 458 |
this.index_modified = true; |
| 459 |
adapter.queue.shift(); |
| 460 |
} catch (e) {} |
| 461 |
} |
| 462 |
} else { |
| 463 |
const newids = adapter.queue.filter((id) => !adapter.pending.has(id)); |
| 464 |
newids.forEach((id) => adapter.pending.add(id)); |
| 465 |
const res = await adapter.f(newids); |
| 466 |
newids.forEach((id) => adapter.pending.delete(id)); |
| 467 |
for (let i=0; i<adapter.queue.length; i++) { |
| 468 |
const item = res[i]; |
| 469 |
const id = this.getid(item) ?? adapter.queue[i]; |
| 470 |
(this.index[key] ||= {})[id] = item; |
| 471 |
this.index_modified = true; |
| 472 |
if (key in this.data) { |
| 473 |
this.data[key].push(item); |
| 474 |
} |
| 475 |
} |
| 476 |
this.adapters[key].queue = []; |
| 477 |
} |
| 478 |
} |
| 479 |
} |
| 480 |
} |
| 481 |
|
| 482 |
async rehope() { |
| 483 |
Object.keys(this.index).forEach((k) => { |
| 484 |
const todelete = Object.keys(this.index[k]).filter((dk) => this.index[k][dk] == null || this.index[k][dk] === undefined); |
| 485 |
todelete.forEach((dk) => delete(this.index[k][dk])); |
| 486 |
}); |
| 487 |
} |
| 488 |
|
| 489 |
load(something, named, append=false) { |
| 490 |
if (!named) return |
| 491 |
if (!append || !(named in this.data)) this.data[named] = []; |
| 492 |
if (Array.isArray(something)) { |
| 493 |
something.forEach((somethingx) => this.data[named].push(somethingx)); |
| 494 |
} else if (typeof something == 'object') { |
| 495 |
const firstkey = Object.keys(something)[0]; |
| 496 |
const firstobj = something[firstkey]; |
| 497 |
if (typeof firstobj === 'object' && !Array.isArray(firstobj) && (firstkey in Object.values(firstobj) || 'id' in firstobj)) { |
| 498 |
this.data[named].push(...Object.values(something)); |
| 499 |
} else if (typeof firstobj === 'object' && !Array.isArray(firstobj)) { |
| 500 |
this.data[named].push(...Object.entries(firstobj).map(([key, val]) => ({id: key, ...val}))); |
| 501 |
} else if (typeof firstobj === 'string') { |
| 502 |
this.data[named].push(...Object.entries(something).map(([key, val]) => ({id: key, name: val}))); |
| 503 |
} else { |
| 504 |
this.data[named].push(...Object.entries(something).map(([key, val]) => ({id: key, value: val}))); |
| 505 |
} |
| 506 |
} else { |
| 507 |
this.data[named].push(something); |
| 508 |
} |
| 509 |
return this.data[named]; |
| 510 |
} |
| 511 |
|
| 512 |
async loadjsonl(something, named, append=false) { |
| 513 |
if (!named) return |
| 514 |
if (!append || !(named in this.data)) this.data[named] = []; |
| 515 |
if (typeof something == 'string' && (something.startsWith('http') || something.startsWith('file://'))) { |
| 516 |
const fetchres = await fetch(something); |
| 517 |
something = await fetchres.text(); |
| 518 |
} |
| 519 |
const rows = something.trim().split(/[\n\r]+/); |
| 520 |
for (const row of rows) { |
| 521 |
const rowdata = JSON.parse(row); |
| 522 |
if (rowdata) this.data[named].push(rowdata); |
| 523 |
} |
| 524 |
} |
| 525 |
|
| 526 |
async loadcsv(something, named, quoteChar = '"', delimiter = ',', headerrows=1) { |
| 527 |
if (typeof something == 'string' && (something.startsWith('http') || something.startsWith('file://'))) { |
| 528 |
const fetchres = await fetch(something); |
| 529 |
something = await fetchres.text(); |
| 530 |
} |
| 531 |
const rows = something.split(/[\n\r]+/); |
| 532 |
|
| 533 |
const regex = new RegExp(`\\s*(${quoteChar})?(.*?)\\1\\s*(?:${delimiter}|$)`, 'gs'); |
| 534 |
|
| 535 |
const match = (line) => Array.from(line.matchAll(regex), (m) => m[2]); |
| 536 |
|
| 537 |
const headers = []; |
| 538 |
for (let hrowx=0; hrowx<headerrows; hrowx++) { |
| 539 |
const hrow = rows.shift(); |
| 540 |
match(hrow).forEach((h, hx) => { |
| 541 |
if (hrowx === 0) { |
| 542 |
headers.push(h); |
| 543 |
} else { |
| 544 |
headers[hx] = headers[hx] + ' ' + h; |
| 545 |
} |
| 546 |
}); |
| 547 |
} |
| 548 |
const heads = headers ?? match(rows.shift()); |
| 549 |
const lines = rows.slice(0).filter((line) => line); |
| 550 |
const parsed = lines.map((line) => { |
| 551 |
return match(line).reduce((acc, cur, i) => { |
| 552 |
// replace blank matches with `null` |
| 553 |
const val = cur.length <= 0 ? null : (!isNaN(cur) ? Number(cur) : cur); |
| 554 |
const key = heads[i] ?? `{i}`; |
| 555 |
if (key === '') { |
| 556 |
return { ...acc}; |
| 557 |
} else { |
| 558 |
return { ...acc, [key]: val }; |
| 559 |
} |
| 560 |
}, {}); |
| 561 |
}); |
| 562 |
this.load(parsed, named); |
| 563 |
} |
| 564 |
|
| 565 |
apacheLogToDate(apacheTimestamp) { |
| 566 |
// Apache log format: [10/Oct/2000:13:55:36 -0700] |
| 567 |
// Remove brackets if present |
| 568 |
const cleanTimestamp = apacheTimestamp.replace(/^\[|]$/g, ''); |
| 569 |
|
| 570 |
// Split into date/time and timezone parts |
| 571 |
const [dateTimePart, timezone] = cleanTimestamp.split(' '); |
| 572 |
|
| 573 |
// Parse the date/time part: dd/MMM/yyyy:HH:mm:ss |
| 574 |
const [datePart, hour, minute, second] = dateTimePart.split(':'); |
| 575 |
const [day, month, year] = datePart.split('/'); |
| 576 |
|
| 577 |
// Month mapping |
| 578 |
const months = { |
| 579 |
'Jan': 0, 'Feb': 1, 'Mar': 2, 'Apr': 3, 'May': 4, 'Jun': 5, |
| 580 |
'Jul': 6, 'Aug': 7, 'Sep': 8, 'Oct': 9, 'Nov': 10, 'Dec': 11 |
| 581 |
}; |
| 582 |
|
| 583 |
// Create Date object (months are 0-indexed in JS) |
| 584 |
const date = new Date( |
| 585 |
parseInt(year), |
| 586 |
months[month], |
| 587 |
parseInt(day), |
| 588 |
parseInt(hour), |
| 589 |
parseInt(minute), |
| 590 |
parseInt(second) |
| 591 |
); |
| 592 |
|
| 593 |
// Handle timezone offset if present |
| 594 |
if (timezone) { |
| 595 |
const sign = timezone[0] === '+' ? 1 : -1; |
| 596 |
const tzHours = parseInt(timezone.slice(1, 3)); |
| 597 |
const tzMinutes = parseInt(timezone.slice(3, 5)); |
| 598 |
const offsetMs = sign * (tzHours * 60 + tzMinutes) * 60 * 1000; |
| 599 |
|
| 600 |
// Adjust for timezone (Apache logs are in local time, JS Date assumes UTC) |
| 601 |
date.setTime(date.getTime() - offsetMs); |
| 602 |
} |
| 603 |
|
| 604 |
return date; |
| 605 |
} |
| 606 |
|
| 607 |
async loadclf(loglines, named, apiroutes=[]) { |
| 608 |
const lines = loglines.trim().split('\n'); |
| 609 |
const cols9 = ['ip', 'name', 'username', 'timestamp', 'requestraw', 'status', 'bytes', 'referrer', 'useragent']; |
| 610 |
const cols10 = ['host'].concat(cols9); |
| 611 |
const parsed = lines.map((line) => { |
| 612 |
const vals = Array.from(line.matchAll(/"(?:\\"|.)*?"|\[.*?]|\S+/g)).map((m) => m[0]); |
| 613 |
const cols = vals.length === 10 ? cols10 : cols9; |
| 614 |
const obj = Object.fromEntries(vals.map((v, vx) => ([cols[vx], v]))); |
| 615 |
const date = this.apacheLogToDate(obj.timestamp); |
| 616 |
obj.timestamp = date.toISOString(); |
| 617 |
[obj.date, obj.time] = obj.timestamp.slice(0, -1).split('T'); |
| 618 |
if (obj.requestraw.match(/ [^ ]+ /) && !obj.requestraw.startsWith('"{')) { |
| 619 |
[obj.method, obj.request, obj.protocol] = obj.requestraw.slice(1, -1).split(' '); |
| 620 |
if (this.data['.clf API routes']) { |
| 621 |
for (const apiroute of this.data['.clf API routes'].sort((a, b) => b.length - a.length)) { |
| 622 |
if (obj.request.startsWith(apiroute)) { |
| 623 |
obj.page = apiroute; |
| 624 |
break; |
| 625 |
} |
| 626 |
} |
| 627 |
} |
| 628 |
obj.page ??= obj.request.split('?')[0]; |
| 629 |
} |
| 630 |
obj.referrer = obj.referrer.slice(1, -1); |
| 631 |
obj.useragent = obj.useragent.slice(1, -1); |
| 632 |
obj.logline = line; |
| 633 |
return obj; |
| 634 |
}); |
| 635 |
this.load(parsed, named); |
| 636 |
} |
| 637 |
|
| 638 |
async loadrss(rsstext, named) { |
| 639 |
const rssval = (k, rawval) => { |
| 640 |
if (k.match(/date/i)) { |
| 641 |
return new Date(rawval).toISOString(); |
| 642 |
} else if (!isNaN(rawval)) { |
| 643 |
return Number(rawval); |
| 644 |
} else { |
| 645 |
return rawval; |
| 646 |
} |
| 647 |
} |
| 648 |
const rssdom = new window.DOMParser().parseFromString(rsstext, "text/xml"); |
| 649 |
const items = Array.from(rssdom.querySelectorAll('item')).map((i) => { |
| 650 |
const obj = {}; |
| 651 |
Array.from(i.children).forEach((c) => { |
| 652 |
const k = c.tagName; |
| 653 |
const rawval = c.textContent; |
| 654 |
obj[k] = rssval(k, rawval); |
| 655 |
}); |
| 656 |
return obj; |
| 657 |
}); |
| 658 |
this.load(items, named); |
| 659 |
} |
| 660 |
|
| 661 |
connect(key, adapter, annotator=false) { |
| 662 |
this.adapters[key] = {queue: [], pending: new Set(), f: adapter, annotator: annotator}; |
| 663 |
this.data.adapters ??= []; |
| 664 |
if (!this.data.adapters.includes(key)) this.data.adapters.push(key); |
| 665 |
} |
| 666 |
|
| 667 |
connect_annotator(key, adapter) { |
| 668 |
this.connect(key, adapter, true); |
| 669 |
} |
| 670 |
|
| 671 |
register(key, aggregator) { |
| 672 |
this.aggregators[key] = aggregator; |
| 673 |
this.data.aggregators ??= []; |
| 674 |
if (!this.data.aggregators.includes(key)) this.data.aggregators.push(key); |
| 675 |
} |
| 676 |
|
| 677 |
unbracket(token) { |
| 678 |
if (!(typeof token == 'string')) token = token.toString(); |
| 679 |
if (token.startsWith('[') && token.endsWith(']')) { |
| 680 |
return token.slice(1, -1).replaceAll(']]', ']'); |
| 681 |
} |
| 682 |
return token; |
| 683 |
} |
| 684 |
|
| 685 |
bracket(token) { |
| 686 |
if (!(typeof token == 'string')) token = token?.toString() ?? ''; |
| 687 |
if (token.match(/[?.:#\/|!<>=~@\[\](),;+-]/) || token.startsWith(' ') || token.endsWith(' ')) { |
| 688 |
return '[' + token.replaceAll(']', ']]') + ']'; |
| 689 |
} |
| 690 |
return token; |
| 691 |
} |
| 692 |
|
| 693 |
escapeRegExp(string) { |
| 694 |
return string.replace(/[.*+?${}()|[\]\\]/g, "\\$&"); |
| 695 |
} |
| 696 |
|
| 697 |
tokenize(text) { |
| 698 |
if (typeof text != 'string') text = String(text); |
| 699 |
const matches = text.matchAll(/(\[(?:]]|[^\]])*])|(\?{1,3})|(\.{1,4})|(\|{1,2})|(\/{1,2})|([:#!])|(\()|(\))|([@~=<>+-]+)|(,)|(;)|([^\[\]().?:\/#|!~=<>@,;+-]+)|([\[\]])/gms); |
| 700 |
return Array.from(matches, m => m[0].trim()).filter((token) => token.length > 0); |
| 701 |
} |
| 702 |
|
| 703 |
assemble(tokenized) { |
| 704 |
const isOperator = (token) => '???....://#||!'.includes(token); |
| 705 |
const tokenlist = '= ~ =~ ~< ~> > < <> >< >= <= - =- ~- + =+ @ @@ @- @= @@= =@ =@@ @< @@< @<= @@<= @> @@> @>= @@>= >> >>= << <<= ~~ =>'.split(' '); |
| 706 |
const isSubop = (token) => tokenlist.includes(token) || (token?.startsWith('-') && (token.length === 1 || tokenlist.includes(token.slice(1)))); |
| 707 |
const isSeparator = (token) => ',;'.includes(token); |
| 708 |
const isValue = (token) => !isOperator(token) && !isSubop(token) && !isSeparator(token); |
| 709 |
|
| 710 |
let tokens = tokenized.slice(); |
| 711 |
let level = 0; |
| 712 |
const operations = []; |
| 713 |
while (tokens.length > 0) { |
| 714 |
const op = {operator: null, args: []}; |
| 715 |
let token = tokens.shift(); |
| 716 |
if (isOperator(token) || ((isValue(token) || isSubop(token)) && operations.length === 0)) { |
| 717 |
if ((isValue(token) || isSubop(token)) && operations.length === 0) { |
| 718 |
op.operator = '?'; |
| 719 |
tokens.unshift(token); |
| 720 |
} else { |
| 721 |
op.operator = token; |
| 722 |
} |
| 723 |
while (tokens.length > 0 && !isOperator(tokens[0])) { |
| 724 |
const arg = {separator: null, label: null, subop: null, value: null}; |
| 725 |
if (isSeparator(tokens[0])) { |
| 726 |
arg.separator = tokens.shift(); |
| 727 |
} |
| 728 |
while (tokens.length > 0 && !isOperator(tokens[0]) && !isSeparator(tokens[0])) { |
| 729 |
const frag = tokens.shift(); |
| 730 |
if (frag !== '(' && isValue(frag) && isSubop(tokens[0])) { |
| 731 |
arg.label = this.unbracket(frag); |
| 732 |
arg.subop = tokens.shift(); |
| 733 |
} else if (isSubop(frag)) { |
| 734 |
arg.subop = frag; |
| 735 |
} else if (frag === '(') { |
| 736 |
const subquery = []; |
| 737 |
level++; |
| 738 |
while (tokens.length > 0 && level > 0) { |
| 739 |
const sub = tokens.shift(); |
| 740 |
if (sub === '(') { |
| 741 |
level++; |
| 742 |
if (level > 0) { |
| 743 |
subquery.push(sub); |
| 744 |
} |
| 745 |
} else if (sub === ')') { |
| 746 |
level--; |
| 747 |
if (level > 0) { |
| 748 |
subquery.push(sub); |
| 749 |
} |
| 750 |
} else if (level > 0) { |
| 751 |
subquery.push(sub); |
| 752 |
} |
| 753 |
} |
| 754 |
arg.value = this.assemble(subquery); |
| 755 |
} else if (!arg.value) { |
| 756 |
arg.value = this.unbracket(frag); |
| 757 |
} else { |
| 758 |
const failure = {tokens: tokenized, assembled: operations.slice(0), assembling: {op: op, arg: arg, unexpected: frag}, unassembled: tokens.slice(0)} |
| 759 |
throw new Error("Unexpected token", {cause: failure}); |
| 760 |
} |
| 761 |
} |
| 762 |
op.args.push(arg); |
| 763 |
} |
| 764 |
} |
| 765 |
operations.push(op); |
| 766 |
|
| 767 |
} |
| 768 |
if (level > 0) console.warn({parentropy: level, operations: operations}); |
| 769 |
return operations; |
| 770 |
} |
| 771 |
|
| 772 |
parse(querystr) { |
| 773 |
return this.assemble(this.tokenize(querystr)); |
| 774 |
} |
| 775 |
|
| 776 |
disassemble(assembly) { |
| 777 |
return assembly.map((operation, ox) => operation.operator + ((operation.args.length === 0 && ox < assembly.length - 1 && operation.operator[0] === assembly[ox + 1].operator[0]) ? ' ' : operation.args.map((arg) => (arg.separator ?? '') + (arg.label ? this.bracket(arg.label) : '') + (arg.subop ?? '') + (Array.isArray(arg.value) ? ('(' + this.disassemble(arg.value) + ')') : (arg.value ? this.bracket(arg.value) : ''))).join(''))).join(''); |
| 778 |
} |
| 779 |
|
| 780 |
compact(querystr) { |
| 781 |
return this.disassemble(this.parse(querystr)); |
| 782 |
} |
| 783 |
|
| 784 |
executeq(querystr) { |
| 785 |
return this.execute([], this.assemble(this.tokenize(querystr))); |
| 786 |
} |
| 787 |
|
| 788 |
timecheck(timer, i, count, op) { |
| 789 |
if (i === 1) timer.loopstart = new Date(); |
| 790 |
if (i >= 10) { |
| 791 |
const taken = new Date() - timer.loopstart; |
| 792 |
const projected = count * taken / i; |
| 793 |
if (projected > this.timelimit) throw new Error('Query overrun.', {cause: {operation: dactal.disassemble([op]), items: count, done: i, elapsed: taken + 'ms', projected: Math.round(projected / 60000) + ' minutes', timelimit: Math.round(this.timelimit / 60000) + ' minutes'}}); |
| 794 |
} |
| 795 |
} |
| 796 |
|
| 797 |
execute(inputlist, operations, labeled=null, level=null) { |
| 798 |
let currentlist = inputlist?.slice(0) || []; |
| 799 |
const getname = this.getname; |
| 800 |
const getid = this.getid; |
| 801 |
const dtype = this.dtype; |
| 802 |
const dcopy = this.dcopy; |
| 803 |
const step = this.step; |
| 804 |
const escapeRegExp = this.escapeRegExp; |
| 805 |
const unmmss = (mmssstr) => { |
| 806 |
const parts = mmssstr.split(':'); |
| 807 |
let s = 0; |
| 808 |
const seconds = parts.pop(); |
| 809 |
if (seconds) s += Number(seconds); |
| 810 |
const minutes = parts.pop(); |
| 811 |
if (minutes) s += 60 * Number(minutes); |
| 812 |
const hours = parts.pop(); |
| 813 |
if (hours) s += 60 * 60 * Number(hours); |
| 814 |
return s; |
| 815 |
}; |
| 816 |
const compvals = (araw, braw) => { |
| 817 |
const a = araw.toString(); |
| 818 |
const b = braw.toString(); |
| 819 |
const atime = a.match(/^\d+(?::\d{2,})+$/); |
| 820 |
const btime = b.match(/^\d+(?::\d{2,})+$/); |
| 821 |
if (atime && btime) { |
| 822 |
return unmmss(btime[0]) - unmmss(atime[0]); |
| 823 |
} |
| 824 |
return dethe(a).localeCompare(dethe(b)); |
| 825 |
} |
| 826 |
|
| 827 |
const dethe = (value) => value.toLowerCase().replace(/^the /, ''); |
| 828 |
|
| 829 |
labeled ||= {}; |
| 830 |
let outputlist = []; |
| 831 |
for (let opx=0; opx<operations.length; opx++) { |
| 832 |
const op = operations[opx]; |
| 833 |
outputlist = []; |
| 834 |
switch (op.operator) { |
| 835 |
case '?': // start |
| 836 |
if (op.args.length === 0) { |
| 837 |
outputlist = Object.keys(this.data).sort((a, b) => dethe(a).localeCompare(dethe(b))); |
| 838 |
break; |
| 839 |
} |
| 840 |
outputlist = []; |
| 841 |
op.args.forEach((arg) => { |
| 842 |
let startitems; |
| 843 |
if (arg.label && arg.subop?.match(/\+/)) labeled[arg.label] ||= []; |
| 844 |
if (arg.separator !== ';' || outputlist.length === 0) { |
| 845 |
if (Array.isArray(arg.value)) { |
| 846 |
startitems = this.execute([], arg.value, labeled); |
| 847 |
if (arg.label) { |
| 848 |
if (arg.subop?.match(/\+/)) { |
| 849 |
startitems.forEach((i) => labeled[arg.label].push(i)); |
| 850 |
} else { |
| 851 |
labeled[arg.label] = startitems; |
| 852 |
this.index[arg.label] = {}; |
| 853 |
} |
| 854 |
labeled[arg.label].forEach((i) => outputlist.push(i)); |
| 855 |
} else { |
| 856 |
startitems.forEach((i) => outputlist.push(i)); |
| 857 |
} |
| 858 |
} else if (arg.subop === '~') { |
| 859 |
outputlist.push(arg.value); |
| 860 |
} else if (arg.value in labeled) { |
| 861 |
labeled[arg.value].forEach((i) => outputlist.push(i)); |
| 862 |
} else if (startitems = this.gettype(arg.value, false)) { |
| 863 |
startitems.forEach((i) => outputlist.push(i)); |
| 864 |
} else if (arg.value?.trim().length > 0 && this.dtype(arg.value, 'number')) { |
| 865 |
if (arg.subop === '-' && arg.value === '0') { |
| 866 |
outputlist.push(-0); |
| 867 |
} else { |
| 868 |
outputlist.push((arg.subop === '-' ? -1 : 1) * Number(arg.value)); |
| 869 |
} |
| 870 |
} else if (arg.value && arg.subop !== '=') { |
| 871 |
outputlist.push(arg.value); |
| 872 |
} |
| 873 |
} |
| 874 |
}); |
| 875 |
break; |
| 876 |
case '??': // label |
| 877 |
outputlist = currentlist; |
| 878 |
let ended = false; |
| 879 |
op.args.forEach((arg) => { |
| 880 |
if (!ended) { |
| 881 |
if (arg.label === '_timelimit' && !isNaN(arg.value)) { |
| 882 |
this.timelimit = Number(arg.value) * 60000; |
| 883 |
} else if (arg.label) { |
| 884 |
if (arg.subop.includes('~') && dtype(arg.value, 'string')) { |
| 885 |
labeled[arg.label] = [arg.value]; |
| 886 |
} else if (arg.labeled) { |
| 887 |
labeled[arg.label] = arg.labeled; |
| 888 |
} else { |
| 889 |
if (arg.subop?.match(/\+/)) labeled[arg.label] ||= []; |
| 890 |
const newvals = this.execute(currentlist, Array.isArray(arg.value) ? arg.value : '.' + arg.value, labeled, level); |
| 891 |
if (arg.subop?.match(/\+/)) { |
| 892 |
newvals.forEach((nv) => labeled[arg.label].push(nv)); |
| 893 |
} else { |
| 894 |
labeled[arg.label] = newvals; |
| 895 |
} |
| 896 |
if (!level && opx === 0) arg.labeled = newvals; |
| 897 |
} |
| 898 |
this.index[arg.label] = {}; |
| 899 |
} else if (dtype(arg.value, 'string')) { |
| 900 |
if (arg.subop?.match(/\+/)) { |
| 901 |
labeled[arg.value] ??= []; |
| 902 |
currentlist.forEach((i) => labeled[arg.value].push(i)); |
| 903 |
} else { |
| 904 |
labeled[arg.value] = currentlist.slice(0); |
| 905 |
} |
| 906 |
this.index[arg.value] = {}; |
| 907 |
if (arg.value === 'end') { |
| 908 |
ended = true; |
| 909 |
} |
| 910 |
} |
| 911 |
} |
| 912 |
}); |
| 913 |
if (ended) return (this.debug && !inputlist) ? operations : outputlist; |
| 914 |
break; |
| 915 |
case '!': // repeat |
| 916 |
if (opx > 0) { |
| 917 |
const repeat_ops = operations.slice(opx - 1, opx + 1); |
| 918 |
outputlist = currentlist.slice(0); |
| 919 |
level ??= 0; |
| 920 |
level += 1; |
| 921 |
const maxrecursion = ((op.args.length > 0 && op.args[0].value) || 1000); |
| 922 |
if (outputlist.length > 0 && level < maxrecursion && (!this.samearray(inputlist, outputlist) || level === 1)) { |
| 923 |
const recursed = this.execute(outputlist, repeat_ops, labeled, level); |
| 924 |
if (recursed.length > 0 && !this.samearray(recursed, outputlist)) outputlist = recursed.slice(0); |
| 925 |
} |
| 926 |
} |
| 927 |
break; |
| 928 |
case '.': // traverse |
| 929 |
case '..': // traverse with duplicates |
| 930 |
const seen = new Set(); |
| 931 |
const sofar = Array.isArray(op.args?.[0]?.value) && op.args?.[0]?.label; |
| 932 |
let transq = null; |
| 933 |
if (sofar) labeled[sofar] = []; |
| 934 |
if (op.operator === '.' && op.args?.length === 1) { |
| 935 |
const firstarg = op.args[0]; |
| 936 |
const firstval = firstarg?.value; |
| 937 |
if (dtype(firstval, 'string')) { |
| 938 |
transq = this.data.queries?.find((q) => q.relative === true && q.name === firstval); |
| 939 |
} |
| 940 |
} |
| 941 |
if (transq) { |
| 942 |
outputlist = this.execute(currentlist, this.parse(transq.query), labeled, level); |
| 943 |
} else { |
| 944 |
const traversetimer = {}; |
| 945 |
outputlist = currentlist.reduce((acc, item, i) => { |
| 946 |
this.timecheck(traversetimer, i, currentlist.length, op); |
| 947 |
if (op.args.length === 0) { |
| 948 |
const itemkey = getid(item); |
| 949 |
if (op.operator === '..' || !seen.has(itemkey)) { |
| 950 |
acc.push(item); |
| 951 |
seen.add(itemkey); |
| 952 |
} |
| 953 |
return acc; |
| 954 |
} |
| 955 |
if (op.args[0].subop?.includes('<') && acc.length > 0) return acc; |
| 956 |
let itemvals = []; |
| 957 |
let toremoveids = {}; |
| 958 |
for (const arg of op.args.filter((arg) => arg.value != null)) { |
| 959 |
let passdown = null; |
| 960 |
if (arg.subop?.includes('>') && typeof arg.value == 'string') { |
| 961 |
if (arg?.label in item) { |
| 962 |
passdown = item[arg.label]; |
| 963 |
} else { |
| 964 |
passdown = JSON.parse(JSON.stringify(item, Object.keys(item).filter((k) => k !== arg.value))); |
| 965 |
} |
| 966 |
} |
| 967 |
if (arg.separator !== ';' || itemvals.length === 0) { |
| 968 |
const itemval = step(item, arg.value, arg.subop, labeled); |
| 969 |
if (arg.subop?.includes('-') && isNaN(arg.value)) { |
| 970 |
itemval.forEach((subitem) => { |
| 971 |
const subid = getid(subitem); |
| 972 |
toremoveids[subid] ||= 0; |
| 973 |
toremoveids[subid] += 1; |
| 974 |
}); |
| 975 |
} else { |
| 976 |
itemval.forEach((subitem) => { |
| 977 |
if (passdown) { |
| 978 |
if (!dtype(subitem, 'object')) subitem = {value: subitem}; |
| 979 |
if (arg.label) { |
| 980 |
subitem[arg.label] = [passdown]; |
| 981 |
} else { |
| 982 |
Object.assign(subitem, passdown); |
| 983 |
} |
| 984 |
} |
| 985 |
const subitemkey = getid(subitem); |
| 986 |
if (op.operator === '..' || !seen.has(subitemkey)) { |
| 987 |
seen.add(subitemkey); |
| 988 |
itemvals.push(subitem); |
| 989 |
if (sofar) labeled[sofar].push(subitem); |
| 990 |
} |
| 991 |
}) |
| 992 |
} |
| 993 |
} |
| 994 |
if (Object.keys(toremoveids).length > 0) { |
| 995 |
if (op.operator === '.') { |
| 996 |
itemvals = itemvals.filter((subitem) => !(getid(subitem) in toremoveids)); |
| 997 |
} else { |
| 998 |
itemvals = itemvals.filter((subitem) => { |
| 999 |
const subid = getid(subitem); |
| 1000 |
if (toremoveids[subid] > 0) { |
| 1001 |
toremoveids[subid] -= 1; |
| 1002 |
return false; |
| 1003 |
} else { |
| 1004 |
return true; |
| 1005 |
} |
| 1006 |
}) |
| 1007 |
} |
| 1008 |
} |
| 1009 |
} |
| 1010 |
itemvals.forEach((x) => acc.push(x)); |
| 1011 |
return acc; |
| 1012 |
}, []); |
| 1013 |
} |
| 1014 |
outputlist = outputlist.filter((item) => item != null); |
| 1015 |
break; |
| 1016 |
case ':': // filter |
| 1017 |
if (!op?.args?.length > 0) { |
| 1018 |
outputlist = currentlist; |
| 1019 |
break; |
| 1020 |
} |
| 1021 |
for(const arg of op.args) { |
| 1022 |
if (arg.subop && ['>>', '>>=', '<<', '<<=', '~~'].includes(arg.subop.replace(/^-/, ''))) { |
| 1023 |
arg.tests = {id: new Set(), name: new Set()}; |
| 1024 |
if (Array.isArray(arg.value)) { |
| 1025 |
const testitems = this.execute(currentlist, arg.value, labeled); |
| 1026 |
testitems.forEach((testitem) => arg.tests.id.add(getid(testitem))); |
| 1027 |
testitems.forEach((testitem) => arg.tests.name.add(getname(testitem))); |
| 1028 |
} |
| 1029 |
|
| 1030 |
if (['>>', '>>=', '<<', '<<='].includes(arg.subop)) { |
| 1031 |
for (const [tryval, tryf] of [['id', getid], ['name', getname]]) { |
| 1032 |
for (let j=0; j < currentlist.length; j++) { |
| 1033 |
if (arg.tests[tryval].has(tryf(currentlist[j]))) { |
| 1034 |
arg.matchpos = j; |
| 1035 |
break; |
| 1036 |
} |
| 1037 |
} |
| 1038 |
if (arg.matchpos) break |
| 1039 |
} |
| 1040 |
} |
| 1041 |
} else if ([null, '', '=', '~'].includes((arg.subop || '').replace(/-/, '')) && dtype(arg.value, 'string') && arg.value.startsWith('~')) { |
| 1042 |
const flags = arg.value.startsWith('~~') ? 'i' : ''; |
| 1043 |
const pattern = arg.value.replace(/^~*/, ''); |
| 1044 |
const fullpattern = (arg.subop || '').replace(/-/, '') === '=' ? ((pattern.startsWith('^') ? '' : '^') + pattern + (pattern.endsWith('$') ? '' : '$')) : pattern; |
| 1045 |
arg.re = new RegExp(fullpattern, flags); |
| 1046 |
} |
| 1047 |
} |
| 1048 |
|
| 1049 |
const ands = [[]]; |
| 1050 |
for (const arg of op.args) { |
| 1051 |
if (arg.separator === ';') { |
| 1052 |
ands.push([arg]); |
| 1053 |
} else { |
| 1054 |
ands[ands.length - 1].push(arg) |
| 1055 |
} |
| 1056 |
} |
| 1057 |
const filtertimer = {}; |
| 1058 |
outputlist = currentlist.filter((item, i) => { |
| 1059 |
this.timecheck(filtertimer, i, currentlist.length, op); |
| 1060 |
return ands.filter((and) => { |
| 1061 |
return and.filter((arg) => { |
| 1062 |
let comparator = arg.subop; |
| 1063 |
let polarize = (x) => x; |
| 1064 |
if (comparator?.startsWith('-') || comparator?.endsWith('-')) { |
| 1065 |
polarize = (x) => !x; |
| 1066 |
comparator = comparator.replace(/-|-$/, ''); |
| 1067 |
} |
| 1068 |
if (!comparator && arg.label == null && Array.isArray(arg.value)) { |
| 1069 |
return polarize(this.execute([item], arg.value, labeled).length > 0); |
| 1070 |
} else if (comparator === '~~') { |
| 1071 |
if (arg.label == null && dtype(arg.value, 'number')) { |
| 1072 |
return polarize(currentlist.length === Number(arg.value)); |
| 1073 |
} else { |
| 1074 |
if (arg.label) { |
| 1075 |
const argitems = step(item, arg.label, null, labeled); |
| 1076 |
return polarize(argitems.find((argitem) => arg.tests.id.has(getid(argitem)) || arg.tests.name.has(getname(argitem)))); |
| 1077 |
} else { |
| 1078 |
return polarize(arg.tests.id.has(getid(item)) || (item?.id === undefined && arg.tests.name.has(getname(item)))); |
| 1079 |
} |
| 1080 |
} |
| 1081 |
} else if (['~~', '<<', '<<=', '>>', '>>='].includes(comparator) && dtype(arg.value, 'number')) { |
| 1082 |
const argval = Number(arg.value); |
| 1083 |
let testlength; |
| 1084 |
if (typeof arg.label == 'string') { |
| 1085 |
testlength = step(item, arg.label, null, labeled).length; |
| 1086 |
} else { |
| 1087 |
testlength = currentlist.length; |
| 1088 |
} |
| 1089 |
switch (comparator) { |
| 1090 |
case '~~': return polarize(testlength === argval); |
| 1091 |
case '<<': return polarize(testlength < argval); |
| 1092 |
case '<<=': return polarize(testlength <= argval); |
| 1093 |
case '>>': return polarize(testlength > argval); |
| 1094 |
case '>>=': return polarize(testlength >= argval); |
| 1095 |
} |
| 1096 |
} else if (['+', ''].includes(comparator) && arg.label && !arg.value && dtype(arg.label, 'string') && dtype(item, 'object')) { |
| 1097 |
const propval = item[arg.label]; |
| 1098 |
// console.log({plusminus: comparator, label: arg.label, propval: propval}) |
| 1099 |
return polarize(Array.isArray(propval) ? propval.length > 0 : propval); |
| 1100 |
} |
| 1101 |
|
| 1102 |
let testitems = [item]; |
| 1103 |
if (arg.label && dtype(item, 'object')) { |
| 1104 |
let found = false; |
| 1105 |
for (const tryval of [arg.label, arg.label + 's']) { |
| 1106 |
if (item?.[tryval] != null) { |
| 1107 |
const propitems = item[tryval]; |
| 1108 |
if (propitems != null) { |
| 1109 |
found = true; |
| 1110 |
testitems = Array.isArray(propitems) ? propitems : [propitems]; |
| 1111 |
break; |
| 1112 |
} |
| 1113 |
} |
| 1114 |
} |
| 1115 |
if (!found) { |
| 1116 |
if (dactal.savedquerynames.has(arg.label) || (labeled['=>'] && arg.label in labeled['=>'])) { |
| 1117 |
testitems = step(item, arg.label, null, labeled); |
| 1118 |
} else { |
| 1119 |
testitems = []; |
| 1120 |
} |
| 1121 |
} |
| 1122 |
} |
| 1123 |
|
| 1124 |
const testvals = testitems.map((testitem) => { |
| 1125 |
if (comparator || arg.re) { |
| 1126 |
return getname(testitem); |
| 1127 |
} else if (this.dtype(testitem, 'literal')) { |
| 1128 |
return testitem; |
| 1129 |
} else if ('id' in testitem) { |
| 1130 |
return testitem.id; |
| 1131 |
} else { |
| 1132 |
return getname(testitem) |
| 1133 |
} |
| 1134 |
}); |
| 1135 |
|
| 1136 |
let argvals; |
| 1137 |
if (Array.isArray(arg.value)) { |
| 1138 |
const argvalitems = this.execute([item], arg.value, labeled); |
| 1139 |
argvals = argvalitems.map((argvalitem) => { |
| 1140 |
let argval; |
| 1141 |
if (dtype(argvalitem, 'literal')) { |
| 1142 |
argval = argvalitem; |
| 1143 |
} else { |
| 1144 |
argval = getname(argvalitem); |
| 1145 |
} |
| 1146 |
if (argval == null) { |
| 1147 |
const argvalitemkeys = Object.keys(argvalitem).filter((key) => key !== 'id'); |
| 1148 |
if (argvalitemkeys.length === 1) { |
| 1149 |
argval = argvalitem[argvalitemkeys[0]]; |
| 1150 |
} |
| 1151 |
} |
| 1152 |
return argval; |
| 1153 |
}); |
| 1154 |
} |
| 1155 |
|
| 1156 |
return testvals.find((testval) => { |
| 1157 |
if (!argvals) { |
| 1158 |
if (dtype(testval, 'number') && dtype(arg.value, 'number')) { |
| 1159 |
testval = Number(testval); |
| 1160 |
argvals = [Number(arg.value)]; |
| 1161 |
} else { |
| 1162 |
argvals = [arg.value]; |
| 1163 |
} |
| 1164 |
} |
| 1165 |
if (comparator?.startsWith('@')) { |
| 1166 |
if (dtype(arg.value, 'number')) { |
| 1167 |
argvals = [Number(arg.value)]; |
| 1168 |
} else if (arg.value in labeled) { |
| 1169 |
argvals = Number(labeled[arg.value]); |
| 1170 |
if (!Array.isArray(argvals)) argvals = [argvals]; |
| 1171 |
} |
| 1172 |
if (comparator.startsWith('@@')) { |
| 1173 |
testval = currentlist.length - i; |
| 1174 |
comparator = comparator.slice(2); |
| 1175 |
} else { |
| 1176 |
testval = i + 1; |
| 1177 |
comparator = comparator.slice(1); |
| 1178 |
} |
| 1179 |
} else if (['<<', '<<=', '>>', '>>='].includes(comparator) && dtype(arg.value, 'number')) { |
| 1180 |
const argval = Number(arg.value); |
| 1181 |
switch (comparator) { |
| 1182 |
case '<<': return polarize(currentlist.length < argval); |
| 1183 |
case '<<=': return polarize(currentlist.length <= argval); |
| 1184 |
case '>>': return polarize(currentlist.length > argval); |
| 1185 |
case '>>=': return polarize(currentlist.length >= argval); |
| 1186 |
} |
| 1187 |
} |
| 1188 |
if (testval == null || argvals == null || argvals.length === 0) return false; |
| 1189 |
return argvals.find((argval) => { |
| 1190 |
if (arg.re) { |
| 1191 |
return polarize(testval.match(arg.re)); |
| 1192 |
} else { |
| 1193 |
switch (comparator) { |
| 1194 |
case null: return polarize(testval === argval); |
| 1195 |
case '': return polarize(testval === argval); |
| 1196 |
case '=': return polarize(testval === argval); |
| 1197 |
case '>=': return polarize(testval >= argval); |
| 1198 |
case '<=': return polarize(testval <= argval); |
| 1199 |
case '>': return polarize(testval > argval); |
| 1200 |
case '<': return polarize(testval < argval); |
| 1201 |
case '~': return polarize(testval?.toString().toLowerCase().includes(argval?.toString().toLowerCase())); |
| 1202 |
case '~<': return polarize(testval?.toString().toLowerCase().startsWith(argval?.toString().toLowerCase())); |
| 1203 |
case '~>': return polarize(testval?.toString().toLowerCase().endsWith(argval?.toString().toLowerCase())); |
| 1204 |
case '>>': return polarize(i > arg.matchpos); |
| 1205 |
case '>>=': return polarize(i >= arg.matchpos); |
| 1206 |
case '<<': return polarize(i < arg.matchpos); |
| 1207 |
case '<<=': return polarize(i <= arg.matchpos); |
| 1208 |
} |
| 1209 |
} |
| 1210 |
}) != null; |
| 1211 |
}) != null; |
| 1212 |
}).length > 0; |
| 1213 |
}).length === ands.length; |
| 1214 |
}); |
| 1215 |
break; |
| 1216 |
case '#': // sort |
| 1217 |
const sortargs = op.args.slice(0); |
| 1218 |
const lastarg = sortargs[sortargs.length - 1]; |
| 1219 |
let temped = false; |
| 1220 |
if (!lastarg || ![';', '=;'].includes(Object.values(lastarg).join(''))) sortargs.push(...[{subop: null, value: 'name'}, {subop: null, value: 'id'}]); |
| 1221 |
if (!dtype(currentlist[0], 'object') && lastarg && lastarg.separator === ';') { |
| 1222 |
currentlist = currentlist.map((v) => ({_value: v})); |
| 1223 |
temped = true; |
| 1224 |
} else if (sortargs[0]?.label) { |
| 1225 |
currentlist = currentlist.map((i) => dcopy(i)); |
| 1226 |
} |
| 1227 |
sortargs.forEach((arg) => { |
| 1228 |
arg.extraindex = {}; |
| 1229 |
const stablesort = (arg.label == null && arg.value == null && arg.separator === ';' ) ? (arg.subop === '-' ? -1 : 1) : null; |
| 1230 |
if (arg.subop?.includes('~')) { |
| 1231 |
arg.sortmode = 'literal'; |
| 1232 |
} else if (arg.subop?.endsWith('-')) { |
| 1233 |
arg.sortmode = 'numeric'; |
| 1234 |
} else if (arg.subop?.endsWith('+')) { |
| 1235 |
arg.sortmode = 'rank'; |
| 1236 |
} else { |
| 1237 |
arg.sortmode = stablesort || ['rank', 'index', 'number', 'id'].includes(arg.value) ? 'rank' : 'numeric'; |
| 1238 |
} |
| 1239 |
const vals = new Set(); |
| 1240 |
const extradone = new Set(); |
| 1241 |
for (let ix=0; ix<currentlist.length; ix++) { |
| 1242 |
const item = currentlist[ix]; |
| 1243 |
const vallist = stablesort ? [ix] : (arg.value ? step(item, arg.value, null, labeled) : (Array.isArray(item) ? item : [item])); |
| 1244 |
if (vallist && !stablesort) vallist.forEach((val) => vals.add(val)); |
| 1245 |
if (typeof item == 'object') { |
| 1246 |
(item._sortindex ||= []).push(vallist); |
| 1247 |
} else if (!dtype(item, 'number') && !extradone.has(item)) { |
| 1248 |
arg.extraindex[item] = vallist; |
| 1249 |
extradone.add(item); |
| 1250 |
} |
| 1251 |
} |
| 1252 |
if (arg.sortmode !== 'literal') { |
| 1253 |
for (const val of vals) { |
| 1254 |
if (val != null && !dtype(val, 'number') && !(dtype(val, 'object') && dtype(getname(val), 'number'))) { |
| 1255 |
arg.sortmode = null; |
| 1256 |
break; |
| 1257 |
} |
| 1258 |
} |
| 1259 |
} |
| 1260 |
}) |
| 1261 |
|
| 1262 |
outputlist = currentlist.sort((a, b) => { |
| 1263 |
let comp = 0; |
| 1264 |
let ii = 0; |
| 1265 |
for (const arg of sortargs) { |
| 1266 |
const alist = typeof a == 'object' ? a._sortindex[ii] : dtype(a, 'number') ? [Number(a)] : arg.extraindex[a]; |
| 1267 |
const blist = typeof b == 'object' ? b._sortindex[ii] : dtype(b, 'number') ? [Number(b)] : arg.extraindex[b]; |
| 1268 |
ii++; |
| 1269 |
if (alist && blist) { |
| 1270 |
if (arg?.subop?.includes('@')) { |
| 1271 |
const alookup = alist.indexOf(getname(a)); |
| 1272 |
const blookup = blist.indexOf(getname(b)); |
| 1273 |
if (alookup >-1 && blookup === -1) { |
| 1274 |
comp = -1; |
| 1275 |
} else if (alookup === -1 && blookup > -1) { |
| 1276 |
comp = 1; |
| 1277 |
} else { |
| 1278 |
comp = alist.indexOf(getname(a)) - alist.indexOf(getname(b)) |
| 1279 |
} |
| 1280 |
} else { |
| 1281 |
for (let i=0; i < Math.min(alist.length, blist.length); i++) { |
| 1282 |
let aitem = alist[i]; |
| 1283 |
let bitem = blist[i]; |
| 1284 |
if (aitem != null && bitem == null) { |
| 1285 |
comp = -1; |
| 1286 |
} else if (aitem == null && bitem != null) { |
| 1287 |
comp = 1; |
| 1288 |
} else { |
| 1289 |
switch (arg.sortmode) { |
| 1290 |
case 'literal': |
| 1291 |
comp = aitem < bitem ? -1 : (bitem < aitem ? 1 : 0); |
| 1292 |
break; |
| 1293 |
case 'numeric': |
| 1294 |
let bnum = Number(bitem); |
| 1295 |
let anum = Number(aitem); |
| 1296 |
if (isNaN(bnum) || isNaN(anum)) { |
| 1297 |
bnum = Number(getname(bitem)); |
| 1298 |
anum = Number(getname(aitem)); |
| 1299 |
} |
| 1300 |
if (isNaN(bnum) || isNaN(anum)) { |
| 1301 |
comp = compvals(aitem, bitem) |
| 1302 |
} else { |
| 1303 |
comp = bnum - anum; |
| 1304 |
} |
| 1305 |
break; |
| 1306 |
case 'rank': |
| 1307 |
comp = Number(aitem) - Number(bitem); |
| 1308 |
break; |
| 1309 |
default: |
| 1310 |
if (dtype(aitem, 'literal') && dtype(bitem, 'literal')) { |
| 1311 |
comp = compvals(aitem, bitem); |
| 1312 |
} else if (typeof aitem == 'object' && typeof bitem == 'object') { |
| 1313 |
let aval = getname(aitem); |
| 1314 |
let bval = getname(bitem); |
| 1315 |
if (aval != null && bval != null) { |
| 1316 |
comp = compvals(aval, bval); |
| 1317 |
} else { |
| 1318 |
aval = getid(aitem); |
| 1319 |
bval = getid(bitem); |
| 1320 |
if (aval != null && bval != null) { |
| 1321 |
comp = compvals(aval, bval); |
| 1322 |
} else { |
| 1323 |
comp = 0; |
| 1324 |
} |
| 1325 |
} |
| 1326 |
} else if (aitem != null && bitem == null) { |
| 1327 |
comp = -1; |
| 1328 |
} else if (aitem == null && bitem != null) { |
| 1329 |
comp = 1; |
| 1330 |
} |
| 1331 |
break; |
| 1332 |
} |
| 1333 |
} |
| 1334 |
if (comp !== 0) break; |
| 1335 |
} |
| 1336 |
} |
| 1337 |
} |
| 1338 |
if (comp === 0) { |
| 1339 |
if (alist && blist) { |
| 1340 |
comp = blist.length - alist.length; |
| 1341 |
} else if (alist) { |
| 1342 |
comp = -1; |
| 1343 |
} else if (blist) { |
| 1344 |
comp = 1; |
| 1345 |
} |
| 1346 |
} |
| 1347 |
if (arg?.subop?.includes('-') && (arg.sortmode === 'literal' || !arg.sortmode)) { |
| 1348 |
comp = -comp; |
| 1349 |
} |
| 1350 |
if (comp !== 0) break; |
| 1351 |
} |
| 1352 |
return comp; |
| 1353 |
}) |
| 1354 |
if (temped) outputlist = outputlist.map((vt) => vt._value); |
| 1355 |
outputlist.forEach((item, i) => { |
| 1356 |
delete item._sortindex; |
| 1357 |
if (op.args.length > 0 && op.args[0].label) item[op.args[0].label] = i + 1; |
| 1358 |
}); |
| 1359 |
break; |
| 1360 |
case '/': // group |
| 1361 |
case '//': // merge |
| 1362 |
const groupindex = {}; |
| 1363 |
let keylists = {}; |
| 1364 |
let ofname = 'of'; |
| 1365 |
let countname = 'count'; |
| 1366 |
let sortgroups = true; |
| 1367 |
const groupargs = []; |
| 1368 |
const merge_ands = []; |
| 1369 |
for (const arg of op.args) { |
| 1370 |
if (arg.separator === ';' && arg.label == null && arg.value == null) { |
| 1371 |
sortgroups = false; |
| 1372 |
} else if (op.operator === '//' && (arg.separator === ';' || merge_ands.length > 0)) { |
| 1373 |
if (arg.separator === ';') { |
| 1374 |
merge_ands.push([arg.value]); |
| 1375 |
} else { |
| 1376 |
merge_ands[merge_ands.length - 1].push(arg.value) |
| 1377 |
} |
| 1378 |
} else { |
| 1379 |
groupargs.push(arg); |
| 1380 |
} |
| 1381 |
} |
| 1382 |
if (groupargs.length === 0) groupargs.push({value: null}); |
| 1383 |
|
| 1384 |
groupargs.forEach((arg) => arg.groupcounter = 0); |
| 1385 |
|
| 1386 |
const grouptimer = {}; |
| 1387 |
for (let ix=0; ix<currentlist.length; ix++) { |
| 1388 |
this.timecheck(grouptimer, ix, currentlist.length, op); |
| 1389 |
const item = currentlist[ix]; |
| 1390 |
let keys = null; |
| 1391 |
let keyi = 0; |
| 1392 |
for (const arg of groupargs) { |
| 1393 |
if (arg.label === 'of') { |
| 1394 |
ofname = arg.value; |
| 1395 |
continue; |
| 1396 |
} else if (arg.label === 'count') { |
| 1397 |
countname = arg.value; |
| 1398 |
continue; |
| 1399 |
} |
| 1400 |
keyi++; |
| 1401 |
let groupnumber = null; |
| 1402 |
if (op.operator === '/' && dtype(arg.value, 'number')) { |
| 1403 |
if (arg.subop?.endsWith('@')) { |
| 1404 |
arg.divisor = Number(arg.value); |
| 1405 |
} else { |
| 1406 |
arg.divisor = currentlist.length / Number(arg.value); |
| 1407 |
} |
| 1408 |
groupnumber = Math.floor(ix / arg.divisor) + 1; |
| 1409 |
} else if (arg.value != null && arg.subop?.endsWith('@@')) { |
| 1410 |
const groupval = step(item, arg.value, null, labeled); |
| 1411 |
if (ix === 0 || groupval?.length > 0) arg.groupcounter += 1; |
| 1412 |
groupnumber = arg.groupcounter; |
| 1413 |
} |
| 1414 |
const label = arg.label ?? (typeof arg.value === 'string' ? arg.value : null) ?? keyi; |
| 1415 |
const newkeyitems = groupnumber != null ? [groupnumber] : arg.value ? step(item, arg.value, arg.subop, labeled) : [null]; |
| 1416 |
const newkeys = newkeyitems.map((newkey) => ([{arglabel: arg.label, label: label, keyitem: this.resolve(arg.value, newkey, labeled)}])); |
| 1417 |
if (keys) { |
| 1418 |
keys = keys.flatMap((oldkeys) => newkeys.filter((newkey) => arg.separator === ',' || oldkeys.filter((oldkey) => compvals(getname(oldkey.keyitem) || oldkey.keyitem, getname(newkey[0].keyitem) || newkey[0].keyitem) >= 0).length === 0).map((newkey) => oldkeys.concat(newkey))); |
| 1419 |
} else { |
| 1420 |
keys = newkeys; |
| 1421 |
} |
| 1422 |
if (arg.subop?.endsWith('@')) { |
| 1423 |
keys.forEach((key) => { |
| 1424 |
const testkey = JSON.stringify(key.slice(0, -1)); |
| 1425 |
const newkeytest = JSON.stringify(key[key.length - 1]); |
| 1426 |
if (testkey in keylists) { |
| 1427 |
const lastkey = keylists[testkey][keylists[testkey].length - 1]; |
| 1428 |
if (newkeytest !== lastkey) keylists[testkey].push(newkeytest); |
| 1429 |
} else { |
| 1430 |
keylists[testkey] = [newkeytest]; |
| 1431 |
} |
| 1432 |
key[key.length - 1].keyindex = keylists[testkey].length; |
| 1433 |
}); |
| 1434 |
} |
| 1435 |
} |
| 1436 |
if (keys) { |
| 1437 |
for (const key of keys) { |
| 1438 |
const keystr = JSON.stringify(key); |
| 1439 |
(groupindex[keystr] ||= []).push(item); |
| 1440 |
} |
| 1441 |
} |
| 1442 |
} |
| 1443 |
for (const [keystr, items] of Object.entries(groupindex)) { |
| 1444 |
if (op.operator === '/') { |
| 1445 |
const newgroup = {}; |
| 1446 |
const keydata = JSON.parse(keystr); |
| 1447 |
if (keydata.length === 1 && keydata[0].keyitem != null) { |
| 1448 |
let keyitemname = keydata[0].keyindex || getname(keydata[0].keyitem); |
| 1449 |
if (keyitemname != null) { |
| 1450 |
newgroup.name = keyitemname; |
| 1451 |
} |
| 1452 |
} |
| 1453 |
let skip = false; |
| 1454 |
let keys = []; |
| 1455 |
for (const {arglabel, label, keyitem, keyindex} of keydata) { |
| 1456 |
if (keyitem != null) { |
| 1457 |
const keyobj = (dtype(keyitem, 'object') || keyindex == null || false) ? keyitem : {name: keyitem}; |
| 1458 |
if (keyindex != null) { |
| 1459 |
keyobj.index = keyindex; |
| 1460 |
} |
| 1461 |
keys.push(keyobj) |
| 1462 |
if (label && dtype(label, 'string') && isNaN(label) && label !== '_') { |
| 1463 |
newgroup[label] = [keyobj]; |
| 1464 |
} |
| 1465 |
} |
| 1466 |
} |
| 1467 |
newgroup[countname] = items.length; |
| 1468 |
if (keys) newgroup.key = keys; |
| 1469 |
newgroup[ofname] = items; |
| 1470 |
outputlist.push(newgroup); |
| 1471 |
} else if (op.operator === '//') { |
| 1472 |
if (merge_ands.length === 0 || !merge_ands.find((mand) => !mand.find((mr) => items.find((item) => mr in item)))) { |
| 1473 |
const newgroup = items.reduce((acc, item) => { |
| 1474 |
for(const prop in item) { |
| 1475 |
if (prop in acc) { |
| 1476 |
if (Array.isArray(acc[prop])) { |
| 1477 |
(Array.isArray(item[prop]) ? item[prop] : [item[prop]]).filter((val) => !acc[prop].includes(val)).forEach((val) => acc[prop].push(val)); |
| 1478 |
} |
| 1479 |
} else { |
| 1480 |
acc[prop] = item[prop]; |
| 1481 |
} |
| 1482 |
} |
| 1483 |
return acc; |
| 1484 |
}, {}); |
| 1485 |
const keydata = JSON.parse(keystr); |
| 1486 |
for (const {arglabel, label, keyitem, keyindex} of keydata) { |
| 1487 |
if (arglabel && typeof arglabel === 'string' && arglabel !== '_') { |
| 1488 |
newgroup[arglabel] = [keyitem]; |
| 1489 |
} |
| 1490 |
} |
| 1491 |
outputlist.push(newgroup); |
| 1492 |
} |
| 1493 |
} |
| 1494 |
} |
| 1495 |
if (sortgroups) { |
| 1496 |
const sortquery = '#' + groupargs.map((arg, argx) => (arg.subop?.endsWith('@') || arg.divisor ? '+' : '') + '(..key:@' + (argx + 1) + (arg.subop?.endsWith('@') ? '.index;_' : '') + ')').join(','); |
| 1497 |
outputlist = this.execute(outputlist, this.assemble(this.tokenize(sortquery)), labeled); |
| 1498 |
} |
| 1499 |
break; |
| 1500 |
case '...': // aggregate |
| 1501 |
case '....': // aggregate to value |
| 1502 |
if (!(op?.args?.length > 0)) { |
| 1503 |
if (op.operator === '...') { |
| 1504 |
outputlist = [{of: currentlist.map((item) => dcopy(item))}]; |
| 1505 |
} else { |
| 1506 |
outputlist = [currentlist.length] |
| 1507 |
} |
| 1508 |
break; |
| 1509 |
} |
| 1510 |
const tempitem = {}; |
| 1511 |
if (op.operator === '...') { |
| 1512 |
tempitem.of = currentlist.map((item) => dcopy(item)); |
| 1513 |
} else { |
| 1514 |
tempitem.of = currentlist; |
| 1515 |
} |
| 1516 |
const finalitem = {}; |
| 1517 |
let aggregated = null; |
| 1518 |
let afteraggregated = 0; |
| 1519 |
let finalvalue = null; |
| 1520 |
op.args.forEach((arg, propx) => { |
| 1521 |
if (dtype(arg.value, 'string') && arg.subop === '~') { |
| 1522 |
tempitem[arg.label || '_' + (propx + 1).toString()] = [arg.value]; |
| 1523 |
} else { |
| 1524 |
const prop = arg.label ?? (typeof arg.value === 'string' ? arg.value : null) ?? '_' + (propx + 1).toString(); |
| 1525 |
const aggname = (arg.label == null && typeof arg.value == 'string' && arg.value) || (arg.value == null && arg.label); |
| 1526 |
const firstitem = currentlist[0]; |
| 1527 |
const isprop = typeof firstitem == 'object' && prop in firstitem; |
| 1528 |
if ((arg.separator === ';' || !isprop) && aggname in this.aggregators) { |
| 1529 |
const aggval = this.aggregators[aggname](tempitem); |
| 1530 |
const agglabel = arg.label || aggname; |
| 1531 |
if (aggval != null) finalitem[agglabel] = aggval; |
| 1532 |
aggregated = agglabel; |
| 1533 |
finalvalue = dcopy(aggval); |
| 1534 |
} else if (!aggregated) { |
| 1535 |
let propitems; |
| 1536 |
if (op === operations[0] && op.operator === '....' && arg.label && !Array.isArray(arg.value) && !arg.value.startsWith('=')) { |
| 1537 |
propitems = [arg.value] |
| 1538 |
} else { |
| 1539 |
propitems = this.execute(currentlist, Array.isArray(arg.value) ? arg.value.map((x) => structuredClone(x)) : [{operator: '..', args: [{value: arg.value}]}], labeled) |
| 1540 |
} |
| 1541 |
tempitem[prop] = propitems; |
| 1542 |
} else if (aggregated in finalitem && Array.isArray(finalitem[aggregated]) && finalitem[aggregated].length > afteraggregated) { |
| 1543 |
tempitem[prop] = finalitem[aggregated][afteraggregated]; |
| 1544 |
afteraggregated++; |
| 1545 |
} |
| 1546 |
} |
| 1547 |
}); |
| 1548 |
if (op.operator === '....' && finalvalue != null) { |
| 1549 |
if (this.dtype(finalvalue, 'array')) { |
| 1550 |
outputlist = finalvalue; |
| 1551 |
} else { |
| 1552 |
outputlist = [finalvalue]; |
| 1553 |
} |
| 1554 |
} else { |
| 1555 |
for (const prop in tempitem) { |
| 1556 |
if (!(prop in finalitem) && prop !== 'of') finalitem[prop] = tempitem[prop]; |
| 1557 |
} |
| 1558 |
if (op.operator === '...') { |
| 1559 |
finalitem.of = tempitem.of; |
| 1560 |
} else { |
| 1561 |
for (const prop in finalitem) { |
| 1562 |
if (Array.isArray(finalitem[prop]) && finalitem[prop].length === 1) finalitem[prop] = finalitem[prop][0]; |
| 1563 |
} |
| 1564 |
} |
| 1565 |
outputlist = [finalitem]; |
| 1566 |
} |
| 1567 |
break; |
| 1568 |
case '|': // annotate |
| 1569 |
case '||': // annotate with values |
| 1570 |
outputlist = currentlist.slice(0).map((item) => dcopy(item)); |
| 1571 |
op.args.filter((arg) => arg.subop?.includes('>') && arg.label != null && true && Array.isArray(arg.value)) |
| 1572 |
.forEach((arg) => (labeled['=>'] ??= {})[arg.label] = arg.value); |
| 1573 |
|
| 1574 |
const annotatetimer = {}; |
| 1575 |
for (const i in outputlist) { |
| 1576 |
this.timecheck(annotatetimer, i, outputlist.length, op); |
| 1577 |
const baseitem = outputlist[i]; |
| 1578 |
if (typeof baseitem != 'object') { |
| 1579 |
outputlist[i] = {}; |
| 1580 |
if (op.args?.[0]?.value !== '_') outputlist[i].name = baseitem; |
| 1581 |
} |
| 1582 |
const item = outputlist[i]; |
| 1583 |
const newprops = op.args.filter((arg) => arg.subop !== '<' && (arg.label || !arg.subop?.includes('-'))).map((arg) => arg.label || arg.value); |
| 1584 |
if (typeof item == 'object') { |
| 1585 |
let argx = -1; |
| 1586 |
for (const arg of op.args) { |
| 1587 |
argx++; |
| 1588 |
if (arg?.subop === '<' && dtype(arg.value, 'string') && Array.isArray(item[arg.value]) && argx < op.args.length - 1) { |
| 1589 |
const sublabeled = {...labeled}; |
| 1590 |
if (arg.label) sublabeled[arg.label] = [item]; |
| 1591 |
item[arg.value] = dactal.execute(item[arg.value], [{operator: op.operator, args: op.args.slice(argx + 1).map((arg) => structuredClone(arg))}], sublabeled); |
| 1592 |
break; |
| 1593 |
} else if (arg.separator === ';' && argx === op.args.length - 1 && arg.label == null && arg.value == null && arg.subop == null) { |
| 1594 |
for (const oldprop in item) { |
| 1595 |
if (!(newprops.includes(oldprop))) { |
| 1596 |
const tempval = item[oldprop]; |
| 1597 |
delete item[oldprop]; |
| 1598 |
item[oldprop] = tempval; |
| 1599 |
} |
| 1600 |
} |
| 1601 |
} else if (arg.subop?.endsWith('-')) { |
| 1602 |
if (arg.value && typeof arg.value == 'string' && arg.value in item) { |
| 1603 |
if (arg.label && typeof arg.label == 'string') item[arg.label] = item[arg.value]; |
| 1604 |
delete item[arg.value]; |
| 1605 |
} else if (arg.label && Array.isArray(arg.value)) { |
| 1606 |
const toremoveids = new Set(step(item, arg.value, null, labeled).map((subitem) => getid(subitem))); |
| 1607 |
item[arg.label] = item[arg.label].filter((subitem) => !toremoveids.has(getid(subitem))); |
| 1608 |
} |
| 1609 |
} else if (argx === 0 && arg.label != null && arg.value === '_') { |
| 1610 |
item[arg.label] = op.operator === '||' ? baseitem : [baseitem]; |
| 1611 |
} else { |
| 1612 |
if (!arg.label && arg.value && typeof arg.value === 'string' && arg.value !== '') { |
| 1613 |
const moveprop = arg.value; |
| 1614 |
const value = structuredClone(item[moveprop]); |
| 1615 |
delete item[moveprop]; |
| 1616 |
item[moveprop] = value; |
| 1617 |
} |
| 1618 |
if (arg.label) { |
| 1619 |
let vals = []; |
| 1620 |
if (arg.subop.endsWith('@')) { |
| 1621 |
if (arg.subop.endsWith('@@')) { |
| 1622 |
arg.counter ??= arg.value == null ? outputlist.length + 1 : 0; |
| 1623 |
} else { |
| 1624 |
arg.counter ??= 1; |
| 1625 |
vals = [arg.counter]; |
| 1626 |
} |
| 1627 |
if (Array.isArray(arg.value)) { |
| 1628 |
arg.counter += step(item, arg.value, null, labeled).length; |
| 1629 |
} else if (dtype(arg.value, 'string') && dtype(item[arg.value], 'number')) { |
| 1630 |
arg.counter += Number(item[arg.value]); |
| 1631 |
} else { |
| 1632 |
arg.counter += arg.subop.endsWith('@@') ? -1 : 1; |
| 1633 |
} |
| 1634 |
if (arg.subop.endsWith('@@')) { |
| 1635 |
vals = [arg.counter]; |
| 1636 |
} |
| 1637 |
} else { |
| 1638 |
vals = step(item, arg.value, arg.subop, labeled); |
| 1639 |
} |
| 1640 |
const base = (arg.subop?.endsWith('+') && arg.label in item && Array.isArray(item[arg.label])) ? item[arg.label] : []; |
| 1641 |
if (this.dtype(vals, 'array')) { |
| 1642 |
if (op.operator === '|') { |
| 1643 |
item[arg.label] = base.concat(vals.map((v) => dcopy(v))); |
| 1644 |
} else if (vals.length > 0 ) { |
| 1645 |
if (typeof vals[0] === 'object') { |
| 1646 |
item[arg.label] = getname(vals[0]) != null ? getname(vals[0]) : vals[0].id != null ? vals[0].id : dcopy(vals[0]); |
| 1647 |
} else { |
| 1648 |
item[arg.label] = vals[0]; |
| 1649 |
} |
| 1650 |
} |
| 1651 |
} else if (vals) { |
| 1652 |
item[arg.label] = base.concat(vals); |
| 1653 |
} |
| 1654 |
} |
| 1655 |
} |
| 1656 |
} |
| 1657 |
} |
| 1658 |
} |
| 1659 |
break; |
| 1660 |
case '???': |
| 1661 |
outputlist = currentlist; |
| 1662 |
const commentval = op.args?.[0]?.value; |
| 1663 |
if (commentval in labeled) console.log({[commentval]: labeled[commentval]}); |
| 1664 |
if (commentval === 'end') return (this.debug && !inputlist) ? operations : outputlist; |
| 1665 |
break; |
| 1666 |
default: |
| 1667 |
outputlist = [] |
| 1668 |
} |
| 1669 |
currentlist = outputlist.slice(0); |
| 1670 |
if (this.debug && !inputlist) op.results = outputlist.slice(0, typeof this.debug == 'number' ? this.debug : outputlist.length); |
| 1671 |
} |
| 1672 |
return (this.debug && !inputlist) ? operations : outputlist; |
| 1673 |
} |
| 1674 |
|
| 1675 |
gettype(value, allow_relative=true) { |
| 1676 |
if (value == null) return null; |
| 1677 |
const trylist = [value]; |
| 1678 |
if (value.endsWith('s')) { |
| 1679 |
trylist.push(value.slice(0, value.length - 1)); |
| 1680 |
trylist.push(value + 'es'); |
| 1681 |
} else { |
| 1682 |
trylist.push(value + 's'); |
| 1683 |
} |
| 1684 |
for (const tryval of trylist) { |
| 1685 |
if (tryval in this.data) { |
| 1686 |
return this.data[tryval]; |
| 1687 |
} |
| 1688 |
} |
| 1689 |
if (this.savedquerynames.has(value)) { |
| 1690 |
const savedqueries = this.data.queries.filter((q) => q.name === value); |
| 1691 |
if (savedqueries?.length === 1) { |
| 1692 |
const sq = savedqueries[0]; |
| 1693 |
if (allow_relative && sq.query.startsWith('?')) { |
| 1694 |
return sq; |
| 1695 |
} else { |
| 1696 |
if (!sq.results) sq.results = this.executeq(sq.query); |
| 1697 |
return sq.results.slice(0); |
| 1698 |
} |
| 1699 |
} |
| 1700 |
} |
| 1701 |
return null; |
| 1702 |
} |
| 1703 |
|
| 1704 |
dcopy = (item) => { |
| 1705 |
if (this.dtype(item, 'literal')) return item; |
| 1706 |
if (this.dtype(item, 'array')) return item.slice(0); |
| 1707 |
return Object.assign({}, item); |
| 1708 |
} |
| 1709 |
|
| 1710 |
dtype(item, test=null) { |
| 1711 |
let type = null; |
| 1712 |
if (item && Array.isArray(item)) { |
| 1713 |
type = 'array'; |
| 1714 |
} else if (item && typeof item === 'object') { |
| 1715 |
type = 'object'; |
| 1716 |
} else if (typeof item === 'string' || typeof item === 'number' || typeof item === 'boolean') { |
| 1717 |
type = 'literal'; |
| 1718 |
} |
| 1719 |
if (test) { |
| 1720 |
if (test === 'number') { |
| 1721 |
return type === 'literal' && !isNaN(item); |
| 1722 |
} else if (test === 'string') { |
| 1723 |
return type === 'literal' && typeof item === 'string'; |
| 1724 |
} else if (test === 'boolean') { |
| 1725 |
return type === 'literal' && typeof item === 'boolean'; |
| 1726 |
} |
| 1727 |
return type === test; |
| 1728 |
} |
| 1729 |
return type; |
| 1730 |
} |
| 1731 |
|
| 1732 |
getname(item) { |
| 1733 |
if (typeof item === 'object' && item != null) { |
| 1734 |
if (item?.name != null) { |
| 1735 |
return item.name; |
| 1736 |
} else if (item?.key?.length > 0) { |
| 1737 |
return item.key.join(' / '); |
| 1738 |
} |
| 1739 |
const {id, ...nonidprops} = item; |
| 1740 |
const names = Object.values(nonidprops).filter((val) => typeof val === 'string' || typeof val === 'number'); |
| 1741 |
if (names.length > 0) { |
| 1742 |
return names[0]; |
| 1743 |
} |
| 1744 |
for(const key in nonidprops) { |
| 1745 |
if (Array.isArray(item[key]) && item[key].length === 1 && typeof item[key][0] == 'string') { |
| 1746 |
return item[key][0]; |
| 1747 |
} |
| 1748 |
} |
| 1749 |
} else if (typeof item === 'string' || typeof item === 'number') { |
| 1750 |
return item; |
| 1751 |
} else if (typeof item === 'boolean') { |
| 1752 |
return item.toString(); |
| 1753 |
} |
| 1754 |
return ''; |
| 1755 |
} |
| 1756 |
|
| 1757 |
getid(item) { |
| 1758 |
if (typeof item === 'object' && item != null) { |
| 1759 |
const iditems = (item.id || item.uri || item.name) ? [item] : item.key?.length > 0 ? item.key : Object.keys(item).length === 3 && item.of?.length > 0 ? item.of : [item]; |
| 1760 |
return iditems.map((item) => { |
| 1761 |
if (item.id) { |
| 1762 |
if (Array.isArray(item.id)) { |
| 1763 |
return item.id[0]; |
| 1764 |
} else { |
| 1765 |
return item.id; |
| 1766 |
} |
| 1767 |
} else if (item.uri) { |
| 1768 |
if (Array.isArray(item.uri)) { |
| 1769 |
return item.uri[0]; |
| 1770 |
} else { |
| 1771 |
return item.uri; |
| 1772 |
} |
| 1773 |
} else if (item.name) { |
| 1774 |
return item.name; |
| 1775 |
} else { |
| 1776 |
// if (stringifiedid.length > 128) console.warn({idstringify: item, idlength: stringifiedid.length}); |
| 1777 |
return JSON.stringify(item); |
| 1778 |
} |
| 1779 |
}).join(','); |
| 1780 |
} else if (typeof item === 'string' || typeof item === 'number') { |
| 1781 |
return item; |
| 1782 |
} |
| 1783 |
return null; |
| 1784 |
} |
| 1785 |
|
| 1786 |
step = (item, property, subop, labeled) => { |
| 1787 |
// (this.data.trace ??= []).push({stepitem: item, property: property, subop: subop, labeled: labeled}); |
| 1788 |
const dtype = this.dtype; |
| 1789 |
const dcopy = this.dcopy; |
| 1790 |
const resolve = this.resolve; |
| 1791 |
const escapeRegExp = this.escapeRegExp; |
| 1792 |
const vals = []; |
| 1793 |
if (subop?.includes('~') && dtype(property, 'literal')) { |
| 1794 |
vals.push(property); |
| 1795 |
} else if (Array.isArray(property)) { |
| 1796 |
this.execute([item], property.map((x) => structuredClone(x)), labeled).forEach((val) => vals.push(val)); |
| 1797 |
} else if (property === '_') { |
| 1798 |
vals.push(dcopy(item)); |
| 1799 |
} else if (item?.toString()?.match(/^-?\d+$/) && property.match(/^\d+$/)) { |
| 1800 |
vals.push((item === 0 && 1 / item === -Infinity ? -1 : 1) * Number(`${item}.${property}`)); |
| 1801 |
} else if (item?.of && dtype(property, 'number')) { |
| 1802 |
const propval = Number(property); |
| 1803 |
(subop === '-' || propval < 0 ? item.of.slice(-1 * Math.abs(Number(property))) : item.of.slice(0, Math.abs(Number(property)))).forEach((val) => vals.push(val)); |
| 1804 |
} else if (property === 'id' && dtype(item, 'literal')) { |
| 1805 |
vals.push(item); |
| 1806 |
} else if (property === 'name') { |
| 1807 |
vals.push(this.getname(item)); |
| 1808 |
} else if ((dtype(item, 'number') || dtype(item, 'object')) && property.startsWith('=')) { |
| 1809 |
let calculation = property.slice(1); |
| 1810 |
const mathwords = Object.getOwnPropertyNames(Math).filter((mathword) => mathword.match(/^[a-z0-9]+$/)).sort((a, b) => b.length - a.length || a.localeCompare(b)); |
| 1811 |
const otherwords = ['split']; |
| 1812 |
const variables = Object.entries(item).concat(Object.entries(labeled)) |
| 1813 |
.map(([k, v]) => k) |
| 1814 |
.sort((a, b) => b.length - a.length || a.localeCompare(b)); |
| 1815 |
if (dtype(this.getname(item), 'number')) variables.push('_'); |
| 1816 |
const allowedwords = mathwords.concat(otherwords).concat(variables).sort((a, b) => b.length - a.length || a.localeCompare(b)); |
| 1817 |
const allowed = new RegExp(`^((\b(${allowedwords.map((w) => escapeRegExp(w)).join('|')})\b)|([0-9_\+\/\*\(\)\[\].%=,'" -]*))*$`); |
| 1818 |
let val; |
| 1819 |
if (calculation.match(allowed)) { |
| 1820 |
for (const variable of variables) { |
| 1821 |
const variableex = new RegExp(`\\b${variable}\\b`, 'g'); |
| 1822 |
if (calculation.match(variableex)) { |
| 1823 |
let vval = variable === '_' ? Number(this.getname(item)) : item[variable] ?? labeled[variable]; |
| 1824 |
if (Array.isArray(vval) && vval.length === 1) vval = vval[0]; |
| 1825 |
calculation = calculation.replaceAll(variableex, dtype(vval, 'number') ? vval : JSON.stringify(vval)); |
| 1826 |
} |
| 1827 |
} |
| 1828 |
if (calculation.match(/[A-Za-z]/)) { |
| 1829 |
for (const mathword of mathwords) { |
| 1830 |
const mathwordex = new RegExp(`\\b${mathword}\\b`, 'g'); |
| 1831 |
calculation = calculation.replaceAll(mathwordex, `Math.${mathword}`); |
| 1832 |
if (!calculation.match(/[A-Za-z]/)) break; |
| 1833 |
} |
| 1834 |
} |
| 1835 |
calculation = calculation.replaceAll(/\b=\b/g, '=='); |
| 1836 |
try { |
| 1837 |
val = eval(calculation); |
| 1838 |
} catch (error) { |
| 1839 |
val = calculation; |
| 1840 |
} |
| 1841 |
} else { |
| 1842 |
val = calculation; |
| 1843 |
} |
| 1844 |
if (val !== false) vals.push(val); |
| 1845 |
} else if (item) { |
| 1846 |
let found = false; |
| 1847 |
if (property != null && !dtype(item, 'literal')) { |
| 1848 |
const tryvals = [property]; |
| 1849 |
tryvals.push(property.endsWith('s') ? property.slice(0, -1) : property + 's'); |
| 1850 |
if (property.match(/ /)) tryvals.push(property.replaceAll(/ /g, '_')); |
| 1851 |
for (const tryval of tryvals) { |
| 1852 |
if (tryval in item) { |
| 1853 |
const resolved = resolve(tryval, item[tryval], labeled); |
| 1854 |
if (Array.isArray(resolved)) { |
| 1855 |
resolved.forEach((x) => vals.push(x)); |
| 1856 |
} else { |
| 1857 |
vals.push(resolved); |
| 1858 |
} |
| 1859 |
found = true; |
| 1860 |
break; |
| 1861 |
} |
| 1862 |
} |
| 1863 |
} |
| 1864 |
if (!found && labeled?.['=>']?.[property]) { |
| 1865 |
this.execute([item], labeled['=>'][property], labeled).forEach((val) => vals.push(val)); |
| 1866 |
found = true; |
| 1867 |
} |
| 1868 |
if (!found && (property in this.data || property in this.adapters || this.savedquerynames.has(property) || property in labeled)) { |
| 1869 |
let typenav; |
| 1870 |
if (!dtype(item, 'literal')) { |
| 1871 |
if (this.adapters?.[property]?.annotator) { |
| 1872 |
typenav = resolve(property, item, labeled); |
| 1873 |
|
| 1874 |
|
| 1875 |
} else if ('id' in item || 'uri' in item) { |
| 1876 |
typenav = resolve(property, this.getid(item), labeled); |
| 1877 |
} else if (this.savedquerynames.has(property)) { |
| 1878 |
const relative_query = dactal.data.queries.find((q) => q.relative && q.name === property); |
| 1879 |
if (relative_query) { |
| 1880 |
typenav = this.step(item, this.parse(relative_query.query), subop, labeled); |
| 1881 |
} else { |
| 1882 |
typenav = this.getname(item); |
| 1883 |
} |
| 1884 |
} else { |
| 1885 |
typenav = this.getname(item); |
| 1886 |
} |
| 1887 |
if (typenav) vals.push(typenav); |
| 1888 |
} else { |
| 1889 |
typenav = resolve(property, item, labeled); |
| 1890 |
if (typenav) { |
| 1891 |
if (Array.isArray(typenav)) { |
| 1892 |
typenav.forEach((t) => vals.push(t)); |
| 1893 |
} else { |
| 1894 |
vals.push(typenav); |
| 1895 |
} |
| 1896 |
} |
| 1897 |
} |
| 1898 |
} |
| 1899 |
} |
| 1900 |
return vals; |
| 1901 |
} |
| 1902 |
|
| 1903 |
resolve = (property, item, labeled) => { |
| 1904 |
const dtype = this.dtype; |
| 1905 |
if (dtype(item, 'object')) { |
| 1906 |
if (this.adapters?.[property]?.annotator) { |
| 1907 |
if ((!this.recache || this.recache.has(property)) && property in this.index && this.getid(item) in this.index[property]) { |
| 1908 |
return this.index[property][this.getid(item)]; |
| 1909 |
} else { |
| 1910 |
this.adapters[property].queue.push(this.adapters[property].annotator ? item : this.getid(item)); |
| 1911 |
return null; |
| 1912 |
} |
| 1913 |
} else { |
| 1914 |
return item; |
| 1915 |
} |
| 1916 |
} else if (dtype(item, 'literal') && typeof property == 'string' && (property in labeled || this.destinations.has(property) || this.destinations.has(property + 's'))) { |
| 1917 |
if ((!this.recache || this.recache.has(property)) && property in this.index && item in this.index[property]) { |
| 1918 |
|
| 1919 |
return this.index[property][item]; |
| 1920 |
} |
| 1921 |
const typeitems = labeled[property] ?? this.gettype(property); |
| 1922 |
if (Array.isArray(typeitems)) { |
| 1923 |
for (const lookupkey of ['id', 'uri', 'name', property, property + 's']) { |
| 1924 |
const found = typeitems.filter((typeitem) => { |
| 1925 |
return dtype(typeitem, 'object') && (lookupkey in typeitem) && (typeitem[lookupkey] === item || (Array.isArray(typeitem[lookupkey]) && typeitem[lookupkey].length === 1 && typeitem[lookupkey][0] === item)); |
| 1926 |
}); |
| 1927 |
if (found.length === 1) { |
| 1928 |
(this.index[property] ||= {})[item] = found[0]; |
| 1929 |
return found[0]; |
| 1930 |
} |
| 1931 |
} |
| 1932 |
if (property in this.adapters && (!this.recache || !this.recache.has(property))) { |
| 1933 |
this.adapters[property].queue.push(this.adapters[property].annotator ? item : this.getid(item)); |
| 1934 |
} |
| 1935 |
return null; |
| 1936 |
} else if (typeitems && typeof typeitems == 'object' && 'query' in typeitems) { |
| 1937 |
const relative_query = this.assemble(this.tokenize(typeitems.query)); |
| 1938 |
while (['?', ':'].includes(relative_query[0].operator)) { |
| 1939 |
relative_query.shift() |
| 1940 |
} |
| 1941 |
return this.execute([item], relative_query, labeled); |
| 1942 |
} |
| 1943 |
if (property in this.adapters) { |
| 1944 |
this.adapters[property].queue.push(this.adapters[property].annotator ? item : this.getid(item)); |
| 1945 |
return null; |
| 1946 |
} |
| 1947 |
} |
| 1948 |
return item; |
| 1949 |
} |
| 1950 |
|
| 1951 |
samearray(a, b) { |
| 1952 |
if ((a && !b) || (!a && b) || a.length !== b.length) return false; |
| 1953 |
for(let x=0; x<a.length; x++) { |
| 1954 |
if (this.getid(a[x]) !== this.getid(b[x])) return false; |
| 1955 |
} |
| 1956 |
return true; |
| 1957 |
} |
| 1958 |
|
| 1959 |
async verify(i = null) { |
| 1960 |
let toverify = this.data.queries; |
| 1961 |
if (!isNaN(i)) toverify = toverify.slice(i - 1, i); |
| 1962 |
let allsame = true; |
| 1963 |
for (const savedq of toverify) { |
| 1964 |
console.log('verifying ' + savedq.name); |
| 1965 |
const testres = await this.query(savedq.query); |
| 1966 |
if (testres.length !== savedq.results.length) { |
| 1967 |
console.log('--x result count changed from ' + savedq.results.length + ' to ' + testres.length); |
| 1968 |
allsame = false; |
| 1969 |
} else { |
| 1970 |
for (let i=0; i<testres.length; i++) { |
| 1971 |
const testrow = testres[i]; |
| 1972 |
const savedrow = savedq.results[i]; |
| 1973 |
if (typeof savedrow == 'object') { |
| 1974 |
for (const prop in savedrow) { |
| 1975 |
if (!(prop in testrow)) { |
| 1976 |
console.log('--x row ' + (i+1) + ': new results missing property ' + prop); |
| 1977 |
allsame = false; |
| 1978 |
} else { |
| 1979 |
const testval = JSON.stringify(testrow[prop]); |
| 1980 |
const savedval = JSON.stringify(savedrow[prop]); |
| 1981 |
if (testval !== savedval) { |
| 1982 |
console.log('--x row ' + (i+1) + ': different value for property ' + prop); |
| 1983 |
console.log({was: savedrow[prop], now: testrow[prop]}) |
| 1984 |
allsame = false; |
| 1985 |
} |
| 1986 |
} |
| 1987 |
} |
| 1988 |
} else { |
| 1989 |
if (testrow !== savedrow) { |
| 1990 |
console.log('--x row ' + (i+1) + ': different value'); |
| 1991 |
console.log({was: savedrow, now: testrow}); |
| 1992 |
allsame = false; |
| 1993 |
} |
| 1994 |
} |
| 1995 |
} |
| 1996 |
} |
| 1997 |
if (allsame) console.log('--- results unchanged'); |
| 1998 |
} |
| 1999 |
} |
| 2000 |
|
| 2001 |
index_check() { |
| 2002 |
console.table(Object.entries(this.index).map(([key, vals]) => ({key: key, vals: Object.keys(vals).length, size: Object.keys(vals).length * JSON.stringify(Object.entries(vals).slice(0, 1)).length})).sort((a, b) => b.size - a.size || b.vals - a.vals || a.key.localeCompare(b.key))) |
| 2003 |
} |
| 2004 |
|
| 2005 |
index_materialize() { |
| 2006 |
this.load(Object.keys(this.index).flatMap((i) => Object.keys(this.index[i]).flatMap((k) => ({index: i, indexed: k, value: this.index[i][k]}))), 'index contents') |
| 2007 |
} |
| 2008 |
} |