| 1 |
export class DACTAL { |
| 2 |
constructor(data = {}) { |
| 3 |
this.data = data; |
| 4 |
this.index = {}; |
| 5 |
this.index_modified = new Set; |
| 6 |
this.adapters = {}; |
| 7 |
this.adaptive = false; |
| 8 |
this.features = { |
| 9 |
autoresolve: true, |
| 10 |
plurality: true, |
| 11 |
inlinemath: true, |
| 12 |
unscore: true, |
| 13 |
guessid: true, |
| 14 |
guessname: true |
| 15 |
} |
| 16 |
this.data['query history'] = []; |
| 17 |
this.savedquerynames = new Set(); |
| 18 |
this.debug = false; |
| 19 |
this.recache = false; |
| 20 |
this.timelimit = 120000; |
| 21 |
this.statusf = (statusmsg) => { |
| 22 |
if (statusmsg) console.log(statusmsg) |
| 23 |
}; |
| 24 |
this.internal_datasets = ['queries', 'query history', 'connectors', 'adapters', 'annotators', 'current results', 'data updates', 'assistance', '.clf API routes']; |
| 25 |
|
| 26 |
this.annotators = { |
| 27 |
group: (item) => item.of, |
| 28 |
label: (item) => item.of, |
| 29 |
as: (item) => item.of, |
| 30 |
count: (item) => this.vals(item).length, |
| 31 |
total: (item) => this.numvals(item).reduce((acc, val) => acc + val, 0), |
| 32 |
average: (item) => this.numvals(item).reduce((acc, val) => acc + val, 0) / this.numvals(item).length, |
| 33 |
median: (item) => { |
| 34 |
const nums = this.numvals(item).sort(); |
| 35 |
return nums[Math.floor(nums.length / 2)]; |
| 36 |
}, |
| 37 |
min: (item) => { |
| 38 |
const nums = this.numvals(item); |
| 39 |
return nums.length == 0 ? [] : Math.min(...nums) |
| 40 |
}, |
| 41 |
max: (item) => { |
| 42 |
const nums = this.numvals(item); |
| 43 |
return nums.length == 0 ? [] : Math.max(...nums) |
| 44 |
}, |
| 45 |
product: (item) => this.numvals(item).reduce((acc, val) => acc * val, 1), |
| 46 |
difference: (item) => this.numvals(item).reduce((acc, val) => acc - val), |
| 47 |
quotient: (item) => this.numvals(item).reduce((acc, val) => acc / val), |
| 48 |
percent: (item) => Math.round(this.numvals(item).reduce((acc, val) => acc / val) * 100), |
| 49 |
sqrt: (item) => Math.sqrt(this.numvals(item)[0]), |
| 50 |
log: (item) => Math.log(this.numvals(item)[0]), |
| 51 |
log10: (item) => Math.log10(this.numvals(item)[0]), |
| 52 |
abs: (item) => Math.abs(this.numvals(item)[0]), |
| 53 |
is: (item) => (item.of?.length > 0) ? 1 : 0, |
| 54 |
isnt: (item) => (item.of?.length === 0) ? 1 : 0, |
| 55 |
yesno: (item) => (item.of?.length > 0) ? 'yes' : 'no', |
| 56 |
missing: (item) => (item.of?.length === 0) ? true : [], |
| 57 |
otherwise: (item) => { |
| 58 |
const ikeys = this.kkeys(item); |
| 59 |
if (item.of?.length > 0 || ikeys.length == 0) { |
| 60 |
return item.of; |
| 61 |
} else { |
| 62 |
return item[ikeys[0]]; |
| 63 |
} |
| 64 |
}, |
| 65 |
sortsame: (item) => { |
| 66 |
const sortforms = this.vals(item).map((v) => v?.toString()?.toLowerCase()?.replace(/^the /, '')); |
| 67 |
const sortformset = new Set(sortforms); |
| 68 |
return (sortformset.size == 1 ? 'sortsame' : []); |
| 69 |
}, |
| 70 |
startsame: (item) => { |
| 71 |
const vals = this.vals(item); |
| 72 |
const shortest = vals.sort((a, b) => a.length - b.length)[0]; |
| 73 |
return (vals.filter((v) => v.startsWith(shortest)).length == vals.length ? shortest : []); |
| 74 |
}, |
| 75 |
concatenate: (item) => this.vals(item).join(' '), |
| 76 |
join: (item) => item.of.map(this.getname).join(this.vals(item)[0]), |
| 77 |
str: (item) => item.of.map(this.getname).join(''), |
| 78 |
'to json': (item) => JSON.stringify(item.of), |
| 79 |
quote: (item) => `“${this.vals(item)[0]}â€`, |
| 80 |
url: (item) => { |
| 81 |
let u = item.of.map(this.getname).join(''); |
| 82 |
if (!u.startsWith('https://')) u = 'https://' + u; |
| 83 |
for (const prop of this.kkeys(item)) { |
| 84 |
const val = item[prop]; |
| 85 |
(Array.isArray(val) ? val : [val]).forEach((vv) => u = u + (u.match(/\?/) ? '&' : '?') + encodeURIComponent(prop) + '=' + encodeURIComponent(vv)); |
| 86 |
} |
| 87 |
return u; |
| 88 |
}, |
| 89 |
remove: (item) => this.vals(item).reduce((acc, val) => acc.replaceAll(val, '')), |
| 90 |
replace: (item) => { |
| 91 |
return item.of.map((i) => { |
| 92 |
let newi = this.getname(i); |
| 93 |
this.kkeys(item).map((k) => { |
| 94 |
newi = newi.replaceAll(k, item[k].toString()); |
| 95 |
}) |
| 96 |
return newi; |
| 97 |
}); |
| 98 |
}, |
| 99 |
matches: (item) => { |
| 100 |
const matchers = item.match.map((m) => m.toLowerCase()); |
| 101 |
return matchers.filter((m) => item.text.find((i) => this.getname(i).toLowerCase().includes(m))); |
| 102 |
}, |
| 103 |
bmk: (item) => this.vals(item).map((v) => v.toString().toLowerCase().replaceAll(/[\$\xA2-\xA5\u058F\u060B\u09F2\u09F3\u09FB\u0AF1\u0BF9\u0E3F\u17DB\u20A0-\u20BD\uA838\uFDFC\uFE69\uFF04\uFFE0\uFFE1\uFFE5\uFFE6,+]/g, '').replaceAll('b', 'kkk').replaceAll('m', 'kk').replaceAll('k', '000')), |
| 104 |
sortform: (item) => this.vals(item).map((val) => val?.toString()?.toLowerCase()?.replace(/^the /, '')), |
| 105 |
zip: (item) => { |
| 106 |
const itemkeys = this.kkeys(item); |
| 107 |
const zipped = []; |
| 108 |
for (let i = 0; i < item[itemkeys[0]].length; i++) { |
| 109 |
const zipline = {}; |
| 110 |
for (const key of itemkeys) { |
| 111 |
zipline[key] = item[key][i]; |
| 112 |
} |
| 113 |
zipped.push(zipline); |
| 114 |
} |
| 115 |
return zipped; |
| 116 |
}, |
| 117 |
pairs: (item) => { |
| 118 |
return item.of.slice(0, -1).map((val, vx) => ({pair: [val, item.of[vx + 1]]})); |
| 119 |
}, |
| 120 |
triples: (item) => { |
| 121 |
return item.of.slice(0, -2).map((val, vx) => ({triple: [val, item.of[vx + 1], item.of[vx + 2]]})); |
| 122 |
}, |
| 123 |
quads: (item) => { |
| 124 |
return item.of.slice(0, -3).map((val, vx) => ({quad: [val, item.of[vx + 1], item.of[vx + 2], item.of[vx + 3]]})); |
| 125 |
}, |
| 126 |
sequences: (item) => { |
| 127 |
return item.of.map((val, valx, vallist) => ({sequence: vallist.slice(0, valx + 1).map((val) => this.dcopy(val))})); |
| 128 |
}, |
| 129 |
split: (item) => { |
| 130 |
const itemvals = this.vals(item); |
| 131 |
let splitter; |
| 132 |
let tobesplit; |
| 133 |
if (Object.keys(item).length == 1) itemvals.push(' '); |
| 134 |
if (itemvals.length == 1) { |
| 135 |
splitter = itemvals[0]; |
| 136 |
tobesplit = item.of.slice(0); |
| 137 |
} else { |
| 138 |
splitter = itemvals.pop(); |
| 139 |
tobesplit = itemvals.slice(0); |
| 140 |
} |
| 141 |
if (splitter.startsWith('~')) splitter = new RegExp(splitter.replace(/^~*/, ''), splitter.startsWith('~~') ? 'i' : ''); |
| 142 |
return tobesplit.flatMap((v) => v.toString().split(splitter)); |
| 143 |
}, |
| 144 |
unpack: (item) => { |
| 145 |
return item.of.flatMap((val) => { |
| 146 |
let valstr = val.toString(); |
| 147 |
return this.vals(item).reduce((acc, size, x) => { |
| 148 |
acc.push(valstr.substring(0, size)); |
| 149 |
valstr = valstr.substring(size); |
| 150 |
return acc; |
| 151 |
}, []); |
| 152 |
}); |
| 153 |
}, |
| 154 |
extract: (item) => { |
| 155 |
const itemvals = this.vals(item); |
| 156 |
const delimiters = itemvals.pop(); |
| 157 |
const res = []; |
| 158 |
for (const itemval of (itemvals.length > 0 ? itemvals : item.of)) { |
| 159 |
if (delimiters.startsWith('~')) { |
| 160 |
itemval.matchAll(new RegExp(delimiters.slice(1), 'g')).forEach((match) => { |
| 161 |
if (match.groups) { |
| 162 |
res.push(match.groups); |
| 163 |
} else { |
| 164 |
match.slice(1).forEach((m) => res.push(m)); |
| 165 |
} |
| 166 |
}); |
| 167 |
} else { |
| 168 |
for (let i = 0; i < delimiters.length; i += 2) { |
| 169 |
const d1 = delimiters[i]; |
| 170 |
const d2 = delimiters[i + 1]; |
| 171 |
const d1x = itemval.indexOf(d1); |
| 172 |
const d2x = itemval.indexOf(d2); |
| 173 |
if (d1x > -1 && d2x > d1x) { |
| 174 |
res.push(itemval.slice(d1x + 1, d2x).trim()); |
| 175 |
break; |
| 176 |
} |
| 177 |
} |
| 178 |
} |
| 179 |
} |
| 180 |
return res; |
| 181 |
}, |
| 182 |
unchain: (item) => { |
| 183 |
const chainprops = this.kkeys(item); |
| 184 |
const unchained = []; |
| 185 |
const queue = item.of.slice(0); |
| 186 |
while (queue.length > 0) { |
| 187 |
const thisitem = queue.shift(); |
| 188 |
if (!unchained.includes(thisitem)) { |
| 189 |
unchained.push(thisitem); |
| 190 |
for (const chainprop of chainprops) { |
| 191 |
if (this.dtype(thisitem, 'object') && chainprop in thisitem) { |
| 192 |
if (this.dtype(thisitem[chainprop], 'array')) { |
| 193 |
for (const x of thisitem[chainprop].slice(0).reverse()) queue.unshift(x); |
| 194 |
} else if (thisitem[chainprop]) { |
| 195 |
queue.unshift(thisitem[chainprop]) |
| 196 |
} |
| 197 |
} |
| 198 |
} |
| 199 |
} |
| 200 |
} |
| 201 |
return unchained; |
| 202 |
}, |
| 203 |
itemize: (item) => { |
| 204 |
const propname = item?.property ?? 'property'; |
| 205 |
const valname = item?.value ?? 'value'; |
| 206 |
return item.of.flatMap((subitem) => Object.entries(subitem).map(([key, val]) => ({ |
| 207 |
[propname]: key, |
| 208 |
[valname]: val |
| 209 |
}))); |
| 210 |
}, |
| 211 |
schematize: (item) => { |
| 212 |
const itemkeys = this.kkeys(item); |
| 213 |
const schematized = {}; |
| 214 |
item[itemkeys[0]].forEach((subitem) => { |
| 215 |
let subkey; |
| 216 |
let subval; |
| 217 |
if (this.dtype(subitem, 'array')) { |
| 218 |
const [subkey, subval] = subitem; |
| 219 |
} else { |
| 220 |
[subkey, subval] = Object.values(subitem); |
| 221 |
} |
| 222 |
schematized[subkey] = subval; |
| 223 |
}); |
| 224 |
return schematized; |
| 225 |
}, |
| 226 |
index: (item) => { |
| 227 |
return Object.entries(item).filter(([k, v]) => k != 'of').map(([k, v]) => ({ |
| 228 |
id: k, |
| 229 |
name: !isNaN(v) ? Number(v) : v |
| 230 |
})); |
| 231 |
}, |
| 232 |
unflatten: (item) => { |
| 233 |
const itemkeys = this.kkeys(item); |
| 234 |
if (itemkeys.length === 0) itemkeys.push(''); |
| 235 |
const newindex = {}; |
| 236 |
const neworder = []; |
| 237 |
item.of.forEach((subitem) => { |
| 238 |
Object.keys(subitem).forEach((field) => { |
| 239 |
itemkeys.forEach((key) => { |
| 240 |
if (field.startsWith(key)) { |
| 241 |
const subid = field.replace(key, ''); |
| 242 |
if (subid.length > 0) { |
| 243 |
if (!(subid in newindex)) { |
| 244 |
newindex[subid] = {}; |
| 245 |
neworder.push(subid); |
| 246 |
} |
| 247 |
newindex[subid][key] = subitem[field]; |
| 248 |
} |
| 249 |
} |
| 250 |
}); |
| 251 |
}); |
| 252 |
}); |
| 253 |
itemkeys.forEach((key) => delete item[key]); |
| 254 |
return neworder.map((k) => { |
| 255 |
const subitem = {}; |
| 256 |
subitem.subid = k; |
| 257 |
Object.assign(subitem, newindex[k]); |
| 258 |
return subitem; |
| 259 |
}); |
| 260 |
}, |
| 261 |
detupled: (item) => { |
| 262 |
const newobj = {}; |
| 263 |
item.of.forEach((subitem) => { |
| 264 |
if (Array.isArray(subitem) && subitem.length == 2) { |
| 265 |
newobj[subitem[0]] = subitem[1]; |
| 266 |
} |
| 267 |
}) |
| 268 |
return [newobj]; |
| 269 |
}, |
| 270 |
csv: (item) => { |
| 271 |
const keys = Object.entries(item).find(([k, v]) => k != 'of')[1]; |
| 272 |
const res = []; |
| 273 |
const vals = item.of.flatMap((val) => this.dtype(val, 'string') ? val.split('\n').map((val) => val.trim()) : val); |
| 274 |
for (let i = 0; i < vals.length; i += keys.length) { |
| 275 |
const newitem = {}; |
| 276 |
for (let k = 0; k < keys.length; k++) { |
| 277 |
newitem[keys[k]] = vals[i + k]; |
| 278 |
} |
| 279 |
res.push(newitem); |
| 280 |
} |
| 281 |
return res; |
| 282 |
}, |
| 283 |
tsv: (item) => { |
| 284 |
const text = this.getname(item); |
| 285 |
const lines = text.split('\n').filter((line) => line != '').map((line) => line.split('\t').map((val) => val.trim())); |
| 286 |
const keys = lines[0]; |
| 287 |
return lines.slice(1).map((vals) => Object.fromEntries(vals.map((val, vi) => [keys[vi], val]))); |
| 288 |
}, |
| 289 |
ssv: (item) => { |
| 290 |
const text = this.getname(item); |
| 291 |
const lines = text.split('\n').filter((line) => line != '').map((line) => line.split(/ +/).map((val) => val.trim())); |
| 292 |
const keys = lines[0]; |
| 293 |
return lines.slice(1).map((vals) => Object.fromEntries(vals.map((val, vi) => [keys[vi], val]))); |
| 294 |
}, |
| 295 |
json: (item) => this.vals(item).flatMap((v) => JSON.parse(v)), |
| 296 |
year: (item) => { |
| 297 |
if (item.date?.toString()?.length > 0) return item.date?.[0]?.toString()?.match(/\d\d\d\d/)?.[0]; |
| 298 |
let yearmatch = this.getname(item.toString()).match(/\d\d\d\d/); |
| 299 |
if (yearmatch) { |
| 300 |
return yearmatch[0]; |
| 301 |
} else { |
| 302 |
yearmatch = this.getid(item.of[0] || '').toString().match(/\d\d\d\d/); |
| 303 |
if (yearmatch) { |
| 304 |
return yearmatch[0]; |
| 305 |
} |
| 306 |
return null; |
| 307 |
} |
| 308 |
}, |
| 309 |
month: (item) => { |
| 310 |
if (item.date?.length > 0) return item.date[0].match(/\d\d\d\d-(\d\d)-\d\d/)[1]; |
| 311 |
let monthmatch = this.getname(item).match(/\d\d\d\d-(\d\d)-\d\d/); |
| 312 |
if (monthmatch) { |
| 313 |
return monthmatch[1]; |
| 314 |
} else { |
| 315 |
monthmatch = this.getid(item.of[0] || '').toString().match(/\d\d\d\d-(\d\d)-\d\d/); |
| 316 |
if (monthmatch) { |
| 317 |
return monthmatch[1]; |
| 318 |
} |
| 319 |
return null; |
| 320 |
} |
| 321 |
}, |
| 322 |
date: (item) => { |
| 323 |
let itemvals = this.vals(item); |
| 324 |
if (itemvals?.length > 0) { |
| 325 |
let datematch = itemvals[0].toString().match(/\d\d\d\d-\d\d-\d\d/); |
| 326 |
if (datematch) { |
| 327 |
return datematch[0]; |
| 328 |
} else { |
| 329 |
datematch = this.getid(item.of[0] || '').toString().match(/\d\d\d\d-\d\d-\d\d/); |
| 330 |
if (datematch) { |
| 331 |
return datematch[0]; |
| 332 |
} else { |
| 333 |
if (this.dtype(itemvals[0], 'number')) { |
| 334 |
datematch = itemvals[0].toString().slice(0, 4); |
| 335 |
if (datematch) { |
| 336 |
return datematch; |
| 337 |
} |
| 338 |
} |
| 339 |
} |
| 340 |
} |
| 341 |
} |
| 342 |
return null; |
| 343 |
}, |
| 344 |
weekday: (item) => { |
| 345 |
const days = ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun']; |
| 346 |
if (item.date?.length > 0) return days[new Date(item.date).getDay()]; |
| 347 |
}, |
| 348 |
time: (item) => this.vals(item).map((v) => v.split('T')[1].slice(0, 5)), |
| 349 |
timeshift: (item) => { |
| 350 |
const vals = this.vals(item); |
| 351 |
let tsx = new Date(vals[0]); |
| 352 |
let adjust = Number(vals[1]) * 60 * 60 * 1000; |
| 353 |
tsx.setTime(tsx.getTime() + adjust); |
| 354 |
return tsx.toISOString(); |
| 355 |
}, |
| 356 |
hour: (item) => this.vals(item).map((v) => v.split('T')[1].split(':')[0]), |
| 357 |
datediff: (item) => { |
| 358 |
const [d1, d2] = this.vals(item); |
| 359 |
return ((d2 ? new Date(d2) : new Date()) - new Date(d1)) / (24 * 60 * 60 * 1000); |
| 360 |
}, |
| 361 |
timediff: (item) => { |
| 362 |
const [d1, d2] = this.vals(item); |
| 363 |
return (new Date(d2) - new Date(d1)); |
| 364 |
}, |
| 365 |
dateforms: (item) => { |
| 366 |
const basedate = this.vals(item)[0]; |
| 367 |
const [baseyear, basemonth, baseday] = basedate.split('-'); |
| 368 |
return [ |
| 369 |
basedate, |
| 370 |
`${basemonth}/${baseday}/${baseyear}`, |
| 371 |
basemonth.startsWith('0') ? `${basemonth.replace(/^0/, '')}/${baseday}/${baseyear}` : null, |
| 372 |
basemonth.startsWith('0') && baseday.startsWith('0') ? `${basemonth.replace(/^0/, '')}/${baseday.replace(/^0/, '')}/${baseyear}` : null, |
| 373 |
`${basemonth}/${baseday}/${baseyear.slice(2)}`, |
| 374 |
`${basemonth.replace(/^0/, '')}/${baseday.replace(/^0/, '')}/${baseyear.slice(2)}`, |
| 375 |
`${['', 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'][Number(basemonth)]} ${baseday.replace(/^0/, '')}, ${baseyear}` |
| 376 |
].filter(x => x); |
| 377 |
}, |
| 378 |
now: (item) => performance.now(), |
| 379 |
spandays: (item) => { |
| 380 |
const startdate = Array.isArray(item.start) ? item.start[0] : item.start; |
| 381 |
const enddate = Array.isArray(item.end) ? item.end[0] : item.end; |
| 382 |
const days = []; |
| 383 |
const d = new Date(startdate); |
| 384 |
d.setUTCHours(0, 0, 0, 0); |
| 385 |
const stop = new Date(enddate); |
| 386 |
stop.setUTCHours(0, 0, 0, 0); |
| 387 |
|
| 388 |
while (d < stop) { |
| 389 |
days.push(d.toISOString().slice(0, 10)); |
| 390 |
d.setUTCDate(d.getUTCDate() + 1); |
| 391 |
} |
| 392 |
return days; |
| 393 |
}, |
| 394 |
round: (item) => this.numvals(item).map((v) => Math.round(v)), |
| 395 |
roundaway: (item) => this.numvals(item).map((v) => Math.sign(v) * Math.round(Math.abs(v))), |
| 396 |
roundm: (item) => { |
| 397 |
const vals = this.numvals(item); |
| 398 |
const multiple = vals.pop(); |
| 399 |
return vals.map((v) => Math.round(v / multiple) * multiple); |
| 400 |
}, |
| 401 |
roundd: (item) => { |
| 402 |
const vals = this.numvals(item); |
| 403 |
const digits = vals.pop(); |
| 404 |
return vals.map((v) => { |
| 405 |
const factor = 10 ** (Math.floor(Math.log10(v)) - digits + 1); |
| 406 |
return Math.round(v / factor) * factor; |
| 407 |
}); |
| 408 |
}, |
| 409 |
floor: (item) => this.numvals(item).map((v) => Math.floor(v)), |
| 410 |
ceil: (item) => this.numvals(item).map((v) => Math.ceil(v)), |
| 411 |
n: (item) => { |
| 412 |
let i = 0; |
| 413 |
const test = this.getname(item.of[0]); |
| 414 |
for (const key in item) { |
| 415 |
if (key != 'of') { |
| 416 |
i--; |
| 417 |
if (key == test) return i; |
| 418 |
} |
| 419 |
} |
| 420 |
return i - 1; |
| 421 |
}, |
| 422 |
numbers: (item) => { |
| 423 |
const res = []; |
| 424 |
const firstnum = item.from != undefined && item.from != null ? Number(item.from) : 1; |
| 425 |
const lastnum = item.to != undefined && item.to != null ? Number(item.to) : Number(this.getname(item.of[0])); |
| 426 |
for (let x = firstnum; x <= lastnum; x++) { |
| 427 |
res.push(x) |
| 428 |
} |
| 429 |
return res; |
| 430 |
}, |
| 431 |
dehyphenate: (item) => this.vals(item).flatMap((v) => v.replaceAll(/(?<=\w)-\n(?=\w)/g, '')), |
| 432 |
deparen: (item) => this.vals(item).flatMap((v) => v.replaceAll(/\s*\(.*?\)\s*/g, '')), |
| 433 |
sentences: (item) => this.vals(item).flatMap((v) => v.split(/[.?!…]['"’â€Â»]?\s+/)), |
| 434 |
allwords: (item) => this.vals(item).flatMap((v) => Array.from(v.toString().toLowerCase().matchAll(/[\p{Letter}\p{Number}]+(?:['‘’][\p{Letter}\p{Number}]+)?/gu).map((m) => m[0]))), |
| 435 |
words: (item) => this.vals(item).flatMap((v) => Array.from(v.toString().toLowerCase().matchAll(/[\p{Letter}\p{Number}]+(?:['‘’][\p{Letter}\p{Number}]+)?/gu).map((m) => m[0])).filter((w) => w.length >= 4)), |
| 436 |
characters: (item) => this.vals(item).flatMap((v) => Array.from(v.toString())), |
| 437 |
'character count': (item) => this.vals(item).flatMap((v) => v.length), |
| 438 |
case: (item) => this.vals(item).map((v) => { |
| 439 |
const hasupper = v.match(/[A-Z]/); |
| 440 |
const haslower = v.match(/[a-z]/); |
| 441 |
if (!hasupper && !haslower) { |
| 442 |
return 'none'; |
| 443 |
} else if (hasupper && !haslower) { |
| 444 |
return 'upper'; |
| 445 |
} else if (haslower && !hasupper) { |
| 446 |
return 'lower'; |
| 447 |
} else { |
| 448 |
if (v[0].match(/[A-Z]/) && !(v.slice(1).match(/[A-Z]/))) { |
| 449 |
return 'initial'; |
| 450 |
} else { |
| 451 |
return 'mixed'; |
| 452 |
} |
| 453 |
} |
| 454 |
}), |
| 455 |
uppercase: (item) => this.vals(item).flatMap((v) => v.toString().toUpperCase()), |
| 456 |
lowercase: (item) => this.vals(item).flatMap((v) => v.toString().toLowerCase().replaceAll(/[‘’`]/g, "'").replaceAll(/[“â€]/g, '"')), |
| 457 |
list: (item) => this.kkeys(item), |
| 458 |
items: (item) => item?.of?.flatMap((typename) => this.data?.[this.getname(typename)]), |
| 459 |
traverse: (item) => { |
| 460 |
const props = this.vals(item); |
| 461 |
return item?.of?.flatMap((subitem) => props.flatMap((prop) => this.step(subitem, prop, null, {}))); |
| 462 |
}, |
| 463 |
random: (item) => Math.random(), |
| 464 |
shuffle: (item) => { |
| 465 |
const newArray = [].concat(item?.of || []); |
| 466 |
for (let i = newArray.length - 1; i > 0; i--) { |
| 467 |
const j = Math.floor(Math.random() * (i + 1)); |
| 468 |
[newArray[i], newArray[j]] = [newArray[j], newArray[i]]; |
| 469 |
} |
| 470 |
return newArray; |
| 471 |
}, |
| 472 |
'weighted shuffle': (item) => { |
| 473 |
return item.of.map((subitem) => ({ |
| 474 |
weight: Number(subitem.weight || 1) * Math.random(), |
| 475 |
subitem: subitem |
| 476 |
})).sort((a, b) => b.weight - a.weight).map((ws) => ws.subitem); |
| 477 |
}, |
| 478 |
pick: (item) => item?.of?.[~~(Math.random() * item?.of?.length)], |
| 479 |
link: (item) => `<a href="${item.url}"${item.target ? ' target="' + item.target + '"' : ''}>${item.text}</a>`, |
| 480 |
img: (item) => { |
| 481 |
if (item.uri) { |
| 482 |
return `<a href="${item.uri}"><img src="${item.src}" height=${item.height}px width=${item.width}px></a>`; |
| 483 |
} else { |
| 484 |
return `<img src="${item.src}" height=${item.height}px width=${item.width}px>`; |
| 485 |
} |
| 486 |
}, |
| 487 |
sign: (item) => item.of.map((subitem) => { |
| 488 |
if (subitem.id) { |
| 489 |
return subitem; |
| 490 |
} else { |
| 491 |
const str = JSON.stringify(subitem); |
| 492 |
let hash = 0; |
| 493 |
for (let i = 0; i < str.length; i++) { |
| 494 |
const char = str.charCodeAt(i); |
| 495 |
hash = (hash << 5) - hash + char; |
| 496 |
} |
| 497 |
return Object.assign({id: (hash >>> 0).toString(36).padStart(7, '0')}, subitem); |
| 498 |
} |
| 499 |
}), |
| 500 |
'parse query': (item) => item.of.map((subitem) => ({ |
| 501 |
name: subitem.name, |
| 502 |
query: subitem.query, |
| 503 |
assembly: this.parse(subitem.query) |
| 504 |
})), |
| 505 |
results: (item) => this.execute(item.of, this.parse(item.query || item.of?.[0])) |
| 506 |
} |
| 507 |
this.data.annotators = Object.entries(this.annotators).map(([key, code]) => ({id: key, code: code})); |
| 508 |
this.destinations = new Set(); |
| 509 |
} |
| 510 |
|
| 511 |
survey() { |
| 512 |
this.destinations = new Set(Object.keys(this.data).concat(this.data?.queries?.map((q) => q.name) || []).concat(Object.keys(this.adapters))); |
| 513 |
} |
| 514 |
|
| 515 |
vals(item) { |
| 516 |
const getname = this.getname; |
| 517 |
const itemvals = []; |
| 518 |
if (Object.keys(item).length > 1) { |
| 519 |
Object.entries(item).filter(([k, v]) => k != 'of').flatMap(([k, v]) => Array.isArray(v) ? v : [v]).map(getname).forEach((v) => itemvals.push(v)); |
| 520 |
} else { |
| 521 |
item.of.map(getname).forEach((v) => itemvals.push(v)); |
| 522 |
} |
| 523 |
return itemvals; |
| 524 |
} |
| 525 |
|
| 526 |
numvals(item) { |
| 527 |
return this.vals(item).filter((val) => this.dtype(val, 'number')).map((val) => Number(val)); |
| 528 |
} |
| 529 |
|
| 530 |
kkeys(item) { |
| 531 |
return Object.keys(item).filter((k) => k != 'of'); |
| 532 |
} |
| 533 |
|
| 534 |
async querylive(query, inputlist = null) { |
| 535 |
this.recache = new Set(); |
| 536 |
const res = await this.query(query, inputlist); |
| 537 |
return res; |
| 538 |
} |
| 539 |
|
| 540 |
async query(query, inputlist = null, loop = 50) { |
| 541 |
this.survey(); |
| 542 |
const operations = Array.isArray(query) ? query : this.assemble(this.tokenize(query)); |
| 543 |
const result = this.execute(inputlist, operations); |
| 544 |
this.data['current results'] = result; |
| 545 |
const queued = Object.keys(this.adapters).filter((key) => this.adapters[key].queue.length > 0); |
| 546 |
if (loop > 0 && queued.length > 0) { |
| 547 |
await this.adapt(); |
| 548 |
if (this.recache) queued.forEach((q) => this.recache.add(q)); |
| 549 |
const reres = await this.query(operations, inputlist, loop - 1); |
| 550 |
return reres; |
| 551 |
} else if (loop === 0) { |
| 552 |
this.recache = false; |
| 553 |
Object.keys(this.adapters).forEach((key) => { |
| 554 |
if (!this.adapters[key].annotator) { |
| 555 |
this.adapters[key].queue.forEach((id) => { |
| 556 |
(this.index[key] ||= {})[id] = null; |
| 557 |
}) |
| 558 |
} |
| 559 |
}) |
| 560 |
} |
| 561 |
this.recache = false; |
| 562 |
return result; |
| 563 |
} |
| 564 |
|
| 565 |
async adapt() { |
| 566 |
for (const key in this.adapters) { |
| 567 |
const adapter = this.adapters[key]; |
| 568 |
if (adapter.queue.length > 0) { |
| 569 |
if (adapter.annotator) { |
| 570 |
while (adapter.queue.length > 0) { |
| 571 |
this.statusf('annotating ' + key + ' ' + adapter.queue.length); |
| 572 |
const item_to_annotate = adapter.queue.shift(); |
| 573 |
try { |
| 574 |
const annotation = await adapter.f(item_to_annotate); |
| 575 |
(this.index[key] ||= {})[this.getid(item_to_annotate)] = annotation; |
| 576 |
this.indexlogit(key, 'write'); |
| 577 |
} catch (e) { |
| 578 |
console.error(e); |
| 579 |
} |
| 580 |
} |
| 581 |
} else { |
| 582 |
const newids = adapter.queue.filter((id) => !adapter.pending.has(id)); |
| 583 |
newids.forEach((id) => adapter.pending.add(id)); |
| 584 |
const res = await adapter.f(Array.from(new Set(newids))); |
| 585 |
newids.forEach((id) => adapter.pending.delete(id)); |
| 586 |
for (const item of res) { |
| 587 |
const id = this.getid(item); |
| 588 |
(this.index[key] ||= {})[id] = item; |
| 589 |
this.indexlogit(key, 'write'); |
| 590 |
if (key in this.data) { |
| 591 |
this.data[key].push(item); |
| 592 |
} |
| 593 |
} |
| 594 |
this.adapters[key].queue = []; |
| 595 |
} |
| 596 |
} |
| 597 |
} |
| 598 |
this.adaptive = false; |
| 599 |
} |
| 600 |
|
| 601 |
async rehope(only = null) { |
| 602 |
var reset = 0; |
| 603 |
Object.keys(this.index).forEach((k) => { |
| 604 |
if (!only || only == k) { |
| 605 |
const todelete = Object.keys(this.index[k]).filter((dk) => this.index[k][dk] == null || this.index[k][dk] == undefined); |
| 606 |
todelete.forEach((dk) => delete (this.index[k][dk])); |
| 607 |
reset += todelete.length; |
| 608 |
} |
| 609 |
}); |
| 610 |
return reset; |
| 611 |
} |
| 612 |
|
| 613 |
load(something, named, append = false) { |
| 614 |
if (!named) return |
| 615 |
if (!append || !(named in this.data)) this.data[named] = []; |
| 616 |
if (Array.isArray(something)) { |
| 617 |
something.forEach((somethingx) => this.data[named].push(somethingx)); |
| 618 |
} else if (typeof something == 'object') { |
| 619 |
const firstkey = Object.keys(something)[0]; |
| 620 |
const firstobj = something[firstkey]; |
| 621 |
if (typeof firstobj === 'object' && !Array.isArray(firstobj) && (firstkey in Object.values(firstobj) || 'id' in firstobj)) { |
| 622 |
this.data[named].push(...Object.values(something)); |
| 623 |
} else if (typeof firstobj === 'object' && !Array.isArray(firstobj)) { |
| 624 |
this.data[named].push(...Object.entries(firstobj).map(([key, val]) => ({id: key, ...val}))); |
| 625 |
} else if (typeof firstobj === 'string') { |
| 626 |
this.data[named].push(...Object.entries(something).map(([key, val]) => ({id: key, name: val}))); |
| 627 |
} else { |
| 628 |
this.data[named].push(...Object.entries(something).map(([key, val]) => ({id: key, value: val}))); |
| 629 |
} |
| 630 |
} else { |
| 631 |
this.data[named].push(something); |
| 632 |
} |
| 633 |
return this.data[named]; |
| 634 |
} |
| 635 |
|
| 636 |
async loadjsonl(something, named, append = false) { |
| 637 |
if (!named) return |
| 638 |
if (!append || !(named in this.data)) this.data[named] = []; |
| 639 |
if (typeof something == 'string' && (something.startsWith('http') || something.startsWith('file://'))) { |
| 640 |
const fetchres = await fetch(something); |
| 641 |
something = await fetchres.text(); |
| 642 |
} |
| 643 |
var rows = something.trim().split(/[\n\r]+/); |
| 644 |
for (const row of rows) { |
| 645 |
const rowdata = JSON.parse(row); |
| 646 |
if (rowdata) this.data[named].push(rowdata); |
| 647 |
} |
| 648 |
} |
| 649 |
|
| 650 |
async loadcsv(something, named, quoteChar = '"', delimiter = ',', headerrows = 1) { |
| 651 |
if (typeof something == 'string' && (something.startsWith('http') || something.startsWith('file://'))) { |
| 652 |
const fetchres = await fetch(something); |
| 653 |
something = await fetchres.text(); |
| 654 |
} |
| 655 |
var rows = something.split(/[\n\r]+/); |
| 656 |
|
| 657 |
const regex = new RegExp(`\\s*(${quoteChar})?(.*?)\\1\\s*(?:${delimiter}|$)`, 'gs'); |
| 658 |
|
| 659 |
const match = (line) => Array.from(line.matchAll(regex), (m) => m[2]); |
| 660 |
|
| 661 |
const headers = []; |
| 662 |
for (let hrowx = 0; hrowx < headerrows; hrowx++) { |
| 663 |
const hrow = rows.shift(); |
| 664 |
match(hrow).forEach((h, hx) => { |
| 665 |
if (hrowx === 0) { |
| 666 |
headers.push(h); |
| 667 |
} else { |
| 668 |
headers[hx] = headers[hx] + ' ' + h; |
| 669 |
} |
| 670 |
}); |
| 671 |
} |
| 672 |
const heads = headers.length > 0 ? headers : match(rows.shift()); |
| 673 |
var lines = rows.slice(0).filter((line) => line); |
| 674 |
const parsed = lines.map((line) => { |
| 675 |
return match(line).reduce((acc, cur, i) => { |
| 676 |
// replace blank matches with `null` |
| 677 |
const val = cur.length <= 0 ? null : (!isNaN(cur) ? Number(cur) : cur); |
| 678 |
const key = heads[i] ?? `{i}`; |
| 679 |
if (key == '') { |
| 680 |
return {...acc}; |
| 681 |
} else { |
| 682 |
return {...acc, [key]: val}; |
| 683 |
} |
| 684 |
}, {}); |
| 685 |
}); |
| 686 |
this.load(parsed, named); |
| 687 |
} |
| 688 |
|
| 689 |
apacheLogToDate(apacheTimestamp) { |
| 690 |
// Apache log format: [10/Oct/2000:13:55:36 -0700] |
| 691 |
// Remove brackets if present |
| 692 |
const cleanTimestamp = apacheTimestamp.replace(/^\[|\]$/g, ''); |
| 693 |
|
| 694 |
// Split into date/time and timezone parts |
| 695 |
const [dateTimePart, timezone] = cleanTimestamp.split(' '); |
| 696 |
|
| 697 |
// Parse the date/time part: dd/MMM/yyyy:HH:mm:ss |
| 698 |
const [datePart, hour, minute, second] = dateTimePart.split(':'); |
| 699 |
const [day, month, year] = datePart.split('/'); |
| 700 |
|
| 701 |
// Month mapping |
| 702 |
const months = { |
| 703 |
'Jan': 0, 'Feb': 1, 'Mar': 2, 'Apr': 3, 'May': 4, 'Jun': 5, |
| 704 |
'Jul': 6, 'Aug': 7, 'Sep': 8, 'Oct': 9, 'Nov': 10, 'Dec': 11 |
| 705 |
}; |
| 706 |
|
| 707 |
// Create Date object (months are 0-indexed in JS) |
| 708 |
const date = new Date( |
| 709 |
parseInt(year), |
| 710 |
months[month], |
| 711 |
parseInt(day), |
| 712 |
parseInt(hour), |
| 713 |
parseInt(minute), |
| 714 |
parseInt(second) |
| 715 |
); |
| 716 |
|
| 717 |
// Handle timezone offset if present |
| 718 |
if (timezone) { |
| 719 |
const sign = timezone[0] === '+' ? 1 : -1; |
| 720 |
const tzHours = parseInt(timezone.slice(1, 3)); |
| 721 |
const tzMinutes = parseInt(timezone.slice(3, 5)); |
| 722 |
const offsetMs = sign * (tzHours * 60 + tzMinutes) * 60 * 1000; |
| 723 |
|
| 724 |
// Adjust for timezone (Apache logs are in local time, JS Date assumes UTC) |
| 725 |
date.setTime(date.getTime() - offsetMs); |
| 726 |
} |
| 727 |
|
| 728 |
return date; |
| 729 |
} |
| 730 |
|
| 731 |
async loadclf(loglines, named, apiroutes = []) { |
| 732 |
const lines = loglines.trim().split('\n'); |
| 733 |
const cols9 = ['ip', 'name', 'username', 'timestamp', 'requestraw', 'status', 'bytes', 'referrer', 'useragent']; |
| 734 |
const cols10 = ['host'].concat(cols9); |
| 735 |
const parsed = lines.filter((line) => line?.length > 0).map((line) => { |
| 736 |
const vals = Array.from(line.matchAll(/\"(?:\\"|.)*?\"|\[.*?\]|\S+/g)).map((m) => m[0]); |
| 737 |
const cols = vals.length == 10 ? cols10 : cols9; |
| 738 |
const obj = Object.fromEntries(vals.map((v, vx) => ([cols[vx], v]))); |
| 739 |
const date = this.apacheLogToDate(obj.timestamp); |
| 740 |
obj.timestamp = date.toISOString(); |
| 741 |
[obj.date, obj.time] = obj.timestamp.slice(0, -1).split('T'); |
| 742 |
if (obj.requestraw.match(/ [^ ]+ /) && !obj.requestraw.startsWith('"{')) { |
| 743 |
[obj.method, obj.request, obj.protocol] = obj.requestraw.slice(1, -1).split(' '); |
| 744 |
if (this.data['.clf API routes']) { |
| 745 |
for (const apiroute of this.data['.clf API routes'].sort((a, b) => b.length - a.length)) { |
| 746 |
if (obj.request.startsWith(apiroute)) { |
| 747 |
obj.page = apiroute; |
| 748 |
break; |
| 749 |
} |
| 750 |
} |
| 751 |
} |
| 752 |
obj.page ??= obj.request.split('?')[0]; |
| 753 |
} |
| 754 |
obj.referrer = obj.referrer.slice(1, -1); |
| 755 |
obj.useragent = obj.useragent.slice(1, -1); |
| 756 |
obj.logline = line; |
| 757 |
return obj; |
| 758 |
}); |
| 759 |
this.load(parsed, named); |
| 760 |
} |
| 761 |
|
| 762 |
async loadrss(rsstext, named) { |
| 763 |
const rssval = (k, rawval) => { |
| 764 |
if (k.match(/date/i)) { |
| 765 |
return new Date(rawval).toISOString(); |
| 766 |
} else if (!isNaN(rawval)) { |
| 767 |
return Number(rawval); |
| 768 |
} else { |
| 769 |
return rawval; |
| 770 |
} |
| 771 |
} |
| 772 |
const rssdom = new window.DOMParser().parseFromString(rsstext, "text/xml"); |
| 773 |
const items = Array.from(rssdom.querySelectorAll('item')).map((i) => { |
| 774 |
const obj = {}; |
| 775 |
Array.from(i.children).forEach((c) => { |
| 776 |
const k = c.tagName; |
| 777 |
const rawval = c.textContent.trim(); |
| 778 |
obj[k] = rssval(k, rawval); |
| 779 |
if (k.match(/date/i) && obj[k].length > 10 && !('date' in obj)) obj.date = obj[k].slice(0, 10) |
| 780 |
}); |
| 781 |
return obj; |
| 782 |
}); |
| 783 |
if (named) { |
| 784 |
this.load(items, named); |
| 785 |
} else { |
| 786 |
return items; |
| 787 |
} |
| 788 |
} |
| 789 |
|
| 790 |
async loadopml(opmltext, named) { |
| 791 |
const opmldom = new window.DOMParser().parseFromString(opmltext, "text/xml"); |
| 792 |
const feeds = Array.from(opmldom.querySelectorAll('outline[type="rss"]')).map((i) => ({ |
| 793 |
title: i.getAttribute('title'), |
| 794 |
feed: i.getAttribute('xmlUrl'), |
| 795 |
site: i.getAttribute('htmlUrl') |
| 796 |
})); |
| 797 |
if (named) { |
| 798 |
this.load(feeds, named); |
| 799 |
} else { |
| 800 |
return feeds; |
| 801 |
} |
| 802 |
} |
| 803 |
|
| 804 |
connect(key, adapter, doc = {}, annotator = null) { |
| 805 |
this.adapters[key] = {queue: [], pending: new Set(), f: adapter, annotator: annotator, doc: doc}; |
| 806 |
this.data.adapters ??= []; |
| 807 |
if (!this.data.adapters.find((a) => a.id == key)) this.data.adapters.push({ |
| 808 |
id: key, |
| 809 |
requires: doc.requires, |
| 810 |
produces: doc.produces, |
| 811 |
code: this.adapters[key].f |
| 812 |
}); |
| 813 |
if (annotator) this.register(key, (item) => this.resolve(key, item), adapter); |
| 814 |
} |
| 815 |
|
| 816 |
connect_annotator(key, adapter, required = [], doc = {}) { |
| 817 |
this.connect(key, adapter, doc, required); |
| 818 |
} |
| 819 |
|
| 820 |
register(key, annotator, displayf = null) { |
| 821 |
this.annotators[key] = annotator; |
| 822 |
this.data.annotators ??= []; |
| 823 |
if (!this.data.annotators.find((a) => a.id == key)) this.data.annotators.push({ |
| 824 |
id: key, |
| 825 |
code: displayf || this.annotators[key] |
| 826 |
}); |
| 827 |
} |
| 828 |
|
| 829 |
unbracket(token) { |
| 830 |
if (!(typeof token == 'string')) token = token.toString(); |
| 831 |
if (token.startsWith('[') && token.endsWith(']')) { |
| 832 |
return token.slice(1, -1).replaceAll(']]', ']'); |
| 833 |
} |
| 834 |
return token; |
| 835 |
} |
| 836 |
|
| 837 |
bracket(token) { |
| 838 |
if (!(typeof token == 'string')) token = token?.toString() ?? ''; |
| 839 |
if (token.match(/[?.:#\/|!<>=~@\[\]\(\),;\+-]/) || token.startsWith(' ') || token.endsWith(' ')) { |
| 840 |
return '[' + token.replaceAll(']', ']]') + ']'; |
| 841 |
} |
| 842 |
return token; |
| 843 |
} |
| 844 |
|
| 845 |
escapeRegExp(string) { |
| 846 |
return string.replace(/[.*+?${}()|[\]\\]/g, "\\$&"); |
| 847 |
} |
| 848 |
|
| 849 |
tokenize(text) { |
| 850 |
if (typeof text != 'string') text = String(text); |
| 851 |
const matches = text.matchAll(/(\[(?:\]\]|[^\]])*\])|(\?{1,3})|(\.{1,4})|(\/{1,2})|([\:\#\|\!])|(\()|(\))|([@~=<>\+-]+)|(\,)|(\;)|([^\[\]\(\)\.\?\:\/\#\|\!~=<>@,;\+-]+)|([\[\]])/gms); |
| 852 |
const tokenlist = Array.from(matches, m => m[0].trim()).filter((token) => token.length > 0); |
| 853 |
return tokenlist; |
| 854 |
} |
| 855 |
|
| 856 |
assemble(tokenized) { |
| 857 |
const isOperator = (token) => '???....://#|!'.includes(token); |
| 858 |
const tokenlist = '= ~ =~ ~< ~> > < <> >< >= <= - =- ~- + =+ @ @@ @- @= @@= =@ =@@ @< @@< @<= @@<= @> @@> @>= @@>= =>'.split(' '); |
| 859 |
const isSubop = (token) => tokenlist.includes(token) || (token?.startsWith('-') && (token.length == 1 || tokenlist.includes(token.slice(1)))); |
| 860 |
const isSeparator = (token) => ',;'.includes(token); |
| 861 |
const isValue = (token) => !isOperator(token) && !isSubop(token) && !isSeparator(token); |
| 862 |
|
| 863 |
let tokens = tokenized.slice(); |
| 864 |
let level = 0; |
| 865 |
const operations = []; |
| 866 |
while (tokens.length > 0) { |
| 867 |
const op = {operator: null, args: []}; |
| 868 |
let token = tokens.shift(); |
| 869 |
if (isOperator(token) || ((isValue(token) || isSubop(token)) && operations.length === 0)) { |
| 870 |
if ((isValue(token) || isSubop(token)) && operations.length === 0) { |
| 871 |
op.operator = '?'; |
| 872 |
tokens.unshift(token); |
| 873 |
} else { |
| 874 |
op.operator = token; |
| 875 |
} |
| 876 |
while (tokens.length > 0 && !isOperator(tokens[0])) { |
| 877 |
const arg = {separator: null, label: null, subop: null, value: null}; |
| 878 |
if (isSeparator(tokens[0])) { |
| 879 |
arg.separator = tokens.shift(); |
| 880 |
} |
| 881 |
while (tokens.length > 0 && !isOperator(tokens[0]) && !isSeparator(tokens[0])) { |
| 882 |
const frag = tokens.shift(); |
| 883 |
if (frag != '(' && isValue(frag) && isSubop(tokens[0])) { |
| 884 |
arg.label = this.unbracket(frag); |
| 885 |
arg.subop = tokens.shift(); |
| 886 |
} else if (isSubop(frag)) { |
| 887 |
arg.subop = frag; |
| 888 |
} else if (frag === '(') { |
| 889 |
const subquery = []; |
| 890 |
level++; |
| 891 |
while (tokens.length > 0 && level > 0) { |
| 892 |
const sub = tokens.shift(); |
| 893 |
if (sub === '(') { |
| 894 |
level++; |
| 895 |
if (level > 0) { |
| 896 |
subquery.push(sub); |
| 897 |
} |
| 898 |
} else if (sub === ')') { |
| 899 |
level--; |
| 900 |
if (level > 0) { |
| 901 |
subquery.push(sub); |
| 902 |
} |
| 903 |
} else if (level > 0) { |
| 904 |
subquery.push(sub); |
| 905 |
} |
| 906 |
} |
| 907 |
const subexpr = this.assemble(subquery); |
| 908 |
arg.value = subexpr; |
| 909 |
} else if (frag.match(/^[=<>@~-]+$/)) { |
| 910 |
const failure = { |
| 911 |
tokens: tokenized, |
| 912 |
assembled: operations.slice(0), |
| 913 |
assembling: {op: op, arg: arg, unexpected: frag}, |
| 914 |
unassembled: tokens.slice(0) |
| 915 |
} |
| 916 |
throw new Error("Invalid subop", {cause: failure}); |
| 917 |
} else if (!arg.value) { |
| 918 |
arg.value = this.unbracket(frag); |
| 919 |
} else { |
| 920 |
const failure = { |
| 921 |
tokens: tokenized, |
| 922 |
assembled: operations.slice(0), |
| 923 |
assembling: {op: op, arg: arg, unexpected: frag}, |
| 924 |
unassembled: tokens.slice(0) |
| 925 |
} |
| 926 |
throw new Error("Unexpected token", {cause: failure}); |
| 927 |
} |
| 928 |
} |
| 929 |
op.args.push(arg); |
| 930 |
} |
| 931 |
} |
| 932 |
operations.push(op); |
| 933 |
|
| 934 |
} |
| 935 |
if (level > 0) console.warn({parentropy: level, operations: operations}); |
| 936 |
return operations; |
| 937 |
} |
| 938 |
|
| 939 |
parse(querystr) { |
| 940 |
return this.assemble(this.tokenize(querystr)); |
| 941 |
} |
| 942 |
|
| 943 |
disassemble(assembly) { |
| 944 |
return assembly.map((operation, ox) => operation.operator + ((operation.args.length == 0 && ox < assembly.length - 1 && operation.operator[0] == assembly[ox + 1].operator[0]) ? ' ' : operation.args.map((arg) => (arg.separator ?? '') + (arg.label ? this.bracket(arg.label) : '') + (arg.subop ?? '') + (Array.isArray(arg.value) ? ('(' + this.disassemble(arg.value) + ')') : (arg.value ? this.bracket(arg.value) : ''))).join(''))).join('').replace(/^\?(?=[a-z])/, ''); |
| 945 |
} |
| 946 |
|
| 947 |
compact(querystr) { |
| 948 |
return this.disassemble(this.parse(querystr)); |
| 949 |
} |
| 950 |
|
| 951 |
executeq(querystr) { |
| 952 |
return this.execute([], this.assemble(this.tokenize(querystr))); |
| 953 |
} |
| 954 |
|
| 955 |
timecheck(timer, i, count, op) { |
| 956 |
if (i == 1) timer.loopstart = new Date(); |
| 957 |
if (i >= 10 && i >= count / 100) { |
| 958 |
const taken = new Date() - timer.loopstart; |
| 959 |
const projected = count * taken / i; |
| 960 |
if (this.timelimit && projected > this.timelimit) throw new Error('Query overrun.', { |
| 961 |
cause: { |
| 962 |
operation: this.disassemble([op]), |
| 963 |
items: count, |
| 964 |
done: i, |
| 965 |
elapsed: taken + 'ms', |
| 966 |
projected: Math.round(projected / 60000) + ' minutes', |
| 967 |
timelimit: Math.round(this.timelimit / 60000) + ' minutes' |
| 968 |
} |
| 969 |
}); |
| 970 |
} |
| 971 |
} |
| 972 |
|
| 973 |
unmmss = (mmssstr) => { |
| 974 |
const parts = mmssstr.split(':'); |
| 975 |
let s = 0; |
| 976 |
const seconds = parts.pop(); |
| 977 |
if (seconds) s += Number(seconds); |
| 978 |
const minutes = parts.pop(); |
| 979 |
if (minutes) s += 60 * Number(minutes); |
| 980 |
const hours = parts.pop(); |
| 981 |
if (hours) s += 60 * 60 * Number(hours); |
| 982 |
return s; |
| 983 |
} |
| 984 |
|
| 985 |
dethe = (value) => { |
| 986 |
return value.toLowerCase().replace(/^the /, '') |
| 987 |
}; |
| 988 |
|
| 989 |
compvals = (araw, braw) => { |
| 990 |
const a = araw.toString().trim(); |
| 991 |
const b = braw.toString().trim(); |
| 992 |
const atime = a.match(/^(?:\d+)(?:\:\d{2,})+$/); |
| 993 |
const btime = b.match(/^(?:\d+)(?:\:\d{2,})+$/); |
| 994 |
if (atime && btime) { |
| 995 |
return this.unmmss(btime[0]) - this.unmmss(atime[0]); |
| 996 |
} |
| 997 |
return this.dethe(a).localeCompare(this.dethe(b)); |
| 998 |
} |
| 999 |
|
| 1000 |
execute(inputlistraw, operations, labeled = null, level = null) { |
| 1001 |
let inputlist = Array.isArray(inputlistraw) ? inputlistraw : inputlistraw ? [inputlistraw] : []; |
| 1002 |
let currentlist = inputlist?.slice(0) || []; |
| 1003 |
const getname = this.getname; |
| 1004 |
const getid = this.getid; |
| 1005 |
const dtype = this.dtype; |
| 1006 |
const nullish = this.nullish; |
| 1007 |
const dcopy = this.dcopy; |
| 1008 |
const opclone = this.opclone; |
| 1009 |
const step = this.step; |
| 1010 |
const dethe = this.dethe; |
| 1011 |
const compvals = this.compvals; |
| 1012 |
|
| 1013 |
labeled ||= {}; |
| 1014 |
let outputlist = []; |
| 1015 |
const toplevel = !inputlistraw && !level; |
| 1016 |
if (toplevel && operations[0]?.progress) { |
| 1017 |
currentlist = operations[0].progress; |
| 1018 |
labeled = operations[0].labeled; |
| 1019 |
outputlist = currentlist; |
| 1020 |
} |
| 1021 |
for (let opx = 0; opx < operations.length; opx++) { |
| 1022 |
const op = operations[opx]; |
| 1023 |
if (op.completed) continue; |
| 1024 |
const opstart = performance.now(); |
| 1025 |
outputlist = []; |
| 1026 |
switch (op.operator) { |
| 1027 |
case '?': // start |
| 1028 |
if (op.args.length == 0) { |
| 1029 |
outputlist = Object.keys(this.data).filter((x) => !this.internal_datasets.includes(x)).sort((a, b) => dethe(a).localeCompare(dethe(b))); |
| 1030 |
break; |
| 1031 |
} |
| 1032 |
outputlist = []; |
| 1033 |
op.args.forEach((arg) => { |
| 1034 |
let startitems = []; |
| 1035 |
if (arg.subop?.includes('+')) { |
| 1036 |
if (arg.label) { |
| 1037 |
if (arg.label in labeled) { |
| 1038 |
labeled[arg.label].forEach((i) => outputlist.push(i)); |
| 1039 |
} else if (arg.label in this.data) { |
| 1040 |
this.data[arg.label].forEach((i) => outputlist.push(i)); |
| 1041 |
} |
| 1042 |
} else { |
| 1043 |
currentlist.forEach((i) => outputlist.push(i)); |
| 1044 |
} |
| 1045 |
} |
| 1046 |
let typeitems; |
| 1047 |
if (arg.separator != ';' || outputlist.length == 0) { |
| 1048 |
if (Array.isArray(arg.value)) { |
| 1049 |
startitems = this.execute(arg.subop?.includes('+') ? outputlist : [], arg.value, labeled); |
| 1050 |
} else if (arg.subop == '~') { |
| 1051 |
startitems = [arg.value]; |
| 1052 |
} else if (arg.value in labeled) { |
| 1053 |
startitems = labeled[arg.value]; |
| 1054 |
} else if (typeitems = this.gettype(arg.value)) { |
| 1055 |
startitems = typeitems; |
| 1056 |
} else if (arg.subop != '=') { |
| 1057 |
if (!isNaN(arg.value)) { |
| 1058 |
startitems = [Number(arg.value)]; |
| 1059 |
} else { |
| 1060 |
startitems = [arg.value]; |
| 1061 |
} |
| 1062 |
} |
| 1063 |
if (arg.label) this.index[arg.label] = {}; |
| 1064 |
if (arg.subop?.includes('+')) { |
| 1065 |
if (arg.label) { |
| 1066 |
startitems.forEach((i) => { |
| 1067 |
if (arg.label in labeled) (labeled[arg.label] ||= []).push(i); |
| 1068 |
outputlist.push(i); |
| 1069 |
}); |
| 1070 |
} else { |
| 1071 |
startitems.forEach((i) => outputlist.push(i)); |
| 1072 |
} |
| 1073 |
} else if (arg.subop?.includes('-')) { |
| 1074 |
const startids = new Set(startitems.map((si) => getid(si))); |
| 1075 |
if (arg.label && labeled[arg.label]) { |
| 1076 |
labeled[arg.label] = labeled[arg.label].filter((li) => !startids.has(getid(li))); |
| 1077 |
outputlist = labeled[arg.label]; |
| 1078 |
} else { |
| 1079 |
outputlist = outputlist.filter((li) => !startids.has(getid(li))); |
| 1080 |
} |
| 1081 |
} else { |
| 1082 |
if (arg.label) labeled[arg.label] = startitems; |
| 1083 |
startitems.forEach((si) => outputlist.push(si)); |
| 1084 |
} |
| 1085 |
} |
| 1086 |
}); |
| 1087 |
break; |
| 1088 |
case '??': // label |
| 1089 |
outputlist = currentlist; |
| 1090 |
let ended = false; |
| 1091 |
op.args.forEach((arg) => { |
| 1092 |
if (!ended) { |
| 1093 |
if (arg.label == '_timelimit' && !isNaN(arg.value)) { |
| 1094 |
this.timelimit = Number(arg.value) * 60000; |
| 1095 |
} else if (arg.label == 'status') { |
| 1096 |
if (arg.value) { |
| 1097 |
this.statusf(arg.value); |
| 1098 |
} else { |
| 1099 |
this.statusf(currentlist[0]); |
| 1100 |
} |
| 1101 |
} else if (arg.label) { |
| 1102 |
if (arg.subop.includes('~') && dtype(arg.value, 'string')) { |
| 1103 |
labeled[arg.label] = [arg.value]; |
| 1104 |
// } else if (arg.labeled) { |
| 1105 |
// labeled[arg.label] = arg.labeled; |
| 1106 |
} else { |
| 1107 |
if (arg.subop?.match(/\+/)) labeled[arg.label] ||= []; |
| 1108 |
const newvals = this.execute(currentlist, Array.isArray(arg.value) ? arg.value : '.' + arg.value, labeled, level); |
| 1109 |
if (arg.subop?.match(/\+/)) { |
| 1110 |
newvals.forEach((nv) => labeled[arg.label].push(nv)); |
| 1111 |
} else if (arg.subop?.includes('-') && labeled[arg.label]) { |
| 1112 |
const newids = new Set(newvals.map((ni) => getid(ni))); |
| 1113 |
labeled[arg.label] = labeled[arg.label].filter((li) => !newids.has(getid(li))); |
| 1114 |
} else { |
| 1115 |
labeled[arg.label] = newvals; |
| 1116 |
} |
| 1117 |
// if (!level && opx == 0) arg.labeled = newvals; |
| 1118 |
} |
| 1119 |
this.index[arg.label] = {}; |
| 1120 |
} else if (dtype(arg.value, 'string')) { |
| 1121 |
if (arg.subop?.match(/\+/)) { |
| 1122 |
labeled[arg.value] ??= []; |
| 1123 |
const currentids = new Set(currentlist.map((ci) => getid(ci))); |
| 1124 |
currentlist.forEach((i) => labeled[arg.value].push(i)); |
| 1125 |
} else if (arg.subop?.includes('-') && labeled[arg.value]) { |
| 1126 |
const currentids = new Set(currentlist.map((ci) => getid(ci))); |
| 1127 |
labeled[arg.value] = labeled[arg.value].filter((li) => !currentids.has(getid(li))); |
| 1128 |
} else { |
| 1129 |
labeled[arg.value] = currentlist.slice(0); |
| 1130 |
} |
| 1131 |
this.index[arg.value] = {}; |
| 1132 |
if (arg.value == 'end') { |
| 1133 |
ended = true; |
| 1134 |
} |
| 1135 |
} |
| 1136 |
} |
| 1137 |
}); |
| 1138 |
if (ended) return (this.debug && !inputlist) ? operations : outputlist; |
| 1139 |
break; |
| 1140 |
case '!': // repeat |
| 1141 |
if (opx > 0) { |
| 1142 |
const repeat_ops = operations.slice(opx - 1, opx + 1); |
| 1143 |
repeat_ops.forEach((rop) => { |
| 1144 |
delete (rop.progress); |
| 1145 |
delete (rop.labeled); |
| 1146 |
delete (rop.completed); |
| 1147 |
}); |
| 1148 |
outputlist = currentlist.slice(0); |
| 1149 |
level ??= 0; |
| 1150 |
level += 1; |
| 1151 |
const maxrecursion = ((op.args.length > 0 && op.args[0].value) || 1000); |
| 1152 |
if (outputlist.length > 0 && level < maxrecursion && (!this.samearray(inputlist, outputlist) || level == 1)) { |
| 1153 |
const recursed = this.execute(outputlist, repeat_ops, labeled, level); |
| 1154 |
if (recursed.length > 0 && !this.samearray(recursed, outputlist)) outputlist = recursed.slice(0); |
| 1155 |
} |
| 1156 |
} |
| 1157 |
break; |
| 1158 |
case '.': // traverse |
| 1159 |
case '..': // traverse with duplicates |
| 1160 |
const seen = new Set(); |
| 1161 |
const sofar = Array.isArray(op.args?.[0]?.value) && op.args?.[0]?.label; |
| 1162 |
if (sofar) labeled[sofar] = []; |
| 1163 |
const traversetimer = {}; |
| 1164 |
outputlist = currentlist.reduce((acc, item, i) => { |
| 1165 |
this.timecheck(traversetimer, i, currentlist.length, op); |
| 1166 |
if (op.args.length == 0) { |
| 1167 |
const itemkey = getid(item); |
| 1168 |
if (op.operator == '..' || !seen.has(itemkey)) { |
| 1169 |
acc.push(item); |
| 1170 |
seen.add(itemkey); |
| 1171 |
} |
| 1172 |
return acc; |
| 1173 |
} |
| 1174 |
if (op.args[0].subop?.includes('<') && acc.length > 0) return acc; |
| 1175 |
let itemvals = []; |
| 1176 |
let toremoveids = {}; |
| 1177 |
for (const arg of op.args.filter((arg) => arg.value != null && arg.value != undefined)) { |
| 1178 |
if (arg.separator != ';' || itemvals.length == 0) { |
| 1179 |
let itemval; |
| 1180 |
if (arg.subop?.includes('@') && arg.label != null && !isNaN(arg.value)) { |
| 1181 |
const itemvalraw = step(item, arg.label, null, labeled); |
| 1182 |
if (arg.subop.includes('@@')) { |
| 1183 |
itemval = itemvalraw.slice(itemvalraw.length - Number(arg.value)); |
| 1184 |
} else { |
| 1185 |
itemval = itemvalraw.slice(0, Number(arg.value)); |
| 1186 |
} |
| 1187 |
} else if (arg.subop?.includes('=') && (arg.subop?.includes('>') || arg.label == null)) { |
| 1188 |
if (arg.value in item) { |
| 1189 |
itemval = item[arg.value]; |
| 1190 |
} |
| 1191 |
} else { |
| 1192 |
itemval = step(item, arg.value, arg.subop, labeled); |
| 1193 |
} |
| 1194 |
if (itemval) { |
| 1195 |
if (!Array.isArray(itemval)) itemval = [itemval]; |
| 1196 |
if (arg.subop?.includes('-') && isNaN(arg.value)) { |
| 1197 |
itemval.forEach((subitem) => { |
| 1198 |
const subid = getid(subitem); |
| 1199 |
toremoveids[subid] ||= 0; |
| 1200 |
toremoveids[subid] += 1; |
| 1201 |
}); |
| 1202 |
} else { |
| 1203 |
var passdown = null; |
| 1204 |
if (arg.subop?.includes('>')) { |
| 1205 |
if (arg?.label in item) { |
| 1206 |
passdown = item[arg.label]; |
| 1207 |
} else { |
| 1208 |
passdown = JSON.parse(JSON.stringify(item, Object.keys(item).filter((k) => k != arg.value))); |
| 1209 |
} |
| 1210 |
} |
| 1211 |
itemval.forEach((subitem) => { |
| 1212 |
if (passdown) { |
| 1213 |
if (!dtype(subitem, 'object')) subitem = {value: subitem}; |
| 1214 |
if (arg.label) { |
| 1215 |
subitem[arg.label] = Array.isArray(passdown) ? passdown : [passdown]; |
| 1216 |
} else { |
| 1217 |
Object.assign(subitem, passdown); |
| 1218 |
} |
| 1219 |
} |
| 1220 |
const subitemkey = getid(subitem); |
| 1221 |
if (op.operator == '..' || !seen.has(subitemkey)) { |
| 1222 |
seen.add(subitemkey); |
| 1223 |
itemvals.push(subitem); |
| 1224 |
if (sofar) labeled[sofar].push(subitem); |
| 1225 |
} |
| 1226 |
}) |
| 1227 |
} |
| 1228 |
} |
| 1229 |
} |
| 1230 |
if (Object.keys(toremoveids).length > 0) { |
| 1231 |
if (op.operator == '.') { |
| 1232 |
itemvals = itemvals.filter((subitem) => !(getid(subitem) in toremoveids)); |
| 1233 |
} else { |
| 1234 |
itemvals = itemvals.filter((subitem) => { |
| 1235 |
const subid = getid(subitem); |
| 1236 |
if (toremoveids[subid] > 0) { |
| 1237 |
toremoveids[subid] -= 1; |
| 1238 |
return false; |
| 1239 |
} else { |
| 1240 |
return true; |
| 1241 |
} |
| 1242 |
}) |
| 1243 |
} |
| 1244 |
} |
| 1245 |
} |
| 1246 |
itemvals.forEach((x) => acc.push(x)); |
| 1247 |
return acc; |
| 1248 |
}, []); |
| 1249 |
outputlist = outputlist.filter((item) => item != null); |
| 1250 |
break; |
| 1251 |
case ':': // filter |
| 1252 |
if (!(op?.args?.length > 0)) { |
| 1253 |
outputlist = currentlist; |
| 1254 |
break; |
| 1255 |
} |
| 1256 |
|
| 1257 |
if (op?.args?.length == 1 && op.args[0].subop && op.args[0].label == null && op.args[0].value == null) { |
| 1258 |
let checklist; |
| 1259 |
if (currentlist.length == 0) { |
| 1260 |
outputlist = []; |
| 1261 |
break; |
| 1262 |
} else if (currentlist.length == 1) { |
| 1263 |
checklist = [currentlist[0], currentlist[0]]; |
| 1264 |
} else { |
| 1265 |
checklist = currentlist; |
| 1266 |
} |
| 1267 |
let comparator = op.args[0].subop; |
| 1268 |
let polarize = (x) => x; |
| 1269 |
let check; |
| 1270 |
if (comparator?.startsWith('-') || comparator?.endsWith('-')) { |
| 1271 |
polarize = (x) => !x; |
| 1272 |
comparator = comparator.replace(/-|-$/, ''); |
| 1273 |
} |
| 1274 |
for (let pi = 0; pi < checklist.length - 1; pi++) { |
| 1275 |
let a = getname(checklist[pi]); |
| 1276 |
let b = getname(checklist[pi + 1]); |
| 1277 |
switch (comparator) { |
| 1278 |
case '': |
| 1279 |
check = polarize(a == b); |
| 1280 |
break; |
| 1281 |
case '=': |
| 1282 |
check = polarize(a == b); |
| 1283 |
break; |
| 1284 |
case '>=': |
| 1285 |
check = polarize(a >= b); |
| 1286 |
break; |
| 1287 |
case '<=': |
| 1288 |
check = polarize(a <= b); |
| 1289 |
break; |
| 1290 |
case '>': |
| 1291 |
check = polarize(a > b); |
| 1292 |
break; |
| 1293 |
case '<': |
| 1294 |
check = polarize(a < b); |
| 1295 |
break; |
| 1296 |
case '~': |
| 1297 |
check = polarize(a?.toString().toLowerCase().includes(b?.toString().toLowerCase())); |
| 1298 |
break; |
| 1299 |
case '~<': |
| 1300 |
check = polarize(a?.toString().toLowerCase().startsWith(b?.toString().toLowerCase())); |
| 1301 |
break; |
| 1302 |
case '~>': |
| 1303 |
check = polarize(a?.toString().toLowerCase().endsWith(b?.toString().toLowerCase())); |
| 1304 |
break; |
| 1305 |
} |
| 1306 |
if (!check) { |
| 1307 |
outputlist = []; |
| 1308 |
break; |
| 1309 |
} |
| 1310 |
} |
| 1311 |
if (check) outputlist = currentlist; |
| 1312 |
break; |
| 1313 |
} |
| 1314 |
|
| 1315 |
for (const arg of op.args) { |
| 1316 |
if ([null, '', '=', '~'].includes((arg.subop || '').replace(/-/, '')) && dtype(arg.value, 'string') && arg.value.startsWith('~')) { |
| 1317 |
const flags = arg.value.startsWith('~~') ? 'i' : ''; |
| 1318 |
const pattern = arg.value.replace(/^~*/, ''); |
| 1319 |
const fullpattern = (arg.subop || '').replace(/-/, '') == '=' ? ((pattern.startsWith('^') ? '' : '^') + pattern + (pattern.endsWith('$') ? '' : '$')) : pattern; |
| 1320 |
arg.re = new RegExp(fullpattern, flags); |
| 1321 |
} |
| 1322 |
} |
| 1323 |
|
| 1324 |
const ands = [[]]; |
| 1325 |
for (const arg of op.args) { |
| 1326 |
if (arg.separator == ';') { |
| 1327 |
ands.push([arg]); |
| 1328 |
} else { |
| 1329 |
ands[ands.length - 1].push(arg) |
| 1330 |
} |
| 1331 |
} |
| 1332 |
const filtertimer = {}; |
| 1333 |
outputlist = currentlist.filter((item, i) => { |
| 1334 |
this.timecheck(filtertimer, i, currentlist.length, op); |
| 1335 |
return ands.filter((and) => { |
| 1336 |
return and.filter((arg) => { |
| 1337 |
let comparator = arg.subop; |
| 1338 |
let polarize = (x) => x; |
| 1339 |
if (comparator?.startsWith('-') || comparator?.endsWith('-')) { |
| 1340 |
polarize = (x) => !x; |
| 1341 |
comparator = comparator.replace(/-|-$/, ''); |
| 1342 |
} |
| 1343 |
if (!comparator && arg.label == null && Array.isArray(arg.value)) { |
| 1344 |
return polarize(this.execute([item], arg.value, labeled).length > 0); |
| 1345 |
} else if (['+', ''].includes(comparator) && arg.label && !arg.value && dtype(arg.label, 'string') && dtype(item, 'object')) { |
| 1346 |
const propval = item[arg.label]; |
| 1347 |
// console.log({plusminus: comparator, label: arg.label, propval: propval, judgment: polarize(Array.isArray(propval) ? propval.length > 0 : propval)}) |
| 1348 |
return polarize(Array.isArray(propval) ? propval.length > 0 : propval); |
| 1349 |
} |
| 1350 |
|
| 1351 |
let testitems = [item]; |
| 1352 |
if (arg.label && dtype(item, 'object')) { |
| 1353 |
testitems = step(item, arg.label, null, labeled); |
| 1354 |
} |
| 1355 |
|
| 1356 |
const testvals = testitems.map((testitem) => { |
| 1357 |
if (comparator || arg.re) { |
| 1358 |
return getname(testitem); |
| 1359 |
} else if (this.dtype(testitem, 'literal')) { |
| 1360 |
return testitem; |
| 1361 |
} else if ('id' in testitem) { |
| 1362 |
return testitem.id; |
| 1363 |
} else { |
| 1364 |
return getname(testitem) |
| 1365 |
} |
| 1366 |
}); |
| 1367 |
|
| 1368 |
let argvals; |
| 1369 |
if (Array.isArray(arg.value) || arg.value?.startsWith('=')) { |
| 1370 |
const argvalitems = this.step(item, arg.value, null, labeled); |
| 1371 |
argvals = argvalitems.map((argvalitem) => { |
| 1372 |
let argval; |
| 1373 |
if (dtype(argvalitem, 'literal')) { |
| 1374 |
argval = argvalitem; |
| 1375 |
} else { |
| 1376 |
argval = getname(argvalitem); |
| 1377 |
} |
| 1378 |
if (argval == null) { |
| 1379 |
const argvalitemkeys = Object.keys(argvalitem).filter((key) => key != 'id'); |
| 1380 |
if (argvalitemkeys.length == 1) { |
| 1381 |
argval = argvalitem[argvalitemkeys[0]]; |
| 1382 |
} |
| 1383 |
} |
| 1384 |
return argval; |
| 1385 |
}); |
| 1386 |
} |
| 1387 |
|
| 1388 |
return testvals.find((testval) => { |
| 1389 |
if (!argvals) { |
| 1390 |
if (dtype(testval, 'number') && dtype(arg.value, 'number')) { |
| 1391 |
testval = Number(testval); |
| 1392 |
argvals = [Number(arg.value)]; |
| 1393 |
} else { |
| 1394 |
if (isNaN(testval) || !isFinite(testval)) testval = testval.toString(); |
| 1395 |
argvals = [arg.value]; |
| 1396 |
} |
| 1397 |
} |
| 1398 |
if (comparator?.startsWith('@')) { |
| 1399 |
if (dtype(arg.value, 'number')) { |
| 1400 |
argvals = [Number(arg.value)]; |
| 1401 |
} else if (arg.value in labeled) { |
| 1402 |
argvals = Number(labeled[arg.value]); |
| 1403 |
if (!Array.isArray(argvals)) argvals = [argvals]; |
| 1404 |
} |
| 1405 |
if (comparator.startsWith('@@')) { |
| 1406 |
testval = currentlist.length - i; |
| 1407 |
comparator = comparator.slice(2); |
| 1408 |
} else { |
| 1409 |
testval = i + 1; |
| 1410 |
comparator = comparator.slice(1); |
| 1411 |
} |
| 1412 |
} |
| 1413 |
if (testval == null || argvals == null || argvals.length == 0) return polarize(false); |
| 1414 |
return argvals.find((argval) => { |
| 1415 |
if (arg.re) { |
| 1416 |
return polarize(testval.match(arg.re)); |
| 1417 |
} else { |
| 1418 |
switch (comparator) { |
| 1419 |
case null: |
| 1420 |
return polarize(testval == argval); |
| 1421 |
case '': |
| 1422 |
return polarize(testval == argval); |
| 1423 |
case '=': |
| 1424 |
return polarize(testval == argval); |
| 1425 |
case '>=': |
| 1426 |
return polarize(testval >= argval); |
| 1427 |
case '<=': |
| 1428 |
return polarize(testval <= argval); |
| 1429 |
case '>': |
| 1430 |
return polarize(testval > argval); |
| 1431 |
case '<': |
| 1432 |
return polarize(testval < argval); |
| 1433 |
case '~': |
| 1434 |
return polarize(testval?.toString().toLowerCase().includes(argval?.toString().toLowerCase())); |
| 1435 |
case '~<': |
| 1436 |
return polarize(testval?.toString().toLowerCase().startsWith(argval?.toString().toLowerCase())); |
| 1437 |
case '~>': |
| 1438 |
return polarize(testval?.toString().toLowerCase().endsWith(argval?.toString().toLowerCase())); |
| 1439 |
} |
| 1440 |
} |
| 1441 |
}) != null; |
| 1442 |
}) != null; |
| 1443 |
}).length > 0; |
| 1444 |
}).length == ands.length; |
| 1445 |
}); |
| 1446 |
break; |
| 1447 |
case '#': // sort |
| 1448 |
const sortargs = op.args.slice(0); |
| 1449 |
const lastarg = sortargs[sortargs.length - 1]; |
| 1450 |
let temped = false; |
| 1451 |
if (!lastarg || ![';', '=;'].includes(Object.values(lastarg).join(''))) sortargs.push(...[{ |
| 1452 |
subop: null, |
| 1453 |
value: 'name' |
| 1454 |
}, {subop: null, value: 'id'}]); |
| 1455 |
if (!dtype(currentlist[0], 'object') && lastarg && lastarg.separator == ';') { |
| 1456 |
currentlist = currentlist.map((v) => ({_value: v})); |
| 1457 |
temped = true; |
| 1458 |
} else if (sortargs[0]?.label) { |
| 1459 |
currentlist = currentlist.map((i) => dcopy(i)); |
| 1460 |
} |
| 1461 |
sortargs.forEach((arg) => { |
| 1462 |
arg.extraindex = {}; |
| 1463 |
const stablesort = (arg.label == null && arg.value == null && arg.separator == ';') ? (arg.subop == '-' ? -1 : 1) : null; |
| 1464 |
if (arg.subop?.includes('~')) { |
| 1465 |
arg.sortmode = 'literal'; |
| 1466 |
} else if (arg.subop?.endsWith('-') || arg.subop?.endsWith('>')) { |
| 1467 |
arg.sortmode = 'numeric'; |
| 1468 |
} else if (arg.subop?.endsWith('+') || arg.subop?.endsWith('<')) { |
| 1469 |
arg.sortmode = 'rank'; |
| 1470 |
} else { |
| 1471 |
arg.sortmode = stablesort || ['rank', 'index', 'number', 'id'].includes(arg.value) ? 'rank' : 'numeric'; |
| 1472 |
} |
| 1473 |
const vals = new Set(); |
| 1474 |
const extradone = new Set(); |
| 1475 |
for (let ix = 0; ix < currentlist.length; ix++) { |
| 1476 |
const item = currentlist[ix]; |
| 1477 |
const vallist = stablesort ? [ix] : (arg.value ? step(item, arg.value, null, labeled) : (Array.isArray(item) ? item : [item])); |
| 1478 |
if (vallist && !stablesort) vallist.forEach((val) => vals.add(val)); |
| 1479 |
if (typeof item == 'object') { |
| 1480 |
(item._sortindex ||= []).push(vallist); |
| 1481 |
} else if (!dtype(item, 'number') && !extradone.has(item)) { |
| 1482 |
arg.extraindex[item] = vallist; |
| 1483 |
extradone.add(item); |
| 1484 |
} |
| 1485 |
} |
| 1486 |
; |
| 1487 |
if (arg.sortmode != 'literal' && !arg.subop?.endsWith('>') && !arg.subop?.endsWith('<')) { |
| 1488 |
for (const val of vals) { |
| 1489 |
if (val != null && !dtype(val, 'number') && !(dtype(val, 'object') && dtype(getname(val), 'number'))) { |
| 1490 |
arg.sortmode = null; |
| 1491 |
break; |
| 1492 |
} |
| 1493 |
} |
| 1494 |
; |
| 1495 |
} |
| 1496 |
}) |
| 1497 |
|
| 1498 |
outputlist = currentlist.sort((a, b) => { |
| 1499 |
let comp = 0; |
| 1500 |
let ii = 0; |
| 1501 |
for (const arg of sortargs) { |
| 1502 |
const alist = typeof a == 'object' ? a._sortindex[ii] : dtype(a, 'number') ? [Number(a)] : arg.extraindex[a]; |
| 1503 |
const blist = typeof b == 'object' ? b._sortindex[ii] : dtype(b, 'number') ? [Number(b)] : arg.extraindex[b]; |
| 1504 |
ii++; |
| 1505 |
if (alist && blist) { |
| 1506 |
if (arg?.subop?.includes('@')) { |
| 1507 |
const ifunc = arg.subop == '@=' ? getname : getid; |
| 1508 |
const alookup = alist.indexOf(ifunc(a)); |
| 1509 |
const blookup = blist.indexOf(ifunc(b)); |
| 1510 |
if (alookup > -1 && blookup == -1) { |
| 1511 |
comp = -1; |
| 1512 |
} else if (alookup == -1 && blookup > -1) { |
| 1513 |
comp = 1; |
| 1514 |
} else { |
| 1515 |
const aliststrs = alist.map((ax) => ax.toString()); |
| 1516 |
comp = aliststrs.indexOf(ifunc(a).toString()) - aliststrs.indexOf(ifunc(b).toString()); |
| 1517 |
} |
| 1518 |
} else { |
| 1519 |
for (let i = 0; i < Math.min(alist.length, blist.length); i++) { |
| 1520 |
let aitem = alist[i]; |
| 1521 |
let bitem = blist[i]; |
| 1522 |
const anull = nullish(aitem); |
| 1523 |
const bnull = nullish(bitem); |
| 1524 |
if (anull && bnull) { |
| 1525 |
comp = 0; |
| 1526 |
} else if (bnull) { |
| 1527 |
comp = -1; |
| 1528 |
} else if (anull && !bnull) { |
| 1529 |
comp = 1; |
| 1530 |
} else { |
| 1531 |
switch (arg.sortmode) { |
| 1532 |
case 'literal': |
| 1533 |
const atrim = typeof aitem == 'string' ? aitem.trim() : aitem; |
| 1534 |
const btrim = typeof bitem == 'string' ? bitem.trim() : bitem; |
| 1535 |
comp = atrim < btrim ? -1 : (btrim < atrim ? 1 : 0); |
| 1536 |
break; |
| 1537 |
case 'numeric': |
| 1538 |
case 'rank': |
| 1539 |
const polarity = (arg.sortmode == 'numeric' ? -1 : 1); |
| 1540 |
let bnum = Number(bitem); |
| 1541 |
let anum = Number(aitem); |
| 1542 |
if (isNaN(bnum) || isNaN(anum)) { |
| 1543 |
bnum = Number(getname(bitem)); |
| 1544 |
anum = Number(getname(aitem)); |
| 1545 |
} |
| 1546 |
if (isNaN(bnum) && isNaN(anum)) { |
| 1547 |
comp = polarity * compvals(aitem, bitem); |
| 1548 |
} else if (isNaN(anum)) { |
| 1549 |
comp = 1; |
| 1550 |
} else if (isNaN(bnum)) { |
| 1551 |
comp = -1; |
| 1552 |
} else { |
| 1553 |
comp = polarity * (anum - bnum); |
| 1554 |
} |
| 1555 |
break; |
| 1556 |
default: |
| 1557 |
if (dtype(aitem, 'literal') && dtype(bitem, 'literal')) { |
| 1558 |
comp = compvals(aitem, bitem); |
| 1559 |
} else if (typeof aitem == 'object' && typeof bitem == 'object') { |
| 1560 |
let aval = getname(aitem); |
| 1561 |
let bval = getname(bitem); |
| 1562 |
if (!nullish(aval) && !nullish(bval)) { |
| 1563 |
comp = compvals(aval, bval); |
| 1564 |
} else { |
| 1565 |
aval = getid(aitem); |
| 1566 |
bval = getid(bitem); |
| 1567 |
if (!nullish(aval) && !nullish(bval)) { |
| 1568 |
comp = compvals(aval, bval); |
| 1569 |
} else { |
| 1570 |
comp = 0; |
| 1571 |
} |
| 1572 |
} |
| 1573 |
} |
| 1574 |
break; |
| 1575 |
} |
| 1576 |
} |
| 1577 |
if (comp != 0) break; |
| 1578 |
} |
| 1579 |
} |
| 1580 |
} |
| 1581 |
if (comp === 0) { |
| 1582 |
if (alist && blist) { |
| 1583 |
comp = blist.length - alist.length; |
| 1584 |
} else if (alist) { |
| 1585 |
comp = -1; |
| 1586 |
} else if (blist) { |
| 1587 |
comp = 1; |
| 1588 |
} |
| 1589 |
} |
| 1590 |
if (arg?.subop?.includes('-') && (arg.sortmode == 'literal' || !arg.sortmode)) { |
| 1591 |
comp = -comp; |
| 1592 |
} |
| 1593 |
if (comp != 0) break; |
| 1594 |
} |
| 1595 |
return comp; |
| 1596 |
}) |
| 1597 |
if (temped) outputlist = outputlist.map((vt) => vt._value); |
| 1598 |
outputlist.forEach((item, i) => { |
| 1599 |
delete item._sortindex; |
| 1600 |
if (op.args.length > 0 && op.args[0].label) item[op.args[0].label] = i + 1; |
| 1601 |
}); |
| 1602 |
break; |
| 1603 |
case '/': // group |
| 1604 |
case '//': // merge |
| 1605 |
const groupindex = new Map(); |
| 1606 |
let keylists = {}; |
| 1607 |
let ofname = 'of'; |
| 1608 |
let countname = 'count'; |
| 1609 |
let sortgroups = true; |
| 1610 |
const groupargs = []; |
| 1611 |
const accumulates = {}; |
| 1612 |
const discards = new Set(); |
| 1613 |
const merge_ands = []; |
| 1614 |
for (const arg of op.args) { |
| 1615 |
if (op.operator == '//' && arg.subop?.includes('+') && typeof arg.value == 'string') { |
| 1616 |
accumulates[arg.value] = arg.label || arg.value; |
| 1617 |
} else if (op.operator == '//' && arg.subop?.includes('-') && typeof arg.value == 'string') { |
| 1618 |
discards.add(arg.value); |
| 1619 |
} else if (arg.separator == ';' && arg.label == null && arg.value == null) { |
| 1620 |
sortgroups = false; |
| 1621 |
} else if (op.operator == '//' && (arg.separator == ';' || merge_ands.length > 0)) { |
| 1622 |
if (arg.separator == ';') { |
| 1623 |
merge_ands.push([arg.value]); |
| 1624 |
} else { |
| 1625 |
merge_ands[merge_ands.length - 1].push(arg.value) |
| 1626 |
} |
| 1627 |
} else { |
| 1628 |
groupargs.push(arg); |
| 1629 |
} |
| 1630 |
} |
| 1631 |
if (groupargs.length == 0) groupargs.push({value: null}); |
| 1632 |
|
| 1633 |
groupargs.forEach((arg) => arg.groupcounter = 0); |
| 1634 |
|
| 1635 |
const grouptimer = {}; |
| 1636 |
for (let ix = 0; ix < currentlist.length; ix++) { |
| 1637 |
this.timecheck(grouptimer, ix, currentlist.length, op); |
| 1638 |
const item = currentlist[ix]; |
| 1639 |
let keys = null; |
| 1640 |
let keyi = 0; |
| 1641 |
let itemsleft = currentlist.length - ix; |
| 1642 |
for (const arg of groupargs) { |
| 1643 |
if (arg.label == 'of') { |
| 1644 |
ofname = arg.value; |
| 1645 |
continue; |
| 1646 |
} else if (arg.label == 'count') { |
| 1647 |
countname = arg.value; |
| 1648 |
continue; |
| 1649 |
} |
| 1650 |
keyi++; |
| 1651 |
let groupnumber = null; |
| 1652 |
if (op.operator == '/' && dtype(arg.value, 'number')) { |
| 1653 |
if (arg.subop?.endsWith('@')) { |
| 1654 |
arg.divisor = Number(arg.value); |
| 1655 |
} else { |
| 1656 |
arg.divisor = currentlist.length / Number(arg.value); |
| 1657 |
} |
| 1658 |
groupnumber = Math.floor(ix / arg.divisor) + 1; |
| 1659 |
} else if (arg.value != null && arg.subop?.endsWith('@@')) { |
| 1660 |
const groupval = step(item, arg.value, null, labeled); |
| 1661 |
if (ix == 0 || groupval?.length > 0) arg.groupcounter += 1; |
| 1662 |
groupnumber = arg.groupcounter; |
| 1663 |
} |
| 1664 |
const label = arg.label ?? (typeof arg.value === 'string' ? arg.value : null) ?? keyi; |
| 1665 |
const newkeyitems = groupnumber != null ? [groupnumber] : arg.value ? step(item, arg.value, arg.subop, labeled) : [null]; |
| 1666 |
const newkeys = newkeyitems.map((newkey) => ([{ |
| 1667 |
arglabel: arg.label, |
| 1668 |
label: label, |
| 1669 |
keyitem: this.resolve(arg.value, newkey, labeled) |
| 1670 |
}])); |
| 1671 |
if (arg.subop?.includes('~') && Array.isArray(arg.value)) newkeys.forEach((newkey) => newkey[0].keyitem = getname(newkey[0].keyitem)); |
| 1672 |
if (keys) { |
| 1673 |
keys = keys.flatMap((oldkeys) => newkeys.filter((newkey) => arg.separator == ',' || oldkeys.filter((oldkey) => compvals(getname(oldkey.keyitem) || oldkey.keyitem, getname(newkey[0].keyitem) || newkey[0].keyitem) >= 0).length === 0).map((newkey) => oldkeys.concat(newkey))); |
| 1674 |
} else { |
| 1675 |
keys = newkeys; |
| 1676 |
} |
| 1677 |
if (arg.subop?.endsWith('@') || dtype(arg.value, 'number')) { |
| 1678 |
keys.forEach((key) => { |
| 1679 |
const testkey = JSON.stringify(key.slice(0, -1)); |
| 1680 |
const newkeytest = JSON.stringify(key[key.length - 1]); |
| 1681 |
if (testkey in keylists) { |
| 1682 |
const lastkey = keylists[testkey][keylists[testkey].length - 1]; |
| 1683 |
if (newkeytest != lastkey) keylists[testkey].push(newkeytest); |
| 1684 |
} else { |
| 1685 |
keylists[testkey] = [newkeytest]; |
| 1686 |
} |
| 1687 |
key[key.length - 1].keyindex = keylists[testkey].length; |
| 1688 |
}); |
| 1689 |
} |
| 1690 |
} |
| 1691 |
if (keys) { |
| 1692 |
for (const key of keys) { |
| 1693 |
const keystr = JSON.stringify(key); |
| 1694 |
if (!groupindex.has(keystr)) groupindex.set(keystr, []); |
| 1695 |
groupindex.get(keystr).push(item); |
| 1696 |
} |
| 1697 |
} |
| 1698 |
} |
| 1699 |
for (const [keystr, items] of groupindex) { |
| 1700 |
if (op.operator == '/') { |
| 1701 |
const newgroup = {}; |
| 1702 |
const keydata = JSON.parse(keystr); |
| 1703 |
if (keydata.length == 1 && keydata[0].keyitem != null) { |
| 1704 |
if (keydata[0].keyindex) { |
| 1705 |
newgroup.keyindex = keydata[0].keyindex; |
| 1706 |
} else { |
| 1707 |
let keyitemname = getname(keydata[0].keyitem); |
| 1708 |
if (keyitemname != null) { |
| 1709 |
newgroup.name = keyitemname; |
| 1710 |
} |
| 1711 |
} |
| 1712 |
} |
| 1713 |
let skip = false; |
| 1714 |
let keys = []; |
| 1715 |
for (const {arglabel, label, keyitem, keyindex} of keydata) { |
| 1716 |
if (keyitem != null) { |
| 1717 |
const keyobj = (dtype(keyitem, 'object') || keyindex == null || keyindex == undefined) ? keyitem : (keyindex != null && keyindex != undefined) ? {} : {name: keyitem}; |
| 1718 |
if (keyindex != null) { |
| 1719 |
keyobj.keyindex = keyindex; |
| 1720 |
} |
| 1721 |
keys.push(keyobj) |
| 1722 |
if (label && dtype(label, 'string') && isNaN(label) && label != '_') { |
| 1723 |
newgroup[label] = [keyobj]; |
| 1724 |
} |
| 1725 |
} |
| 1726 |
} |
| 1727 |
newgroup[countname] = items.length; |
| 1728 |
if (keys) newgroup.key = keys; |
| 1729 |
newgroup[ofname] = items; |
| 1730 |
outputlist.push(newgroup); |
| 1731 |
} else if (op.operator == '//') { |
| 1732 |
const groupprops = groupargs.map((grouparg) => grouparg.value).filter((gp) => gp); |
| 1733 |
if (merge_ands.length == 0 || !merge_ands.find((mand) => !mand.find((mr) => items.find((item) => mr in item)))) { |
| 1734 |
const newgroup = items.reduce((acc, item) => { |
| 1735 |
const proporder = groupprops.slice(0); |
| 1736 |
Object.keys(item).forEach((prop) => { |
| 1737 |
if (!proporder.includes(prop) && !discards.has(prop)) proporder.push(prop); |
| 1738 |
}); |
| 1739 |
for (const prop of proporder) { |
| 1740 |
const writeprop = accumulates[prop] || prop; |
| 1741 |
if (writeprop in acc) { |
| 1742 |
if (Array.isArray(acc[writeprop])) { |
| 1743 |
(Array.isArray(item[prop]) ? item[prop] : [item[prop]]).filter((val) => !acc[writeprop].includes(val)).forEach((val) => acc[writeprop].push(val)); |
| 1744 |
} |
| 1745 |
} else { |
| 1746 |
if (prop in accumulates && !Array.isArray(item[prop])) { |
| 1747 |
acc[writeprop] = [item[prop]]; |
| 1748 |
} else { |
| 1749 |
acc[writeprop] = item[prop]; |
| 1750 |
} |
| 1751 |
} |
| 1752 |
} |
| 1753 |
return acc; |
| 1754 |
}, {}); |
| 1755 |
const keydata = JSON.parse(keystr); |
| 1756 |
for (const {arglabel, label, keyitem, keyindex} of keydata) { |
| 1757 |
if (arglabel && typeof arglabel === 'string' && arglabel != '_') { |
| 1758 |
newgroup[arglabel] = [keyitem]; |
| 1759 |
} |
| 1760 |
} |
| 1761 |
outputlist.push(newgroup); |
| 1762 |
} |
| 1763 |
} |
| 1764 |
} |
| 1765 |
if (sortgroups) { |
| 1766 |
const sortquery = '#' + groupargs.map((arg, argx) => (arg.subop?.endsWith('@') || arg.divisor ? '+' : '') + '(..key:@' + (argx + 1) + (arg.subop?.endsWith('@') ? '.keyindex;_' : '') + ')').join(','); |
| 1767 |
outputlist = this.execute(outputlist, this.assemble(this.tokenize(sortquery)), labeled); |
| 1768 |
} |
| 1769 |
break; |
| 1770 |
case '...': // synthesize |
| 1771 |
case '....': // synthesize and extract |
| 1772 |
if (!(op?.args?.length > 0)) { |
| 1773 |
if (op.operator == '...') { |
| 1774 |
outputlist = [{of: currentlist.map((item) => dcopy(item))}]; |
| 1775 |
} else { |
| 1776 |
outputlist = [currentlist.length] |
| 1777 |
} |
| 1778 |
break; |
| 1779 |
} |
| 1780 |
const segments = [[]]; |
| 1781 |
op.args.forEach((arg) => { |
| 1782 |
if (arg.separator == ';') segments.push([]); |
| 1783 |
segments[segments.length - 1].push(arg); |
| 1784 |
}); |
| 1785 |
outputlist = []; |
| 1786 |
let labels = null; |
| 1787 |
let masterof = op.operator == '...'; |
| 1788 |
if ( |
| 1789 |
segments.length > 1 && |
| 1790 |
segments[0].filter((s) => s.subop && s.value == null).length == segments[0].length && |
| 1791 |
segments.slice(1).filter((s) => s.length == segments[0].length || (segments[0][segments[0].length - 1].label == null && s.length == segments[0].length - 1)).length == segments.length - 1 |
| 1792 |
) { |
| 1793 |
labels = segments.shift(); |
| 1794 |
const lastlabel = labels[labels.length - 1]; |
| 1795 |
if (lastlabel.subop == '~' && lastlabel.label == null) { |
| 1796 |
masterof = false; |
| 1797 |
labels.pop(); |
| 1798 |
} |
| 1799 |
} |
| 1800 |
for (const segment of segments) { |
| 1801 |
const tempitem = {of: currentlist.map((item) => dcopy(item))}; |
| 1802 |
const finalitem = {}; |
| 1803 |
const postitem = {}; |
| 1804 |
let aggregated = null; |
| 1805 |
let afteraggregated = 0; |
| 1806 |
let finalvalue = null; |
| 1807 |
let includeof = masterof; |
| 1808 |
segment.forEach((arg, propx) => { |
| 1809 |
if (labels?.[propx]) { |
| 1810 |
arg.label = labels[propx].label; |
| 1811 |
arg.subop = labels[propx].subop; |
| 1812 |
} |
| 1813 |
if (arg.subop == '~' && arg.label == null && arg.value == null) { |
| 1814 |
includeof = false |
| 1815 |
} else if (!aggregated && arg.subop?.includes('~') && dtype(arg.value, 'string')) { |
| 1816 |
tempitem[arg.label || '_' + (propx + 1).toString()] = arg.value; |
| 1817 |
} else if (!aggregated && arg.subop?.includes('~') && Array.isArray(arg.value)) { |
| 1818 |
const subresult = this.execute(currentlist, arg.value.map(opclone), labeled); |
| 1819 |
tempitem[arg.label || '_' + (propx + 1).toString()] = subresult.length > 0 ? getname(subresult[0]) : null; |
| 1820 |
} else { |
| 1821 |
const prop = arg.label || (typeof arg.value === 'string' ? arg.value : null) || '_' + (propx + 1).toString(); |
| 1822 |
const aggname = (arg.label == null && typeof arg.value == 'string' && arg.value) || (arg.label && arg.subop == null && arg.value == null); |
| 1823 |
const firstitem = currentlist[0]; |
| 1824 |
const isprop = typeof firstitem == 'object' && prop in firstitem; |
| 1825 |
if ((arg.subop == '=' || !isprop) && arg.subop != '~' && aggname in this.annotators) { |
| 1826 |
const aggval = this.annotators[aggname](tempitem); |
| 1827 |
const agglabel = arg.label || aggname; |
| 1828 |
if (aggval != null) finalitem[agglabel] = aggval; |
| 1829 |
aggregated = agglabel; |
| 1830 |
finalvalue = dcopy(aggval); |
| 1831 |
} else if (typeof arg.value == 'string' && arg.value.startsWith('~~')) { |
| 1832 |
let template = arg.value.slice(2); |
| 1833 |
const extractionfield = arg.label || template; |
| 1834 |
const targetfields = this.kkeys(tempitem); |
| 1835 |
const fieldmap = {}; |
| 1836 |
template = this.escapeRegExp(template); |
| 1837 |
let tx = 1; |
| 1838 |
targetfields.forEach((tf) => { |
| 1839 |
const fieldwidth = this.getnumber(tempitem[tf]); |
| 1840 |
const innerpattern = fieldwidth == 1 ? '.' : fieldwidth ? `.{${fieldwidth}}` : '.+?'; |
| 1841 |
if (tf == '_') { |
| 1842 |
template = template.replaceAll('_', `(?:${innerpattern})`); |
| 1843 |
} else { |
| 1844 |
const tkey = 'x' + tx; |
| 1845 |
fieldmap[tf] = tkey; |
| 1846 |
template = template.replace(this.escapeRegExp(tf), `(?<${tkey}>${innerpattern})`); |
| 1847 |
tx += 1; |
| 1848 |
} |
| 1849 |
}); |
| 1850 |
template = `^${template}$`; |
| 1851 |
const extractor = new RegExp(template); |
| 1852 |
finalitem[extractionfield] = []; |
| 1853 |
finalitem['_template'] = template; |
| 1854 |
finalitem['_field map'] = Object.entries(fieldmap).map(([k, v]) => ({ |
| 1855 |
field: k, |
| 1856 |
code: v |
| 1857 |
})); |
| 1858 |
tempitem.of.forEach((i) => { |
| 1859 |
const extraction = getname(i).match(extractor); |
| 1860 |
if (extraction) { |
| 1861 |
const extractitem = {}; |
| 1862 |
targetfields.forEach((k) => { |
| 1863 |
extractitem[k.trim()] = extraction.groups[fieldmap[k]]; |
| 1864 |
}); |
| 1865 |
finalitem[extractionfield].push(extractitem); |
| 1866 |
} |
| 1867 |
}); |
| 1868 |
finalvalue = finalitem[extractionfield]; |
| 1869 |
aggregated = template; |
| 1870 |
} else if (!aggregated) { |
| 1871 |
let propitems; |
| 1872 |
if (Array.isArray(arg.value) || !arg.label || arg.value?.startsWith('~')) { |
| 1873 |
const propres = this.execute(tempitem.of, Array.isArray(arg.value) ? arg.value.map(opclone) : [{ |
| 1874 |
operator: '..', |
| 1875 |
args: [{value: arg.value}] |
| 1876 |
}], labeled); |
| 1877 |
if (arg.subop?.includes('~') && propres.length > 0) { |
| 1878 |
propitems = getname(propres[0]); |
| 1879 |
} else { |
| 1880 |
propitems = propres; |
| 1881 |
} |
| 1882 |
} else if (arg.value?.startsWith('=')) { |
| 1883 |
const mathres = this.step(tempitem.of, arg.value, arg.subop, labeled); |
| 1884 |
if (mathres.length > 0) propitems = mathres[0]; |
| 1885 |
} else { |
| 1886 |
if (arg.subop?.includes('~')) { |
| 1887 |
propitems = arg.value; |
| 1888 |
} else { |
| 1889 |
propitems = [arg.value] |
| 1890 |
} |
| 1891 |
} |
| 1892 |
tempitem[prop] = propitems; |
| 1893 |
} else if (aggregated in finalitem && Array.isArray(finalitem[aggregated]) && finalitem[aggregated].length > afteraggregated) { |
| 1894 |
const propval = finalitem[aggregated][afteraggregated]; |
| 1895 |
const mappedval = arg.subop?.endsWith('~') ? propval : [propval]; |
| 1896 |
tempitem[prop] = mappedval; |
| 1897 |
postitem[prop] = mappedval; |
| 1898 |
afteraggregated++; |
| 1899 |
} |
| 1900 |
} |
| 1901 |
}); |
| 1902 |
if (op.operator == '....' && afteraggregated > 0) { |
| 1903 |
outputlist.push(postitem); |
| 1904 |
} else if (op.operator == '....' && finalvalue != null) { |
| 1905 |
if (dtype(finalvalue, 'array')) { |
| 1906 |
outputlist = outputlist.concat(finalvalue); |
| 1907 |
} else { |
| 1908 |
outputlist.push(finalvalue); |
| 1909 |
} |
| 1910 |
} else { |
| 1911 |
for (const prop in tempitem) { |
| 1912 |
if (!(prop in finalitem) && prop != 'of') finalitem[prop] = tempitem[prop]; |
| 1913 |
} |
| 1914 |
if (includeof) finalitem.of = tempitem.of; |
| 1915 |
outputlist.push(finalitem); |
| 1916 |
} |
| 1917 |
} |
| 1918 |
break; |
| 1919 |
case '|': // annotate |
| 1920 |
outputlist = currentlist.slice(0).map((item) => dcopy(item)); |
| 1921 |
op.args.filter((arg) => arg.subop?.includes('>') && arg.label != null && arg.label != undefined && Array.isArray(arg.value)) |
| 1922 |
.forEach((arg) => (labeled['=>'] ??= {})[arg.label] = arg.value); |
| 1923 |
|
| 1924 |
const annotatetimer = {}; |
| 1925 |
op.args.filter((arg) => arg.subop?.endsWith('@')).forEach((arg) => arg.counter = undefined); |
| 1926 |
for (let i = 0; i < outputlist.length; i++) { |
| 1927 |
this.timecheck(annotatetimer, i, outputlist.length, op); |
| 1928 |
const baseitem = outputlist[i]; |
| 1929 |
if (typeof baseitem != 'object') { |
| 1930 |
outputlist[i] = {}; |
| 1931 |
if (op.args?.[0]?.value != '_') outputlist[i].name = baseitem; |
| 1932 |
} |
| 1933 |
const item = outputlist[i]; |
| 1934 |
const newprops = op.args.filter((arg) => arg.subop != '<' && (arg.label || !arg.subop?.includes('-'))).map((arg) => arg.label || arg.value); |
| 1935 |
if (typeof item == 'object') { |
| 1936 |
let argx = -1; |
| 1937 |
for (const arg of op.args) { |
| 1938 |
argx++; |
| 1939 |
if (arg?.subop == '<' && dtype(arg.value, 'string') && argx < op.args.length - 1) { |
| 1940 |
if (arg.value in item && (Array.isArray(item[arg.value]) || dtype(item[arg.value], 'object'))) { |
| 1941 |
const sublabeled = {...labeled}; |
| 1942 |
if (arg.label) sublabeled[arg.label] = [item]; |
| 1943 |
item[arg.value] = this.execute(item[arg.value], [{ |
| 1944 |
operator: op.operator, |
| 1945 |
args: op.args.slice(argx + 1).map(opclone) |
| 1946 |
}], sublabeled); |
| 1947 |
} |
| 1948 |
break; |
| 1949 |
} else if (arg.separator == ';' && argx == op.args.length - 1 && arg.label == null && arg.value == null && (arg.subop == null || arg.subop == '~')) { |
| 1950 |
for (const oldprop in item) { |
| 1951 |
if (!(newprops.includes(oldprop))) { |
| 1952 |
const tempval = item[oldprop]; |
| 1953 |
delete item[oldprop]; |
| 1954 |
if (arg.subop != '~') item[oldprop] = tempval; |
| 1955 |
} |
| 1956 |
} |
| 1957 |
} else if (arg.subop?.endsWith('-')) { |
| 1958 |
if (arg.value && typeof arg.value == 'string' && arg.value in item) { |
| 1959 |
if (arg.label && typeof arg.label == 'string') item[arg.label] = item[arg.value]; |
| 1960 |
delete item[arg.value]; |
| 1961 |
} else if (arg.label && Array.isArray(arg.value)) { |
| 1962 |
const toremoveids = new Set(step(item, arg.value, null, labeled).map((subitem) => getid(subitem))); |
| 1963 |
item[arg.label] = item[arg.label].filter((subitem) => !toremoveids.has(getid(subitem))); |
| 1964 |
} |
| 1965 |
} else if (argx == 0 && arg.label != null && arg.value == '_') { |
| 1966 |
item[arg.label] = [baseitem]; |
| 1967 |
} else { |
| 1968 |
if (!arg.label && arg.value && typeof arg.value === 'string' && arg.value != '') { |
| 1969 |
if (arg.subop == '=' && arg.value in this.annotators) { |
| 1970 |
item[arg.value] = this.annotators[arg.value](item); |
| 1971 |
} else if (arg.value in item) { |
| 1972 |
const moveprop = arg.value; |
| 1973 |
const value = item[moveprop]; |
| 1974 |
delete item[moveprop]; |
| 1975 |
item[moveprop] = value; |
| 1976 |
} else if (arg.value in this.annotators) { |
| 1977 |
item[arg.value] = this.annotators[arg.value](item); |
| 1978 |
} |
| 1979 |
} |
| 1980 |
if (arg.label) { |
| 1981 |
let vals = []; |
| 1982 |
if (arg.subop.endsWith('@')) { |
| 1983 |
if (arg.subop.endsWith('@@')) { |
| 1984 |
arg.counter ??= arg.value == null ? outputlist.length + 1 : 0; |
| 1985 |
} else { |
| 1986 |
arg.counter ??= 1; |
| 1987 |
vals = [arg.counter]; |
| 1988 |
} |
| 1989 |
if (Array.isArray(arg.value)) { |
| 1990 |
arg.counter += step(item, arg.value, null, labeled).length; |
| 1991 |
} else if (dtype(arg.value, 'string') && dtype(item[arg.value], 'number')) { |
| 1992 |
arg.counter += Number(item[arg.value]); |
| 1993 |
} else { |
| 1994 |
arg.counter += arg.subop.endsWith('@@') ? -1 : 1; |
| 1995 |
} |
| 1996 |
if (arg.subop.endsWith('@@')) { |
| 1997 |
vals = [arg.counter]; |
| 1998 |
} |
| 1999 |
} else if (arg.subop == '=' && typeof arg.value == 'string' && arg.value in this.annotators) { |
| 2000 |
vals = [this.annotators[arg.value](item)]; |
| 2001 |
} else { |
| 2002 |
vals = step(item, arg.value, arg.subop, labeled); |
| 2003 |
} |
| 2004 |
const scalar = arg.subop?.includes('~') || arg.subop?.endsWith('@') || (arg.label != '_' && arg.subop == '=' && typeof arg.value == 'string' && (arg.value in this.annotators || arg.value.startsWith('=') || arg.value.startsWith('~'))); |
| 2005 |
const base = (arg.subop?.endsWith('+') && arg.label in item && Array.isArray(item[arg.label])) ? item[arg.label] : []; |
| 2006 |
if (arg.label == '_') { |
| 2007 |
const sourceitem = dtype(vals, 'array') ? vals[0] : val; |
| 2008 |
if (dtype(sourceitem, 'object')) { |
| 2009 |
for (const prop in sourceitem) { |
| 2010 |
if (!(prop in item)) { |
| 2011 |
if (scalar && Array.isArray(sourceitem[prop])) { |
| 2012 |
item[prop] = sourceitem[prop][0]; |
| 2013 |
} else { |
| 2014 |
item[prop] = sourceitem[prop]; |
| 2015 |
} |
| 2016 |
newprops.push(prop); |
| 2017 |
} |
| 2018 |
} |
| 2019 |
} |
| 2020 |
} else if (dtype(vals, 'array')) { |
| 2021 |
if (scalar && vals.length > 0) { |
| 2022 |
item[arg.label] = getname(vals[0]); |
| 2023 |
} else { |
| 2024 |
item[arg.label] = base.concat(vals.map((v) => dcopy(v))); |
| 2025 |
} |
| 2026 |
} else if (vals) { |
| 2027 |
item[arg.label] = base.concat(vals); |
| 2028 |
} |
| 2029 |
} |
| 2030 |
} |
| 2031 |
} |
| 2032 |
} |
| 2033 |
} |
| 2034 |
break; |
| 2035 |
case '???': |
| 2036 |
outputlist = currentlist; |
| 2037 |
const commentval = op.args?.[0]?.value; |
| 2038 |
if (commentval in labeled) console.log({[commentval]: labeled[commentval].slice(0)}); |
| 2039 |
if (commentval?.startsWith('end')) { |
| 2040 |
if (this.debug && !inputlist) { |
| 2041 |
return operations; |
| 2042 |
} else if (commentval.length > 3 && !isNaN(commentval.slice(3))) { |
| 2043 |
return outputlist.slice(0, Number(commentval.slice(3))); |
| 2044 |
} else { |
| 2045 |
return outputlist; |
| 2046 |
} |
| 2047 |
} |
| 2048 |
if (commentval == 'recache') this.recache = new Set(); |
| 2049 |
if (typeof currentlist[0] == 'object' && op.args?.[0]?.value in currentlist[0] && !op.args?.[0]?.label) console.log({[op.args?.[0]?.value]: currentlist[0][op.args?.[0]?.value]}); |
| 2050 |
break; |
| 2051 |
default: |
| 2052 |
outputlist = [] |
| 2053 |
} |
| 2054 |
currentlist = outputlist.slice(0); |
| 2055 |
if (toplevel) { |
| 2056 |
if (!this.adaptive) { |
| 2057 |
op.completed = true; |
| 2058 |
operations[0].progress = currentlist; |
| 2059 |
operations[0].labeled = labeled; |
| 2060 |
} |
| 2061 |
if (this.debug) { |
| 2062 |
op.results = outputlist.slice(0, typeof this.debug == 'number' ? this.debug : outputlist.length); |
| 2063 |
op.time = (performance.now() - opstart) / 1000; |
| 2064 |
} |
| 2065 |
} |
| 2066 |
} |
| 2067 |
return (this.debug && !inputlistraw) ? operations : outputlist; |
| 2068 |
} |
| 2069 |
|
| 2070 |
gettype(value) { |
| 2071 |
if (value == null || value == undefined) return null; |
| 2072 |
const trylist = [value]; |
| 2073 |
if (this.features.plurality) { |
| 2074 |
if (value.endsWith('s')) { |
| 2075 |
trylist.push(value.slice(0, value.length - 1)); |
| 2076 |
trylist.push(value + 'es'); |
| 2077 |
} else { |
| 2078 |
trylist.push(value + 's'); |
| 2079 |
} |
| 2080 |
} |
| 2081 |
for (const tryval of trylist) { |
| 2082 |
if (tryval in this.data) { |
| 2083 |
return this.data[tryval]; |
| 2084 |
} else if (this.internal_datasets.includes(tryval)) { |
| 2085 |
return []; |
| 2086 |
} |
| 2087 |
} |
| 2088 |
if (this.savedquerynames.has(value)) { |
| 2089 |
const savedqueries = this.data.queries.filter((q) => q.name == value && !q.relative); |
| 2090 |
if (savedqueries?.length == 1) { |
| 2091 |
const sq = savedqueries[0]; |
| 2092 |
if (!sq.results) sq.results = this.executeq(sq.query); |
| 2093 |
return sq.results.slice(0); |
| 2094 |
} |
| 2095 |
} |
| 2096 |
return null; |
| 2097 |
} |
| 2098 |
|
| 2099 |
dcopy = (item) => { |
| 2100 |
if (item === null || typeof item !== 'object') return item; |
| 2101 |
if (Array.isArray(item)) return item.slice(); |
| 2102 |
return {...item}; |
| 2103 |
} |
| 2104 |
|
| 2105 |
dtype(item, test = null) { |
| 2106 |
const t = typeof item; |
| 2107 |
const type = t === 'object' ? (item === null ? null : Array.isArray(item) ? 'array' : 'object') |
| 2108 |
: (t === 'string' || t === 'number' || t === 'boolean') ? 'literal' |
| 2109 |
: null; |
| 2110 |
if (!test) return type; |
| 2111 |
if (test === 'number') return type === 'literal' && !isNaN(item); |
| 2112 |
if (test === 'string') return t === 'string'; |
| 2113 |
if (test === 'boolean') return t === 'boolean'; |
| 2114 |
return type === test; |
| 2115 |
} |
| 2116 |
|
| 2117 |
nullish(val) { |
| 2118 |
return val === false || val === null || val === undefined || val === '' || (Array.isArray(val) && val.length == 0) || (typeof val == 'object' && Object.keys(val).length == 0); |
| 2119 |
} |
| 2120 |
|
| 2121 |
getname = (item) => { |
| 2122 |
if (typeof item === 'object' && item != null) { |
| 2123 |
if (item?.name != null) { |
| 2124 |
return item.name; |
| 2125 |
} else if (item?.key?.length > 0) { |
| 2126 |
return item.key.join(' / '); |
| 2127 |
} |
| 2128 |
if (this.features.guessname) { |
| 2129 |
const {id, ...nonidprops} = item; |
| 2130 |
const names = Object.values(nonidprops).filter((val) => typeof val === 'string' || typeof val === 'number'); |
| 2131 |
if (names.length > 0) { |
| 2132 |
return names[0]; |
| 2133 |
} |
| 2134 |
for (const key in nonidprops) { |
| 2135 |
if (Array.isArray(item[key]) && item[key].length == 1 && typeof item[key][0] == 'string') { |
| 2136 |
return item[key][0]; |
| 2137 |
} |
| 2138 |
} |
| 2139 |
} |
| 2140 |
} else if (typeof item === 'string' || typeof item === 'number') { |
| 2141 |
return item; |
| 2142 |
} else if (typeof item === 'boolean') { |
| 2143 |
return item.toString(); |
| 2144 |
} |
| 2145 |
return ''; |
| 2146 |
} |
| 2147 |
|
| 2148 |
getid = (item) => { |
| 2149 |
if (typeof item === 'object' && item != null) { |
| 2150 |
const iditems = (item.id || item.uri || item.name) ? [item] : (item.key?.length > 0 && !item.keyindex) ? item.key : Object.keys(item).length == 3 && item.of?.length > 0 ? item.of : [item]; |
| 2151 |
return iditems.map((item) => { |
| 2152 |
if (item.id) { |
| 2153 |
if (Array.isArray(item.id)) { |
| 2154 |
return item.id[0]; |
| 2155 |
} else { |
| 2156 |
return item.id; |
| 2157 |
} |
| 2158 |
} else if (item.uri) { |
| 2159 |
if (Array.isArray(item.uri)) { |
| 2160 |
return item.uri[0]; |
| 2161 |
} else { |
| 2162 |
return item.uri; |
| 2163 |
} |
| 2164 |
} else if (this.features.guessid && item.name) { |
| 2165 |
return item.name; |
| 2166 |
} else { |
| 2167 |
const stringifiedid = JSON.stringify(item); |
| 2168 |
// if (stringifiedid.length > 128) console.warn({idstringify: item, idlength: stringifiedid.length}); |
| 2169 |
return stringifiedid; |
| 2170 |
} |
| 2171 |
}).join(','); |
| 2172 |
} else if (typeof item === 'string' || typeof item === 'number') { |
| 2173 |
return item; |
| 2174 |
} |
| 2175 |
return null; |
| 2176 |
} |
| 2177 |
|
| 2178 |
getnumber = (val) => { |
| 2179 |
let number = Array.isArray(val) ? val[0] : val; |
| 2180 |
number = !isNaN(number) ? Number(val) : null; |
| 2181 |
return number; |
| 2182 |
} |
| 2183 |
|
| 2184 |
opclone = (x) => { |
| 2185 |
if (x === null || typeof x !== 'object') return x; // primitives |
| 2186 |
if (Array.isArray(x)) { |
| 2187 |
const a = new Array(x.length); |
| 2188 |
for (let i = 0; i < x.length; i++) a[i] = this.opclone(x[i]); |
| 2189 |
return a; |
| 2190 |
} |
| 2191 |
if (x instanceof RegExp) return new RegExp(x.source, x.flags); // preserve the `~` matchers |
| 2192 |
const o = {}; |
| 2193 |
for (const k of Object.keys(x)) o[k] = this.opclone(x[k]); |
| 2194 |
return o; |
| 2195 |
} |
| 2196 |
|
| 2197 |
step = (item, property, subop, labeled) => { |
| 2198 |
// (this.data.trace ??= []).push({stepitem: item, property: property, subop: subop, labeled: labeled}); |
| 2199 |
const dtype = this.dtype; |
| 2200 |
const dcopy = this.dcopy; |
| 2201 |
const resolve = this.resolve; |
| 2202 |
const escapeRegExp = this.escapeRegExp; |
| 2203 |
const opclone = this.opclone; |
| 2204 |
const vals = []; |
| 2205 |
if (subop?.includes('~') && dtype(property, 'literal')) { |
| 2206 |
vals.push(property); |
| 2207 |
} else if (Array.isArray(property)) { |
| 2208 |
const res = this.execute([item], property.map(opclone), labeled).flatMap((x) => x); |
| 2209 |
if (subop?.includes('~') && res.length > 0) { |
| 2210 |
vals.push(this.getname(res[0])); |
| 2211 |
} else { |
| 2212 |
res.forEach((val) => vals.push(val)); |
| 2213 |
} |
| 2214 |
} else if (property == '_') { |
| 2215 |
vals.push(dcopy(item)); |
| 2216 |
} else if (item?.of && dtype(property, 'number')) { |
| 2217 |
const propval = Number(property); |
| 2218 |
(subop == '-' || propval < 0 ? item.of.slice(-1 * Math.abs(Number(property))) : item.of.slice(0, Math.abs(Number(property)))).forEach((val) => vals.push(val)); |
| 2219 |
} else if (property == 'id' && dtype(item, 'literal')) { |
| 2220 |
vals.push(item); |
| 2221 |
} else if (property == 'name') { |
| 2222 |
vals.push(this.getname(item)); |
| 2223 |
} else if (this.features.inlinemath && property.startsWith('=')) { |
| 2224 |
let calculation = property.slice(1); |
| 2225 |
const mathwords = Object.getOwnPropertyNames(Math).filter((mathword) => mathword.match(/^[a-z0-9]+$/)).sort((a, b) => b.length - a.length || a.localeCompare(b)); |
| 2226 |
const otherwords = ['split']; |
| 2227 |
const variables = Object.entries(item).concat(Object.entries(labeled)) |
| 2228 |
.map(([k, v]) => k) |
| 2229 |
.sort((a, b) => b.length - a.length || a.localeCompare(b)); |
| 2230 |
if (dtype(this.getname(item), 'number')) variables.push('_'); |
| 2231 |
const allowedwords = mathwords.concat(otherwords).concat(variables).sort((a, b) => b.length - a.length || a.localeCompare(b)); |
| 2232 |
const allowed = new RegExp(`^((\\b(${allowedwords.map((w) => escapeRegExp(w)).join('|')})\\b)|([0-9_\\+\\/\\*\\(\\)\\[\\]\\.%=,'" -]*))*$`); |
| 2233 |
let val; |
| 2234 |
if (calculation.match(allowed)) { |
| 2235 |
for (const variable of variables) { |
| 2236 |
const variableex = new RegExp(`\\b${escapeRegExp(variable)}\\b`, 'g'); |
| 2237 |
if (calculation.match(variableex)) { |
| 2238 |
let vval = variable == '_' ? Number(this.getname(item)) : item[variable] ?? labeled[variable]; |
| 2239 |
if (Array.isArray(vval) && vval.length == 1) vval = vval[0]; |
| 2240 |
calculation = calculation.replaceAll(variableex, ` ${dtype(vval, 'number') ? vval : JSON.stringify(vval)} `); |
| 2241 |
} |
| 2242 |
} |
| 2243 |
if (calculation.match(/[A-Za-z]/)) { |
| 2244 |
for (const mathword of mathwords) { |
| 2245 |
const mathwordex = new RegExp(`\\b${mathword}\\b`, 'g'); |
| 2246 |
calculation = calculation.replaceAll(mathwordex, `Math.${mathword}`); |
| 2247 |
if (!calculation.match(/[A-Za-z]/)) break; |
| 2248 |
} |
| 2249 |
} |
| 2250 |
calculation = calculation.replaceAll(/\b=\b/g, '=='); |
| 2251 |
try { |
| 2252 |
val = eval(calculation); |
| 2253 |
} catch (error) { |
| 2254 |
val = calculation; |
| 2255 |
} |
| 2256 |
} else { |
| 2257 |
val = calculation; |
| 2258 |
} |
| 2259 |
if (val !== false) vals.push(val); |
| 2260 |
|
| 2261 |
} else if (property.startsWith('~')) { |
| 2262 |
let val = property.slice(1); |
| 2263 |
const variables = Object.entries(item).concat(Object.entries(labeled)) |
| 2264 |
.map(([k, v]) => k) |
| 2265 |
.sort((a, b) => b.length - a.length || a.localeCompare(b)); |
| 2266 |
variables.push('_'); |
| 2267 |
for (const variable of variables) { |
| 2268 |
const variableex = new RegExp(`(?<!\w)${escapeRegExp(variable)}(?!\w)`, 'g'); |
| 2269 |
if (val.match(variableex)) { |
| 2270 |
let vval = variable == '_' ? this.getname(item) : item[variable] ?? labeled[variable]; |
| 2271 |
if (Array.isArray(vval) && vval.length == 1) vval = vval[0]; |
| 2272 |
if (!dtype(vval, 'string')) vval = this.getname(vval); |
| 2273 |
if (vval) val = val.replaceAll(variableex, vval.toString()); |
| 2274 |
} |
| 2275 |
} |
| 2276 |
if (val) vals.push(val); |
| 2277 |
} else if (item) { |
| 2278 |
let found = false; |
| 2279 |
if (property != null && !dtype(item, 'literal')) { |
| 2280 |
const tryvals = [property]; |
| 2281 |
if (this.features.plurality) { |
| 2282 |
if (!property.endsWith('s') && !this.internal_datasets.includes(property + 's')) tryvals.push(property + 's'); |
| 2283 |
if (property.endsWith('s') && !this.internal_datasets.includes(property)) tryvals.push(property.replace(/s$/, '')); |
| 2284 |
} |
| 2285 |
if (this.features.unscore && property.match(/ /)) tryvals.push(property.replaceAll(/ /g, '_')); |
| 2286 |
for (const tryval of tryvals) { |
| 2287 |
if (tryval in item) { |
| 2288 |
const resolved = resolve(tryval, item[tryval], labeled); |
| 2289 |
if (Array.isArray(resolved)) { |
| 2290 |
resolved.forEach((x) => vals.push(x)); |
| 2291 |
} else { |
| 2292 |
vals.push(resolved); |
| 2293 |
} |
| 2294 |
found = true; |
| 2295 |
break; |
| 2296 |
} |
| 2297 |
} |
| 2298 |
} |
| 2299 |
if (!found && labeled?.['=>']?.[property]) { |
| 2300 |
this.execute([item], labeled['=>'][property], labeled).forEach((val) => vals.push(val)); |
| 2301 |
found = true; |
| 2302 |
} |
| 2303 |
if (!found && (property in this.data || property in this.adapters || this.savedquerynames.has(property) || property in labeled)) { |
| 2304 |
const typenav = resolve(property, item, labeled, true); |
| 2305 |
if (typenav) { |
| 2306 |
(Array.isArray(typenav) ? typenav : [typenav]).forEach((t) => vals.push(t)); |
| 2307 |
found = true; |
| 2308 |
} |
| 2309 |
} |
| 2310 |
if (!found && property.includes('→')) { |
| 2311 |
const parts = property.split(/\s*→\s*/).filter((part) => part?.length > 0); |
| 2312 |
if (dtype(item, 'object')) { |
| 2313 |
this.step(item, parts.map((part) => ({ |
| 2314 |
operator: '..', |
| 2315 |
args: [{value: part}] |
| 2316 |
}))).forEach((val) => vals.push(val)); |
| 2317 |
} else { |
| 2318 |
const asnum = dtype(item, 'number'); |
| 2319 |
if (item == (asnum ? Number(parts[0]) : parts[0])) { |
| 2320 |
parts.slice(1).forEach((newval) => vals.push(asnum ? Number(newval) : newval)); |
| 2321 |
} else { |
| 2322 |
vals.push(item); |
| 2323 |
} |
| 2324 |
} |
| 2325 |
} |
| 2326 |
} |
| 2327 |
return vals; |
| 2328 |
} |
| 2329 |
|
| 2330 |
indexlogit = (property, type) => { |
| 2331 |
const today = new Date().toISOString().slice(0, 10); |
| 2332 |
((this.index._ ||= {})[property] ||= {})[today] ||= {read: 0, write: 0}; |
| 2333 |
this.index._[property][today][type]++; |
| 2334 |
this.index_modified.add('_'); |
| 2335 |
if (type == 'write') this.index_modified.add(property); |
| 2336 |
} |
| 2337 |
|
| 2338 |
lookup = (item, list) => { |
| 2339 |
return list.filter((li) => { |
| 2340 |
if (item.id) return item.id == li.id; |
| 2341 |
if (item.uri) return item.uri == li.uri; |
| 2342 |
for (const k of this.kkeys(item)) { |
| 2343 |
if (item[k]?.toString() != li?.[k]?.toString()) return false; |
| 2344 |
} |
| 2345 |
return true; |
| 2346 |
}); |
| 2347 |
} |
| 2348 |
|
| 2349 |
indexed = (property, key, item, func) => { |
| 2350 |
if ((!this.recache || this.recache.has(property)) && this.index[property]?.[key] !== undefined) { |
| 2351 |
this.indexlogit(property, 'read'); |
| 2352 |
return this.index[property][key]; |
| 2353 |
} |
| 2354 |
const res = func(item); |
| 2355 |
if (res) { |
| 2356 |
(this.index[property] ||= {})[key] = res; |
| 2357 |
this.indexlogit(property, 'write'); |
| 2358 |
} |
| 2359 |
return res; |
| 2360 |
} |
| 2361 |
|
| 2362 |
resolve = (property, item, labeled, navigate = false) => { |
| 2363 |
const dtype = this.dtype; |
| 2364 |
const aqueue = this.aqueue; |
| 2365 |
if (dtype(item, 'object')) { |
| 2366 |
if (this.adapters?.[property]?.annotator) { |
| 2367 |
return this.indexed(property, this.getid(item), item, () => aqueue(property, item)); |
| 2368 |
} else if (navigate) { |
| 2369 |
const res = this.indexed(property, this.getid(item), item, () => { |
| 2370 |
if (property in this.data) { |
| 2371 |
return this.lookup(item, this.data[property]); |
| 2372 |
} else if (property in labeled) { |
| 2373 |
return this.lookup(item, labeled[property]); |
| 2374 |
} else if (this.savedquerynames.has(property)) { |
| 2375 |
const saved_query = this.data.queries.find((q) => q.name == property); |
| 2376 |
if (saved_query.results && !saved_query.relative) { |
| 2377 |
return this.lookup(item, saved_query.results); |
| 2378 |
} else if (saved_query.relative) { |
| 2379 |
return this.execute([item], this.parse(saved_query.query), labeled); |
| 2380 |
} |
| 2381 |
} |
| 2382 |
return null; |
| 2383 |
}); |
| 2384 |
if (res) return res; |
| 2385 |
if ('id' in item || 'uri' in item) { |
| 2386 |
return this.resolve(property, this.getid(item), labeled); |
| 2387 |
} |
| 2388 |
return null; |
| 2389 |
} else { |
| 2390 |
return item; |
| 2391 |
} |
| 2392 |
} else if (dtype(item, 'literal') && typeof property == 'string' && (property in labeled || this.destinations.has(property) || (this.features.plurality && !this.internal_datasets.includes(property + 's') && this.destinations.has(property + 's')))) { |
| 2393 |
let dataset = false; |
| 2394 |
const res = this.indexed(property, item, item, () => { |
| 2395 |
if (!(this.features.autoresolve || navigate)) return null; |
| 2396 |
const typeitems = labeled[property] ?? this.gettype(property); |
| 2397 |
if (Array.isArray(typeitems)) { |
| 2398 |
dataset = true; |
| 2399 |
const trykeys = ['id', 'uri', 'name']; |
| 2400 |
if (this.features.plurality) { |
| 2401 |
trykeys.push(property); |
| 2402 |
trykeys.push(property.endsWith('s') ? property.slice(0, property.length - 1) : property + 's'); |
| 2403 |
} |
| 2404 |
for (const lookupkey of trykeys) { |
| 2405 |
const found = typeitems.filter((typeitem) => { |
| 2406 |
return dtype(typeitem, 'object') && (lookupkey in typeitem) && (typeitem[lookupkey].toString() == item || (Array.isArray(typeitem[lookupkey]) && typeitem[lookupkey].length == 1 && typeitem[lookupkey][0].toString() == item)); |
| 2407 |
}); |
| 2408 |
if (found.length == 1) { |
| 2409 |
return found[0]; |
| 2410 |
} |
| 2411 |
} |
| 2412 |
return null; |
| 2413 |
} else if (this.savedquerynames.has(property)) { |
| 2414 |
const saved_query = this.data.queries.find((q) => q.name == property); |
| 2415 |
if (saved_query.results && !saved_query.relative) { |
| 2416 |
return this.lookup(item, saved_query.results); |
| 2417 |
} else if (saved_query.relative) { |
| 2418 |
return this.execute([item], this.parse(saved_query.query), labeled); |
| 2419 |
} |
| 2420 |
} |
| 2421 |
return null; |
| 2422 |
}); |
| 2423 |
if (res) return res; |
| 2424 |
if (dataset) { |
| 2425 |
if (property in this.adapters && (!this.recache || this.recache.has(property))) { |
| 2426 |
return aqueue(property, item); |
| 2427 |
} |
| 2428 |
return null; |
| 2429 |
} |
| 2430 |
if (property in this.adapters) { |
| 2431 |
return aqueue(property, item); |
| 2432 |
} |
| 2433 |
} |
| 2434 |
return item; |
| 2435 |
} |
| 2436 |
|
| 2437 |
aqueue = (property, item) => { |
| 2438 |
const pending_queues = Object.keys(this.adapters).filter((key) => this.adapters[key].queue.length > 0); |
| 2439 |
if (pending_queues.length == 0 || (pending_queues.length == 1 && pending_queues[0] == property)) { |
| 2440 |
this.adapters[property].queue.push(this.adapters[property].annotator ? item : this.getid(item)); |
| 2441 |
} |
| 2442 |
this.adaptive = true; |
| 2443 |
return undefined; |
| 2444 |
} |
| 2445 |
|
| 2446 |
samearray(a, b) { |
| 2447 |
if ((a && !b) || (!a && b) || a.length != b.length) return false; |
| 2448 |
for (let x = 0; x < a.length; x++) { |
| 2449 |
if (this.getid(a[x]) != this.getid(b[x])) return false; |
| 2450 |
} |
| 2451 |
return true; |
| 2452 |
} |
| 2453 |
|
| 2454 |
verify = async (i = null) => { |
| 2455 |
let toverify = this.data.queries; |
| 2456 |
if (i && !isNaN(i)) toverify = toverify.slice(i - 1, i); |
| 2457 |
let allsame = true; |
| 2458 |
for (const savedq of toverify) { |
| 2459 |
console.log('verifying ' + savedq.name); |
| 2460 |
const testres = await this.query(savedq.query); |
| 2461 |
if (testres.length != savedq.results.length) { |
| 2462 |
console.log('--x result count changed from ' + savedq.results.length + ' to ' + testres.length); |
| 2463 |
allsame = false; |
| 2464 |
} else { |
| 2465 |
for (let i = 0; i < testres.length; i++) { |
| 2466 |
const testrow = testres[i]; |
| 2467 |
const savedrow = savedq.results[i]; |
| 2468 |
if (typeof savedrow == 'object') { |
| 2469 |
for (const prop in savedrow) { |
| 2470 |
if (!(prop in testrow)) { |
| 2471 |
console.log('--x row ' + (i + 1) + ': new results missing property ' + prop); |
| 2472 |
allsame = false; |
| 2473 |
} else { |
| 2474 |
const testval = JSON.stringify(testrow[prop]); |
| 2475 |
const savedval = JSON.stringify(savedrow[prop]); |
| 2476 |
if (testval != savedval) { |
| 2477 |
console.log('--x row ' + (i + 1) + ': different value for property ' + prop); |
| 2478 |
console.log({was: savedrow[prop], now: testrow[prop]}) |
| 2479 |
allsame = false; |
| 2480 |
} |
| 2481 |
} |
| 2482 |
} |
| 2483 |
} else { |
| 2484 |
if (testrow != savedrow) { |
| 2485 |
console.log('--x row ' + (i + 1) + ': different value'); |
| 2486 |
console.log({was: savedrow, now: testrow}); |
| 2487 |
allsame = false; |
| 2488 |
} |
| 2489 |
} |
| 2490 |
} |
| 2491 |
} |
| 2492 |
if (allsame) console.log('--- results unchanged'); |
| 2493 |
} |
| 2494 |
} |
| 2495 |
|
| 2496 |
index_check() { |
| 2497 |
console.table(Object.entries(this.index).map(([key, vals]) => ({ |
| 2498 |
key: key, |
| 2499 |
vals: Object.keys(vals).length, |
| 2500 |
size: Object.keys(vals).length * JSON.stringify(Object.entries(vals).slice(0, 1)).length |
| 2501 |
})).sort((a, b) => b.size - a.size || b.vals - a.vals || a.key.localeCompare(b.key))) |
| 2502 |
} |
| 2503 |
|
| 2504 |
index_materialize() { |
| 2505 |
this.load(Object.keys(this.index).flatMap((i) => Object.keys(this.index[i]).flatMap((k) => ({ |
| 2506 |
index: i, |
| 2507 |
indexed: k, |
| 2508 |
value: this.index[i][k] |
| 2509 |
}))), 'index contents') |
| 2510 |
} |
| 2511 |
|
| 2512 |
indexlog_materialize() { |
| 2513 |
this.load(Object.entries(this.index._).map(([k, v]) => ({ |
| 2514 |
index: k, |
| 2515 |
log: Object.entries(v).map(([date, readwrite]) => ({ |
| 2516 |
date: date, |
| 2517 |
read: readwrite.read, |
| 2518 |
write: readwrite.write |
| 2519 |
})) |
| 2520 |
})), 'indexlog'); |
| 2521 |
} |
| 2522 |
|
| 2523 |
queries_check() { |
| 2524 |
console.table(dactal.data.queries.map((q) => ({ |
| 2525 |
queryname: q.name, |
| 2526 |
results: q.results?.length || 0, |
| 2527 |
size: q.results?.length > 0 ? q.results.length * JSON.stringify(q.results[0]).length : 0 |
| 2528 |
})).sort((a, b) => b.size - a.size || b.results - a.results || a.queryname.localeCompare(b.queryname))) |
| 2529 |
} |
| 2530 |
|
| 2531 |
data_check() { |
| 2532 |
console.table(Object.entries(this.data).map(([key, vals]) => ({ |
| 2533 |
key: key, |
| 2534 |
vals: Object.keys(vals).length, |
| 2535 |
size: Object.keys(vals).length * JSON.stringify(Object.entries(vals).slice(0, 1)).length |
| 2536 |
})).sort((a, b) => b.size - a.size || b.vals - a.vals || a.key.localeCompare(b.key))) |
| 2537 |
} |
| 2538 |
} |
| 2539 |
|
| 2540 |
window.DACTAL = new DACTAL(); |