export class DACTAL {
    constructor(data={}) {
        this.data = data;
        this.index = {};
        this.index_modified = false;
        this.adapters = {};
        this.data['query history'] = [];
        this.savedquerynames = new Set();
        this.debug = false;
        this.recache = false;
        this.timelimit = 120000;
        this.statusf = (statusmsg) => console.log(statusmsg);
        this.aggregators = {
            group: (item) => item.of,
            count: (item) => this.vals(item).length,
            total: (item) => this.numvals(item).reduce((acc, val) => acc + (this.dtype(val, 'number') ? val : 0), 0),
            average: (item) => this.numvals(item).reduce((acc, val) => acc + (this.dtype(val, 'number') ? val : 0), 0) / this.numvals(item).length,
            min: (item) => {
                const nums = this.numvals(item);
                return nums.length === 0 ? [] : Math.min(...nums)
            },
            max: (item) => {
                const nums = this.numvals(item);
                return nums.length === 0 ? [] : Math.max(...nums)
            },
            product: (item) => this.numvals(item).reduce((acc, val) => acc * (this.dtype(val, 'number') ? val : 1), 1),
            difference: (item) => this.numvals(item).reduce((acc, val) => acc - (this.dtype(val, 'number') ? val : 0)),
            quotient: (item) => this.numvals(item).reduce((acc, val) => acc / (this.dtype(val, 'number') ? val : 1)),
            percent: (item) => Math.round(this.numvals(item).reduce((acc, val) => acc / (this.dtype(val, 'number') ? val : 1)) * 100),
            sqrt: (item) => Math.sqrt(this.numvals(item)[0]),
            log: (item) => Math.log(this.numvals(item)[0]),
            log10: (item) => Math.log10(this.numvals(item)[0]),
            abs: (item) => Math.abs(this.numvals(item)[0]),
            is: (item) => (item.of?.length > 0) ? 1 : 0,
            isnt: (item) => (item.of?.length === 0) ? 1 : 0,
            yesno: (item) => (item.of?.length > 0) ? 'yes' : 'no', 
            missing: (item) => (item.of?.length === 0) ? true : null,
            concatenate: (item) => this.vals(item).join(' '),
            join: (item) => item.of.map(this.getname).join(this.vals(item)[0]),
            str: (item) => item.of.map(this.getname).join(''),
            'to json': (item) => JSON.stringify(item.of),
            quote: (item) => `“${this.vals(item)[0]}”`,
            url: (item) => {
                let u = item.of.map(this.getname).join('');
                if (!u.startsWith('https://')) u = 'https://' + u;
                for (const prop of Object.keys(item).filter((key) => key !== 'of')) {
                    const val = item[prop];
                    (Array.isArray(val) ? val : [val]).forEach((vv) => u = u + (u.match(/\?/) ? '&' : '?') + encodeURIComponent(prop) + '=' + encodeURIComponent(vv)); 
                }
                return u;
            },
            remove: (item) => this.vals(item).reduce((acc, val) => acc.replaceAll(val, '')),
            bmk: (item) => this.vals(item).map((v) => v.toString().toLowerCase().replaceAll(/[$\xA2-\xA5\u058F\u060B\u09F2\u09F3\u09FB\u0AF1\u0BF9\u0E3F\u17DB\u20A0-\u20BD\uA838\uFDFC\uFE69\uFF04\uFFE0\uFFE1\uFFE5\uFFE6,+]/g, '').replaceAll('b', 'kkk').replaceAll('m', 'kk').replaceAll('k', '000')),
            sortform: (item) => this.vals(item).map((val) => val.toString().toLowerCase().replace(/^the /, '')),
            zip: (item) => {
                const itemkeys = Object.keys(item).filter((k) => k !== 'of');
                const zipped = [];
                for (let i=0; i<item[itemkeys[0]].length; i++) {
                    const zipline = {};
                    for (const key of itemkeys) {
                        zipline[key] = item[key][i];
                    }
                    zipped.push(zipline);
                }
                return zipped;
            },
            pairs: (item) => {
                return item.of.slice(0, -1).map((val, vx) => ({pair: [val, item.of[vx+1]]}));
            },
            triples: (item) => {
                return item.of.slice(0, -2).map((val, vx) => ({triple: [val, item.of[vx+1], item.of[vx+2]]}));
            },
            quads: (item) => {
                return item.of.slice(0, -3).map((val, vx) => ({quad: [val, item.of[vx+1], item.of[vx+2], item.of[vx+3]]}));
            },
            sequences: (item) => {
                return item.of.map((val, valx, vallist) => ({sequence: vallist.slice(0, valx+1).map((val) => this.dcopy(val))}));
            },
            split: (item) => {
                const itemvals = this.vals(item);
                let splitter;
                let tobesplit;
                if (Object.keys(item).length === 1) itemvals.push(' ');
                if (itemvals.length === 1) {
                    splitter = itemvals[0];
                    tobesplit = [this.getname(item)];
                } else {
                    splitter = itemvals.pop();
                    tobesplit = itemvals.slice(0);
                }
                if (splitter.startsWith('~')) splitter = new RegExp(splitter.replace(/^~*/, ''), splitter.startsWith('~~') ? 'i' : '');
                return tobesplit.flatMap((v) => v.toString().split(splitter));
            },
            extract: (item) => {
                const itemvals = this.vals(item);
                const delimiters = itemvals.pop();
                const res = [];
                for (const itemval of itemvals) {
                    for(let i=0; i<delimiters.length; i+=2) {
                        const d1 = delimiters[i];
                        const d2 = delimiters[i+1];
                        const d1x = itemval.indexOf(d1);
                        const d2x = itemval.indexOf(d2);
                        if (d1x > -1 && d2x > d1x) {
                            res.push(itemval.slice(d1x+1, d2x).trim());
                            break;
                        }
                    }
                }
                return res;
            },
            unchain: (item) => {
                const chainprops = Object.keys(item).filter((key) => key !== 'of');
                const unchained = [];
                const queue = item.of.slice(0);
                while (queue.length > 0) {
                    const thisitem = queue.shift();
                    if (!unchained.includes(thisitem)) {
                        unchained.push(thisitem);
                        for (const chainprop of chainprops) {
                            if (this.dtype(thisitem, 'object') && chainprop in thisitem) {
                                if (this.dtype(thisitem[chainprop], 'array')) {
                                    for (const x of thisitem[chainprop].slice(0).reverse()) queue.unshift(x);
                                } else if (thisitem[chainprop]) {
                                    queue.unshift(thisitem[chainprop])
                                }
                            }
                        }
                    }
                }
                return unchained;        
            },
            itemize: (item) => {
                const propname = item?.property ?? 'property';
                const valname = item?.value ?? 'value';
                return item.of.flatMap((subitem) => Object.entries(subitem).map(([key, val]) => ({[propname]: key, [valname]: val})));
            },
            schematize: (item) => {
                const itemkeys = Object.keys(item).filter((k) => k !== 'of');
                const schematized = {};
                item[itemkeys[0]].forEach((subitem) => {
                    let subkey;
                    let subval;
                    if (this.dtype(subitem, 'array')) {
                        const [subkey, subval] = subitem;
                    } else {
                        [subkey, subval] = Object.values(subitem);
                    }
                    schematized[subkey] = subval;
                });
                return schematized;
            },
            index: (item) => {
                return Object.entries(item).filter(([k, v]) => k !== 'of').map(([k, v]) => ({id: k, name: !isNaN(v) ? Number(v) : v}));
            },
            unflatten: (item) => {
                const itemkeys = Object.keys(item).filter((k) => k !== 'of');
                if (itemkeys.length === 0) itemkeys.push('');
                const newindex = {};
                const neworder = [];
                item.of.forEach((subitem) => {
                    Object.keys(subitem).forEach((field) => {
                        itemkeys.forEach((key) => {
                            if (field.startsWith(key)) {
                                const subid = field.replace(key, '');
                                if (subid.length > 0) {
                                    if (!(subid in newindex)) {
                                        newindex[subid] = {};
                                        neworder.push(subid);
                                    }
                                    newindex[subid][key] = subitem[field];
                                }
                            }
                        });
                    });
                });
                itemkeys.forEach((key) => delete item[key]);
                return neworder.map((k) => {
                    const subitem = {};
                    subitem.subid = k;
                    Object.assign(subitem, newindex[k]);
                    return subitem;
                });
            },
            detupled: (item) => {
                const newobj = {};
                item.of.forEach((subitem) => {
                    if (Array.isArray(subitem) && subitem.length === 2) {
                        newobj[subitem[0]] = subitem[1];
                    }
                })
                return [newobj];
            },
            csv: (item) => {
                const keys = Object.entries(item).find(([k, v]) => k !== 'of')[1];
                const res = [];
                const vals = item.of.flatMap((val) => this.dtype(val, 'string') ? val.split('\n').map((val) => val.trim()) : val);
                for (let i=0; i<vals.length; i+=keys.length) {
                    const newitem = {};
                    for (let k=0; k<keys.length; k++) {
                        newitem[keys[k]] = vals[i+k];
                    }
                    res.push(newitem);
                }
                return res;
            },
            tsv: (item) => {
                const text = this.getname(item);
                const lines = text.split('\n').filter((line) => line !== '').map((line) => line.split('\t').map((val) => val.trim()));
                const keys = lines[0];
                return lines.slice(1).map((vals) => Object.fromEntries(vals.map((val, vi) => [keys[vi], val])));
            },
            ssv: (item) => {
                const text = this.getname(item);
                const lines = text.split('\n').filter((line) => line !== '').map((line) => line.split(/ +/).map((val) => val.trim()));
                const keys = lines[0];
                return lines.slice(1).map((vals) => Object.fromEntries(vals.map((val, vi) => [keys[vi], val])));
            },
            json: (item) => this.vals(item).flatMap((v) => JSON.parse(v)),
            year: (item) => {
                if (item.date?.length > 0) return item.date[0].match(/\d\d\d\d/)[0];
                let yearmatch = this.getname(item).match(/\d\d\d\d/);
                if (yearmatch) {
                    return yearmatch[0];
                } else {
                    yearmatch = this.getid(item.of[0] || '').toString().match(/\d\d\d\d/);
                    if (yearmatch) {
                        return yearmatch[0];
                    }
                    return null;
                }
            },
            month: (item) => {
                if (item.date?.length > 0) return item.date[0].match(/\d\d\d\d-(\d\d)-\d\d/)[1];
                let monthmatch = this.getname(item).match(/\d\d\d\d-(\d\d)-\d\d/);
                if (monthmatch) {
                    return monthmatch[1];
                } else {
                    monthmatch = this.getid(item.of[0] || '').toString().match(/\d\d\d\d-(\d\d)-\d\d/);
                    if (monthmatch) {
                        return monthmatch[1];
                    }
                    return null;
                }
            },
            date: (item) => {
                let itemvals = this.vals(item);
                if (itemvals) {
                    let datematch = itemvals[0].match(/\d\d\d\d-\d\d-\d\d/);
                    if (datematch) {
                        return datematch[0];
                    } else {
                        datematch = this.getid(item.of[0] || '').toString().match(/\d\d\d\d-\d\d-\d\d/);
                        if (datematch) {
                            return datematch[0];
                        }
                    }
                }
                return null;
            },
            weekday: (item) => {
                const days = ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'];
                if (item.date?.length > 0) return days[new Date(item.date).getDay()];
            },
            timeshift: (item) => {
                const vals = this.vals(item);
                let tsx = new Date(vals[0]);
                let adjust = Number(vals[1]) * 60*60*1000;
                tsx.setTime(tsx.getTime() + adjust);
                return tsx.toISOString();
            },
            hour: (item) => this.vals(item).map((v) => v.split('T')[1].split(':')[0]),
            datediff: (item) => {
                const [d1, d2] = this.vals(item);
                return (new Date(d2) - new Date(d1)) / (24*60*60*1000);
            },
            timediff: (item) => {
                const [d1, d2] = this.vals(item);
                return (new Date(d2) - new Date(d1));
            },
            dateforms: (item) => {
                const basedate = this.vals(item)[0];
                const [baseyear, basemonth, baseday] = basedate.split('-');
                return [
                    basedate,
                    `${basemonth}/${baseday}/${baseyear}`,
                    basemonth.startsWith('0') ? `${basemonth.replace(/^0/,'')}/${baseday}/${baseyear}` : null,
                    basemonth.startsWith('0') && baseday.startsWith('0') ? `${basemonth.replace(/^0/,'')}/${baseday.replace(/^0/,'')}/${baseyear}` : null,
                    `${basemonth}/${baseday}/${baseyear.slice(2)}`,
                    `${basemonth.replace(/^0/,'')}/${baseday.replace(/^0/,'')}/${baseyear.slice(2)}`,
                    `${['', 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'][Number(basemonth)]} ${baseday.replace(/^0/,'')}, ${baseyear}`
                ].filter(x => x);
            },
            now: (item) => performance.now(),
            round: (item) => this.numvals(item).map((v) => Math.round(v)),
            roundaway: (item) => this.numvals(item).map((v) => Math.sign(v) * Math.round(Math.abs(v))),
            roundm: (item) => {
                const vals = this.numvals(item);
                const multiple = vals.pop();
                return vals.map((v) => Math.round(v / multiple) * multiple);
            },
            roundd: (item) => {
                const vals = this.numvals(item);
                const digits = vals.pop();
                return vals.map((v) => {
                    const factor = 10 ** (Math.floor(Math.log10(v)) - digits + 1);
                    return Math.round(v / factor) * factor;
                });
            },
            floor: (item) => this.numvals(item).map((v) => Math.floor(v)),
            ceil: (item) => this.numvals(item).map((v) => Math.ceil(v)),
            n: (item) => {
                let i = 0;
                const test = this.getname(item.of[0]);
                for (const key in item) {
                    if (key !== 'of') {
                        i--;
                        if (key === test) return i;
                    }
                }
                return i - 1;
            },
            numbers: (item) => {
                const res = [];
                const firstnum = item.from !== undefined && item.from != null ? Number(item.from) : 1;
                const lastnum = item.to !== undefined && item.to != null ? Number(item.to) : Number(this.getname(item.of[0]));
                for(let x=firstnum; x<=lastnum; x++) {
                    res.push(x)
                }
                return res;
            },
            dehyphenate: (item) => this.vals(item).flatMap((v) => v.replaceAll(/(?<=\w)-\n(?=\w)/g, '')),
            sentences: (item) => this.vals(item).flatMap((v) => v.split(/[.?!…]['"’”»]?\s+/)),
            allwords: (item) => this.vals(item).flatMap((v) => Array.from(v.toString().toLowerCase().matchAll(/[\p{Letter}\p{Number}]+(?:['‘’][\p{Letter}\p{Number}]+)?/gu).map((m) => m[0]))),
            words: (item) => this.vals(item).flatMap((v) => Array.from(v.toString().toLowerCase().matchAll(/[\p{Letter}\p{Number}]+(?:['‘’][\p{Letter}\p{Number}]+)?/gu).map((m) => m[0])).filter((w) => w.length >= 4)),
            characters: (item) => this.vals(item).flatMap((v) => Array.from(v)),
            'character count': (item) => this.vals(item).flatMap((v) => v.length),
            case: (item) => this.vals(item).map((v) => {
                const hasupper = v.match(/[A-Z]/);
                const haslower = v.match(/[a-z]/);
                if (!hasupper && !haslower) {
                    return 'none';
                } else if (hasupper && !haslower) {
                    return 'upper';
                } else if (haslower && !hasupper) {
                    return 'lower';
                } else {
                    if (v[0].match(/[A-Z]/) && !(v.slice(1).match(/[A-Z]/))) {
                        return 'initial';
                    } else {
                        return 'mixed';
                    }
                }
            }),
            uppercase: (item) => this.vals(item).flatMap((v) => v.toString().toUpperCase()),
            lowercase: (item) => this.vals(item).flatMap((v) => v.toString().toLowerCase().replaceAll(/[‘’`]/g, "'").replaceAll(/[“”]/g, '"')),
            list: (item) => Object.keys(item).filter((k) => k !== 'of'),
            items: (item) => item?.of?.flatMap((typename) => this.data?.[this.getname(typename)]),
            random: (item) => Math.random(),
            shuffle: (item) => {
                const newArray = [].concat(item?.of || []);
                for (let i = newArray.length - 1; i > 0; i--) {
                    const j = Math.floor(Math.random() * (i + 1));
                    [newArray[i], newArray[j]] = [newArray[j], newArray[i]];
                }
                return newArray;
            },
            'weighted shuffle': (item) => {
                return item.of.map((subitem) => ({weight: Number(subitem.weight || 1) * Math.random(), subitem: subitem})).sort((a, b) => b.weight - a.weight).map((ws) => ws.subitem);
            },
            pick: (item) => item?.of?.[~~(Math.random() * item?.of?.length)],
            link: (item) => `<a href="${item.url}"${item.target ? ' target="' + item.target + '"' : ''}>${item.text}</a>`,
            img: (item) => {
                if (item.uri) {
                    return `<a href="${item.uri}"><img src="${item.src}" height=${item.height}px width=${item.width}px></a>`;
                } else {
                    return `<img src="${item.src}" height=${item.height}px width=${item.width}px>`;
                }
            },
            sign: (item) => item.of.map((subitem) => {
                if (subitem.id) {
                    return subitem;
                } else {
                    const str = JSON.stringify(subitem);
                    let hash = 0;
                    for (let i = 0; i < str.length; i++) {
                        const char = str.charCodeAt(i);
                        hash = (hash << 5) - hash + char;
                    }
                    // Convert to 32bit unsigned integer in base 36 and pad with "0" to ensure length is 7.
                    return Object.assign({id: (hash >>> 0).toString(36).padStart(7, '0')}, subitem);
                }
            }),
            'parse query': (item) => item.of.map((subitem) => ({name: subitem.name, query: subitem.query, assembly: this.parse(subitem.query)}))
        }
        this.data.aggregators = Object.keys(this.aggregators);
        // this.cachehitcount = 0;
        // this.cachemisscount = {};
        this.destinations = new Set();
    }
    
    survey() {
        this.destinations = new Set(Object.keys(this.data).concat(this.data?.queries?.map((q) => q.name) || []).concat(Object.keys(this.adapters)));
    }
    
    vals(item) {
        const getname = this.getname;
        const itemvals = [];
        if (Object.keys(item).length > 1) {
            Object.entries(item).filter(([k, v]) => k !== 'of').flatMap(([k, v]) => Array.isArray(v) ? v : [v]).map(getname).forEach((v) => itemvals.push(v));
        } else {
            item.of.map(getname).forEach((v) => itemvals.push(v));
        }
        return itemvals;
    }

    numvals(item) {
        return this.vals(item).filter((val) => this.dtype(val, 'number')).map((val) => Number(val));
    }

    async querylive(query, inputlist=null) {
        this.recache = new Set();
        return await this.query(query, inputlist);
    }

    async query(query, inputlist=null, loop=50) {
        this.survey();
        const operations = Array.isArray(query) ? query : this.assemble(this.tokenize(query));
        const result = this.execute(inputlist, operations);
        this.data['current results'] = result;
        const queued = Object.keys(this.adapters).filter((key) => this.adapters[key].queue.length > 0);
        if (loop > 0 && queued.length > 0) {
            await this.adapt();
            if (this.recache) queued.forEach((q) => this.recache.add(q));
            return await this.query(operations, inputlist, loop - 1);
        } else if (loop === 0) {
            this.recache = false;
            Object.keys(this.adapters).forEach((key) => {
                if (!this.adapters[key].annotator) {
                    this.adapters[key].queue.forEach((id) => {
                        (this.index[key] ||= {})[id] = null;
                    })
                }
            })
        }
        return result;
    }
    
    async adapt() {
        for (const key in this.adapters) {
            const adapter = this.adapters[key];
            if (adapter.queue.length > 0) {
                if (adapter.annotator) {
                    while (adapter.queue.length > 0) {
                        const item_to_annotate = adapter.queue[0];
                        try {
                            (this.index[key] ||= {})[this.getid(item_to_annotate)] = await adapter.f(item_to_annotate);
                            this.index_modified = true;
                            adapter.queue.shift();
                        } catch (e) {}                    
                    }
                } else {
                    const newids = adapter.queue.filter((id) => !adapter.pending.has(id));
                    newids.forEach((id) => adapter.pending.add(id));
                    const res = await adapter.f(newids);
                    newids.forEach((id) => adapter.pending.delete(id));
                    for (let i=0; i<adapter.queue.length; i++) {
                        const item = res[i];
                        const id = this.getid(item) ?? adapter.queue[i];
                        (this.index[key] ||= {})[id] = item;
                        this.index_modified = true;
                        if (key in this.data) {
                            this.data[key].push(item);
                        }
                    }
                    this.adapters[key].queue = [];
                }
            }
        }
    }
    
    async rehope() {
        Object.keys(this.index).forEach((k) => {
            const todelete = Object.keys(this.index[k]).filter((dk) => this.index[k][dk] == null || this.index[k][dk] === undefined);
            todelete.forEach((dk) => delete(this.index[k][dk]));
        });
    }

    load(something, named, append=false) {
        if (!named) return
        if (!append || !(named in this.data)) this.data[named] = [];
        if (Array.isArray(something)) {
            something.forEach((somethingx) => this.data[named].push(somethingx));
        } else if (typeof something == 'object') {
            const firstkey = Object.keys(something)[0];
            const firstobj = something[firstkey];
            if (typeof firstobj === 'object' && !Array.isArray(firstobj) && (firstkey in Object.values(firstobj) || 'id' in firstobj)) {
                this.data[named].push(...Object.values(something));
            } else if (typeof firstobj === 'object' && !Array.isArray(firstobj)) {
                this.data[named].push(...Object.entries(firstobj).map(([key, val]) => ({id: key, ...val})));
            } else if (typeof firstobj === 'string') {
                this.data[named].push(...Object.entries(something).map(([key, val]) => ({id: key, name: val})));
            } else {
                this.data[named].push(...Object.entries(something).map(([key, val]) => ({id: key, value: val})));
            }
        } else {
            this.data[named].push(something);
        }
        return this.data[named];
    }
    
    async loadjsonl(something, named, append=false) {
        if (!named) return
        if (!append || !(named in this.data)) this.data[named] = [];
        if (typeof something == 'string' && (something.startsWith('http') || something.startsWith('file://'))) {
            const fetchres = await fetch(something);
            something = await fetchres.text();
        }
        const rows = something.trim().split(/[\n\r]+/);
        for (const row of rows) {
            const rowdata = JSON.parse(row);
            if (rowdata) this.data[named].push(rowdata);
        }
    }

    async loadcsv(something, named, quoteChar = '"', delimiter = ',', headerrows=1) {
        if (typeof something == 'string' && (something.startsWith('http') || something.startsWith('file://'))) {
            const fetchres = await fetch(something);
            something = await fetchres.text();
        }
        const rows = something.split(/[\n\r]+/);

        const regex = new RegExp(`\\s*(${quoteChar})?(.*?)\\1\\s*(?:${delimiter}|$)`, 'gs');
      
        const match = (line) => Array.from(line.matchAll(regex), (m) => m[2]);

        const headers = [];
        for (let hrowx=0; hrowx<headerrows; hrowx++) {
            const hrow = rows.shift();
            match(hrow).forEach((h, hx) => {
                if (hrowx === 0) {
                    headers.push(h);
                } else {
                    headers[hx] = headers[hx] + ' ' + h;
                }
            });
        }
        const heads = headers ?? match(rows.shift());
        const lines = rows.slice(0).filter((line) => line);
        const parsed = lines.map((line) => {
          return match(line).reduce((acc, cur, i) => {
            // replace blank matches with `null`
            const val = cur.length <= 0 ? null : (!isNaN(cur) ? Number(cur) : cur);
            const key = heads[i] ?? `{i}`;
            if (key === '') {
                return { ...acc};
            } else {
                return { ...acc, [key]: val };
            }
          }, {});
        });
        this.load(parsed, named);
    }
    
    apacheLogToDate(apacheTimestamp) {
        // Apache log format: [10/Oct/2000:13:55:36 -0700]
        // Remove brackets if present
        const cleanTimestamp = apacheTimestamp.replace(/^\[|]$/g, '');
        
        // Split into date/time and timezone parts
        const [dateTimePart, timezone] = cleanTimestamp.split(' ');
        
        // Parse the date/time part: dd/MMM/yyyy:HH:mm:ss
        const [datePart, hour, minute, second] = dateTimePart.split(':');
        const [day, month, year] = datePart.split('/');
        
        // Month mapping
        const months = {
            'Jan': 0, 'Feb': 1, 'Mar': 2, 'Apr': 3, 'May': 4, 'Jun': 5,
            'Jul': 6, 'Aug': 7, 'Sep': 8, 'Oct': 9, 'Nov': 10, 'Dec': 11
        };
        
        // Create Date object (months are 0-indexed in JS)
        const date = new Date(
            parseInt(year),
            months[month],
            parseInt(day),
            parseInt(hour),
            parseInt(minute),
            parseInt(second)
        );
        
        // Handle timezone offset if present
        if (timezone) {
            const sign = timezone[0] === '+' ? 1 : -1;
            const tzHours = parseInt(timezone.slice(1, 3));
            const tzMinutes = parseInt(timezone.slice(3, 5));
            const offsetMs = sign * (tzHours * 60 + tzMinutes) * 60 * 1000;
            
            // Adjust for timezone (Apache logs are in local time, JS Date assumes UTC)
            date.setTime(date.getTime() - offsetMs);
        }
        
        return date;
    }
    
    async loadclf(loglines, named, apiroutes=[]) {
        const lines = loglines.trim().split('\n');
        const cols9 = ['ip', 'name', 'username', 'timestamp', 'requestraw', 'status', 'bytes', 'referrer', 'useragent'];
        const cols10 = ['host'].concat(cols9);
        const parsed = lines.map((line) => {
            const vals = Array.from(line.matchAll(/"(?:\\"|.)*?"|\[.*?]|\S+/g)).map((m) => m[0]);
            const cols = vals.length === 10 ? cols10 : cols9;
            const obj = Object.fromEntries(vals.map((v, vx) => ([cols[vx], v])));
            const date = this.apacheLogToDate(obj.timestamp);
            obj.timestamp = date.toISOString();
            [obj.date, obj.time] = obj.timestamp.slice(0, -1).split('T');
            if (obj.requestraw.match(/ [^ ]+ /) && !obj.requestraw.startsWith('"{')) {
                [obj.method, obj.request, obj.protocol] = obj.requestraw.slice(1, -1).split(' ');
                if (this.data['.clf API routes']) {
                    for (const apiroute of this.data['.clf API routes'].sort((a, b) => b.length - a.length)) {
                        if (obj.request.startsWith(apiroute)) {
                            obj.page = apiroute;
                            break;
                        }
                    }
                }
                obj.page ??= obj.request.split('?')[0];
            }
            obj.referrer = obj.referrer.slice(1, -1);
            obj.useragent = obj.useragent.slice(1, -1);
            obj.logline = line;
            return obj;
        });
        this.load(parsed, named);
    }
    
    async loadrss(rsstext, named) {
        const rssval = (k, rawval) => {
            if (k.match(/date/i)) {
                return new Date(rawval).toISOString();
            } else if (!isNaN(rawval)) {
                return Number(rawval);
            } else {
                return rawval;
            }
        }
        const rssdom = new window.DOMParser().parseFromString(rsstext, "text/xml");
        const items = Array.from(rssdom.querySelectorAll('item')).map((i) => {
            const obj = {};
            Array.from(i.children).forEach((c) => {
                const k = c.tagName;
                const rawval = c.textContent;
                obj[k] = rssval(k, rawval);
            });
            return obj;
        });
        this.load(items, named);
    }
    
    connect(key, adapter, annotator=false) {
        this.adapters[key] = {queue: [], pending: new Set(), f: adapter, annotator: annotator};
        this.data.adapters ??= [];
        if (!this.data.adapters.includes(key)) this.data.adapters.push(key);
    }

    connect_annotator(key, adapter) {
        this.connect(key, adapter, true);
    }
    
    register(key, aggregator) {
        this.aggregators[key] = aggregator;
        this.data.aggregators ??= [];
        if (!this.data.aggregators.includes(key)) this.data.aggregators.push(key);
    }
    
    unbracket(token) {
        if (!(typeof token == 'string')) token = token.toString();
        if (token.startsWith('[') && token.endsWith(']')) {
            return token.slice(1, -1).replaceAll(']]', ']');
        }
        return token;
    }

    bracket(token) {
        if (!(typeof token == 'string')) token = token?.toString() ?? '';
        if (token.match(/[?.:#\/|!<>=~@\[\](),;+-]/) || token.startsWith(' ') || token.endsWith(' ')) {
            return '[' + token.replaceAll(']', ']]') + ']';
        }
        return token;
    }

    escapeRegExp(string) {
        return string.replace(/[.*+?${}()|[\]\\]/g, "\\$&");
    }

    tokenize(text) {
        if (typeof text != 'string') text = String(text);
        const matches = text.matchAll(/(\[(?:]]|[^\]])*])|(\?{1,3})|(\.{1,4})|(\|{1,2})|(\/{1,2})|([:#!])|(\()|(\))|([@~=<>+-]+)|(,)|(;)|([^\[\]().?:\/#|!~=<>@,;+-]+)|([\[\]])/gms);
        return Array.from(matches, m => m[0].trim()).filter((token) => token.length > 0);
    }

    assemble(tokenized) {
        const isOperator = (token) => '???....://#||!'.includes(token);
        const tokenlist = '= ~ =~ ~< ~> > < <> >< >= <= - =- ~- + =+ @ @@ @- @= @@= =@ =@@ @< @@< @<= @@<= @> @@> @>= @@>= >> >>= << <<= ~~ =>'.split(' ');
        const isSubop = (token) => tokenlist.includes(token) || (token?.startsWith('-') && (token.length === 1 || tokenlist.includes(token.slice(1))));
        const isSeparator = (token) => ',;'.includes(token);
        const isValue = (token) => !isOperator(token) && !isSubop(token) && !isSeparator(token);
                
        let tokens = tokenized.slice();
        let level = 0;
        const operations = [];
        while (tokens.length > 0) {
            const op = {operator: null, args: []};
            let token = tokens.shift();
            if (isOperator(token) || ((isValue(token) || isSubop(token)) && operations.length === 0)) {
                if ((isValue(token) || isSubop(token)) && operations.length === 0) {
                    op.operator = '?';
                    tokens.unshift(token);
                } else {
                    op.operator = token;
                }
                while (tokens.length > 0 && !isOperator(tokens[0])) {
                    const arg = {separator: null, label: null, subop: null, value: null};
                    if (isSeparator(tokens[0])) {
                        arg.separator = tokens.shift();
                    }
                    while (tokens.length > 0 && !isOperator(tokens[0]) && !isSeparator(tokens[0])) {
                        const frag = tokens.shift();
                        if (frag !== '(' && isValue(frag) && isSubop(tokens[0])) {
                            arg.label = this.unbracket(frag);
                            arg.subop = tokens.shift();
                        } else if (isSubop(frag)) {
                            arg.subop = frag;
                        } else if (frag === '(') {
                            const subquery = [];
                            level++;
                            while (tokens.length > 0 && level > 0) {
                                const sub = tokens.shift();
                                if (sub === '(') {
                                    level++;
                                    if (level > 0) {
                                        subquery.push(sub);
                                    }
                                } else if (sub === ')') {
                                    level--;
                                    if (level > 0) {
                                        subquery.push(sub);
                                    }
                                } else if (level > 0) {
                                    subquery.push(sub);
                                }
                            }
                            arg.value = this.assemble(subquery);
                        } else if (!arg.value) {
                            arg.value = this.unbracket(frag);
                        } else {
                            const failure = {tokens: tokenized, assembled: operations.slice(0), assembling: {op: op, arg: arg, unexpected: frag}, unassembled: tokens.slice(0)}
                            throw new Error("Unexpected token", {cause: failure});
                        }
                    }
                    op.args.push(arg);
                }
            }
            operations.push(op);
            
        }
        if (level > 0) console.warn({parentropy: level, operations: operations});
        return operations;
    }
    
    parse(querystr) {
        return this.assemble(this.tokenize(querystr));
    }
    
    disassemble(assembly) {
        return assembly.map((operation, ox) => operation.operator + ((operation.args.length === 0 && ox < assembly.length - 1 && operation.operator[0] === assembly[ox + 1].operator[0]) ? ' ' : operation.args.map((arg) => (arg.separator ?? '') + (arg.label ? this.bracket(arg.label) : '') + (arg.subop ?? '') + (Array.isArray(arg.value) ? ('(' + this.disassemble(arg.value) + ')') : (arg.value ? this.bracket(arg.value) : ''))).join(''))).join('');
    }
    
    compact(querystr) {
        return this.disassemble(this.parse(querystr));
    }
    
    executeq(querystr) {
        return this.execute([], this.assemble(this.tokenize(querystr)));
    }
    
    timecheck(timer, i, count, op) {
        if (i === 1) timer.loopstart = new Date();
        if (i >= 10) {
            const taken = new Date() - timer.loopstart;
            const projected = count * taken / i;
            if (projected > this.timelimit) throw new Error('Query overrun.', {cause: {operation: dactal.disassemble([op]), items: count, done: i, elapsed: taken + 'ms', projected: Math.round(projected / 60000) + ' minutes', timelimit: Math.round(this.timelimit / 60000) + ' minutes'}});
        }
    }

    execute(inputlist, operations, labeled=null, level=null) {
        let currentlist = inputlist?.slice(0) || [];
        const getname = this.getname;
        const getid = this.getid;
        const dtype = this.dtype;
        const dcopy = this.dcopy;
        const step = this.step;
        const escapeRegExp = this.escapeRegExp;
        const unmmss = (mmssstr) => {
            const parts = mmssstr.split(':');
            let s = 0;
            const seconds = parts.pop();
            if (seconds) s += Number(seconds);
            const minutes = parts.pop();
            if (minutes) s += 60 * Number(minutes);
            const hours = parts.pop();
            if (hours) s += 60 * 60 * Number(hours);
            return s;
        };
        const compvals = (araw, braw) => {  
            const a = araw.toString();
            const b = braw.toString();  
            const atime = a.match(/^\d+(?::\d{2,})+$/);
            const btime = b.match(/^\d+(?::\d{2,})+$/);
            if (atime && btime) {
                return unmmss(btime[0]) - unmmss(atime[0]);
            }
            return dethe(a).localeCompare(dethe(b));
        }
        
        const dethe = (value) => value.toLowerCase().replace(/^the /, '');
            
        labeled ||= {};
        let outputlist = [];
        for (let opx=0; opx<operations.length; opx++) {
            const op = operations[opx];
            outputlist = [];
            switch (op.operator) {
                case '?': // start
                    if (op.args.length === 0) {
                        outputlist = Object.keys(this.data).sort((a, b) => dethe(a).localeCompare(dethe(b)));
                        break;
                    }
                    outputlist = [];
                    op.args.forEach((arg) => {
                        let startitems;
                        if (arg.label && arg.subop?.match(/\+/)) labeled[arg.label] ||= [];
                        if (arg.separator !== ';' || outputlist.length === 0) {
                            if (Array.isArray(arg.value)) {
                                startitems = this.execute([], arg.value, labeled);
                                if (arg.label) {
                                    if (arg.subop?.match(/\+/)) {
                                        startitems.forEach((i) => labeled[arg.label].push(i));
                                    } else {
                                        labeled[arg.label] = startitems;
                                        this.index[arg.label] = {};
                                    }
                                    labeled[arg.label].forEach((i) => outputlist.push(i));
                                } else {
                                    startitems.forEach((i) => outputlist.push(i));
                                }
                            } else if (arg.subop === '~') {
                                outputlist.push(arg.value);
                            } else if (arg.value in labeled) {
                                labeled[arg.value].forEach((i) => outputlist.push(i));
                            } else if (startitems = this.gettype(arg.value, false)) {
                                startitems.forEach((i) => outputlist.push(i));
                            } else if (arg.value?.trim().length > 0 && this.dtype(arg.value, 'number')) {
                                if (arg.subop === '-' && arg.value === '0') {
                                    outputlist.push(-0);
                                } else {
                                    outputlist.push((arg.subop === '-' ? -1 : 1) * Number(arg.value));
                                }
                            } else if (arg.value && arg.subop !== '=') {
                                outputlist.push(arg.value);
                            }
                        }
                    });
                    break;
                case '??': // label
                    outputlist = currentlist;
                    let ended = false;
                    op.args.forEach((arg) => {
                        if (!ended) {
                            if (arg.label === '_timelimit' && !isNaN(arg.value)) {
                                this.timelimit = Number(arg.value) * 60000;
                            } else if (arg.label) {
                                if (arg.subop.includes('~') && dtype(arg.value, 'string')) {
                                    labeled[arg.label] = [arg.value];
                                } else if (arg.labeled) {
                                    labeled[arg.label] = arg.labeled;
                                } else {
                                    if (arg.subop?.match(/\+/)) labeled[arg.label] ||= [];
                                    const newvals = this.execute(currentlist, Array.isArray(arg.value) ? arg.value : '.' + arg.value, labeled, level);
                                    if (arg.subop?.match(/\+/)) {
                                        newvals.forEach((nv) => labeled[arg.label].push(nv));
                                    } else {
                                        labeled[arg.label] = newvals;
                                    }
                                    if (!level && opx === 0) arg.labeled = newvals;
                                }
                                this.index[arg.label] = {};
                            } else if (dtype(arg.value, 'string')) {
                                if (arg.subop?.match(/\+/)) {
                                    labeled[arg.value] ??= [];
                                    currentlist.forEach((i) => labeled[arg.value].push(i));
                                } else {
                                    labeled[arg.value] = currentlist.slice(0);
                                }
                                this.index[arg.value] = {};
                                if (arg.value === 'end') {
                                    ended = true;
                                }
                            }
                        }
                    });
                    if (ended) return (this.debug && !inputlist) ? operations : outputlist;
                    break;
                case '!': // repeat
                    if (opx > 0) {
                        const repeat_ops = operations.slice(opx - 1, opx + 1);
                        outputlist = currentlist.slice(0);
                        level ??= 0;
                        level += 1;
                        const maxrecursion = ((op.args.length > 0 && op.args[0].value) || 1000);
                        if (outputlist.length > 0 && level < maxrecursion && (!this.samearray(inputlist, outputlist) || level === 1)) {
                            const recursed = this.execute(outputlist, repeat_ops, labeled, level);
                            if (recursed.length > 0 && !this.samearray(recursed, outputlist)) outputlist = recursed.slice(0);
                        }
                    }
                    break;
                case '.': // traverse
                case '..': // traverse with duplicates
                    const seen = new Set();
                    const sofar =  Array.isArray(op.args?.[0]?.value) && op.args?.[0]?.label;
                    let transq = null;
                    if (sofar) labeled[sofar] = [];
                    if (op.operator === '.' && op.args?.length === 1) {
                        const firstarg = op.args[0];
                        const firstval = firstarg?.value;
                        if (dtype(firstval, 'string')) {
                            transq = this.data.queries?.find((q) => q.relative === true && q.name === firstval);
                        }
                    }
                    if (transq) {
                        outputlist = this.execute(currentlist, this.parse(transq.query), labeled, level);
                    } else {
                        const traversetimer = {};
                        outputlist = currentlist.reduce((acc, item, i) => {
                            this.timecheck(traversetimer, i, currentlist.length, op);
                            if (op.args.length === 0) {
                                const itemkey = getid(item);
                                if (op.operator === '..' || !seen.has(itemkey)) {
                                    acc.push(item);
                                    seen.add(itemkey);
                                }
                                return acc;
                            }
                            if (op.args[0].subop?.includes('<') && acc.length > 0) return acc;
                            let itemvals = [];
                            let toremoveids = {};
                            for (const arg of op.args.filter((arg) => arg.value != null)) {
                                let passdown = null;
                                if (arg.subop?.includes('>') && typeof arg.value == 'string') {
                                    if (arg?.label in item) {
                                        passdown = item[arg.label];
                                    } else {
                                        passdown = JSON.parse(JSON.stringify(item, Object.keys(item).filter((k) => k !== arg.value)));
                                    }
                                }
                                if (arg.separator !== ';' || itemvals.length === 0) {
                                    const itemval = step(item, arg.value, arg.subop, labeled);
                                    if (arg.subop?.includes('-') && isNaN(arg.value)) {
                                        itemval.forEach((subitem) => {
                                            const subid = getid(subitem);
                                            toremoveids[subid] ||= 0;
                                            toremoveids[subid] += 1;
                                        });
                                    } else {
                                        itemval.forEach((subitem) => {
                                            if (passdown) {
                                                if (!dtype(subitem, 'object')) subitem = {value: subitem};
                                                if (arg.label) {
                                                    subitem[arg.label] = [passdown];
                                                } else {
                                                    Object.assign(subitem, passdown);
                                                }
                                            }
                                            const subitemkey = getid(subitem);
                                            if (op.operator === '..' || !seen.has(subitemkey)) {
                                                seen.add(subitemkey);
                                                itemvals.push(subitem);
                                                if (sofar) labeled[sofar].push(subitem);
                                            }
                                        })
                                    }
                                }
                                if (Object.keys(toremoveids).length > 0) {
                                    if (op.operator === '.') {
                                        itemvals = itemvals.filter((subitem) => !(getid(subitem) in toremoveids));
                                    } else {
                                        itemvals = itemvals.filter((subitem) => {
                                            const subid = getid(subitem);
                                            if (toremoveids[subid] > 0) {
                                                toremoveids[subid] -= 1;
                                                return false;
                                            } else {
                                                return true;
                                            }
                                        })
                                    }
                                }
                            }
                            itemvals.forEach((x) => acc.push(x));
                            return acc;
                        }, []);
                    }
                    outputlist = outputlist.filter((item) => item != null);
                    break;
                case ':': // filter
                    if (!op?.args?.length > 0) {
                        outputlist = currentlist;
                        break;
                    }
                    for(const arg of op.args) {
                        if (arg.subop && ['>>', '>>=', '<<', '<<=', '~~'].includes(arg.subop.replace(/^-/, ''))) {
                            arg.tests = {id: new Set(), name: new Set()};
                            if (Array.isArray(arg.value)) {
                                const testitems = this.execute(currentlist, arg.value, labeled);
                                testitems.forEach((testitem) => arg.tests.id.add(getid(testitem)));
                                testitems.forEach((testitem) => arg.tests.name.add(getname(testitem)));
                            }
                            
                            if (['>>', '>>=', '<<', '<<='].includes(arg.subop)) {
                                for (const [tryval, tryf] of [['id', getid], ['name', getname]]) {
                                    for (let j=0; j < currentlist.length; j++) {
                                        if (arg.tests[tryval].has(tryf(currentlist[j]))) {
                                            arg.matchpos = j;
                                            break;
                                        }
                                    }
                                    if (arg.matchpos) break
                                }
                            }
                        } else if ([null, '', '=', '~'].includes((arg.subop || '').replace(/-/, '')) && dtype(arg.value, 'string') && arg.value.startsWith('~')) {
                            const flags = arg.value.startsWith('~~') ? 'i' : '';
                            const pattern = arg.value.replace(/^~*/, '');
                            const fullpattern = (arg.subop || '').replace(/-/, '') === '=' ? ((pattern.startsWith('^') ? '' : '^') + pattern + (pattern.endsWith('$') ? '' : '$')) : pattern;
                            arg.re = new RegExp(fullpattern, flags);
                        }
                    }

                    const ands = [[]];
                    for (const arg of op.args) {
                        if (arg.separator === ';') {
                            ands.push([arg]);
                        } else {
                            ands[ands.length - 1].push(arg)
                        }
                    }
                    const filtertimer = {};
                    outputlist = currentlist.filter((item, i) => {
                        this.timecheck(filtertimer, i, currentlist.length, op);
                        return ands.filter((and) => {
                            return and.filter((arg) => {
                                let comparator = arg.subop;
                                let polarize = (x) => x;
                                if (comparator?.startsWith('-') || comparator?.endsWith('-')) {
                                    polarize = (x) => !x;
                                    comparator = comparator.replace(/-|-$/, '');
                                }
                                if (!comparator && arg.label == null && Array.isArray(arg.value)) {
                                    return polarize(this.execute([item], arg.value, labeled).length > 0);
                                } else if (comparator === '~~') {
                                    if (arg.label == null && dtype(arg.value, 'number')) {
                                        return polarize(currentlist.length === Number(arg.value));
                                    } else {
                                        if (arg.label) {
                                            const argitems = step(item, arg.label, null, labeled);
                                            return polarize(argitems.find((argitem) => arg.tests.id.has(getid(argitem)) || arg.tests.name.has(getname(argitem))));
                                        } else {
                                            return polarize(arg.tests.id.has(getid(item)) || (item?.id === undefined && arg.tests.name.has(getname(item))));
                                        }
                                    }
                                } else if (['~~', '<<', '<<=', '>>', '>>='].includes(comparator) && dtype(arg.value, 'number')) {
                                    const argval = Number(arg.value);
                                    let testlength;
                                    if (typeof arg.label == 'string') {
                                        testlength = step(item, arg.label, null, labeled).length;
                                    } else {
                                        testlength = currentlist.length;
                                    }
                                    switch (comparator) {
                                        case '~~': return polarize(testlength === argval);
                                        case '<<': return polarize(testlength < argval);
                                        case '<<=': return polarize(testlength <= argval);
                                        case '>>': return polarize(testlength > argval);
                                        case '>>=': return polarize(testlength >= argval);
                                    }
                                } else if (['+', ''].includes(comparator) && arg.label && !arg.value && dtype(arg.label, 'string') && dtype(item, 'object')) {
                                    const propval = item[arg.label];
                                    // console.log({plusminus: comparator, label: arg.label, propval: propval})
                                    return polarize(Array.isArray(propval) ? propval.length > 0 : propval);
                                }

                                let testitems = [item];
                                if (arg.label && dtype(item, 'object')) {
                                    let found = false;
                                    for (const tryval of [arg.label, arg.label + 's']) {
                                        if (item?.[tryval] != null) {
                                            const propitems = item[tryval];
                                            if (propitems != null) {
                                                found = true; 
                                                testitems = Array.isArray(propitems) ? propitems : [propitems];
                                                break;
                                            }
                                        }
                                    }
                                    if (!found) {
                                        if (dactal.savedquerynames.has(arg.label) || (labeled['=>'] && arg.label in labeled['=>'])) {
                                            testitems = step(item, arg.label, null, labeled);
                                        } else {
                                            testitems = [];
                                        }
                                    }
                                }

                                const testvals = testitems.map((testitem) => {
                                    if (comparator || arg.re) {
                                        return getname(testitem);
                                    } else if (this.dtype(testitem, 'literal')) {
                                        return testitem;
                                    } else if ('id' in testitem) {
                                        return testitem.id;
                                    } else {
                                        return getname(testitem)
                                    }
                                });

                                let argvals;
                                if (Array.isArray(arg.value)) {
                                    const argvalitems = this.execute([item], arg.value, labeled);
                                    argvals = argvalitems.map((argvalitem) => {
                                        let argval;
                                        if (dtype(argvalitem, 'literal')) {
                                            argval = argvalitem;
                                        } else {
                                            argval = getname(argvalitem);
                                        }
                                        if (argval == null) {
                                            const argvalitemkeys = Object.keys(argvalitem).filter((key) => key !== 'id');
                                            if (argvalitemkeys.length === 1) {
                                                argval = argvalitem[argvalitemkeys[0]];
                                            }
                                        }
                                        return argval;
                                    });
                                }

                                return testvals.find((testval) => {
                                    if (!argvals) {
                                        if (dtype(testval, 'number') && dtype(arg.value, 'number')) {
                                            testval = Number(testval);
                                            argvals = [Number(arg.value)];
                                        } else {
                                            argvals = [arg.value];
                                        }
                                    }
                                    if (comparator?.startsWith('@')) {
                                        if (dtype(arg.value, 'number')) {
                                            argvals = [Number(arg.value)];
                                        } else if (arg.value in labeled) {
                                            argvals = Number(labeled[arg.value]);
                                            if (!Array.isArray(argvals)) argvals = [argvals];
                                        }
                                        if (comparator.startsWith('@@')) {
                                            testval = currentlist.length - i;
                                            comparator = comparator.slice(2);
                                        } else {
                                            testval = i + 1;
                                            comparator = comparator.slice(1);
                                        }
                                    } else if (['<<', '<<=', '>>', '>>='].includes(comparator) && dtype(arg.value, 'number')) {
                                        const argval = Number(arg.value);
                                        switch (comparator) {
                                            case '<<': return polarize(currentlist.length < argval);
                                            case '<<=': return polarize(currentlist.length <= argval);
                                            case '>>': return polarize(currentlist.length > argval);
                                            case '>>=': return polarize(currentlist.length >= argval);
                                        }
                                    }
                                    if (testval == null || argvals == null || argvals.length === 0) return false;
                                    return argvals.find((argval) => {
                                        if (arg.re) {
                                            return polarize(testval.match(arg.re));
                                        } else {
                                            switch (comparator) {
                                                case null: return polarize(testval === argval);
                                                case '': return polarize(testval === argval);
                                                case '=': return polarize(testval === argval);
                                                case '>=': return polarize(testval >= argval);    
                                                case '<=': return polarize(testval <= argval);
                                                case '>': return polarize(testval > argval);
                                                case '<': return polarize(testval < argval);
                                                case '~': return polarize(testval?.toString().toLowerCase().includes(argval?.toString().toLowerCase()));
                                                case '~<': return polarize(testval?.toString().toLowerCase().startsWith(argval?.toString().toLowerCase()));
                                                case '~>': return polarize(testval?.toString().toLowerCase().endsWith(argval?.toString().toLowerCase()));
                                                case '>>': return polarize(i > arg.matchpos);
                                                case '>>=': return polarize(i >= arg.matchpos);
                                                case '<<': return polarize(i < arg.matchpos);
                                                case '<<=': return polarize(i <= arg.matchpos);
                                            }
                                        }
                                    }) != null;
                                }) != null;
                            }).length > 0;
                        }).length === ands.length;
                    });
                    break;
                case '#': // sort
                    const sortargs = op.args.slice(0);
                    const lastarg = sortargs[sortargs.length - 1];
                    let temped = false;
                    if (!lastarg || ![';', '=;'].includes(Object.values(lastarg).join(''))) sortargs.push(...[{subop: null, value: 'name'}, {subop: null, value: 'id'}]);
                    if (!dtype(currentlist[0], 'object') && lastarg && lastarg.separator === ';') {
                        currentlist = currentlist.map((v) => ({_value: v}));
                        temped = true;
                    } else if (sortargs[0]?.label) {
                        currentlist = currentlist.map((i) => dcopy(i));
                    }
                    sortargs.forEach((arg) => {
                        arg.extraindex = {};
                        const stablesort = (arg.label == null && arg.value == null && arg.separator === ';' ) ? (arg.subop === '-' ? -1 : 1) : null;
                        if (arg.subop?.includes('~')) {
                            arg.sortmode = 'literal';
                        } else if (arg.subop?.endsWith('-')) {
                            arg.sortmode = 'numeric';
                        } else if (arg.subop?.endsWith('+')) {
                            arg.sortmode = 'rank';
                        } else {
                            arg.sortmode = stablesort || ['rank', 'index', 'number', 'id'].includes(arg.value) ? 'rank' : 'numeric';
                        }
                        const vals = new Set();
                        const extradone = new Set();
                        for (let ix=0; ix<currentlist.length; ix++) {
                            const item = currentlist[ix];
                            const vallist = stablesort ? [ix] : (arg.value ? step(item, arg.value, null, labeled) : (Array.isArray(item) ? item : [item]));
                            if (vallist && !stablesort) vallist.forEach((val) => vals.add(val));
                            if (typeof item == 'object') {
                                (item._sortindex ||= []).push(vallist);
                            } else if (!dtype(item, 'number') && !extradone.has(item)) {
                                arg.extraindex[item] = vallist;
                                extradone.add(item);
                            }
                        }
                        if (arg.sortmode !== 'literal') {
                            for (const val of vals) {
                                if (val != null && !dtype(val, 'number') && !(dtype(val, 'object') && dtype(getname(val), 'number'))) {
                                    arg.sortmode = null;
                                    break;
                                }
                            }
                        }
                    })
                    
                    outputlist = currentlist.sort((a, b) => {
                        let comp = 0;
                        let ii = 0;
                        for (const arg of sortargs) {
                            const alist = typeof a == 'object' ? a._sortindex[ii] : dtype(a, 'number') ? [Number(a)] : arg.extraindex[a];
                            const blist = typeof b == 'object' ? b._sortindex[ii] : dtype(b, 'number') ? [Number(b)] : arg.extraindex[b];
                            ii++;
                            if (alist && blist) {
                                if (arg?.subop?.includes('@')) {
                                    const alookup = alist.indexOf(getname(a));
                                    const blookup = blist.indexOf(getname(b));
                                    if (alookup >-1 && blookup === -1) {
                                        comp = -1;
                                    } else if (alookup === -1 && blookup > -1) {
                                        comp = 1;
                                    } else {
                                        comp = alist.indexOf(getname(a)) - alist.indexOf(getname(b))
                                    }
                                } else {
                                    for (let i=0; i < Math.min(alist.length, blist.length); i++) {
                                        let aitem = alist[i];
                                        let bitem = blist[i];
                                        if (aitem != null && bitem == null) {
                                            comp = -1;
                                        } else if (aitem == null && bitem != null) {
                                            comp = 1;
                                        } else {
                                            switch (arg.sortmode) {
                                                case 'literal':
                                                    comp = aitem < bitem ? -1 : (bitem < aitem ? 1 : 0);
                                                    break;
                                                case 'numeric':
                                                    let bnum = Number(bitem);
                                                    let anum = Number(aitem);
                                                    if (isNaN(bnum) || isNaN(anum)) {
                                                        bnum = Number(getname(bitem));
                                                        anum = Number(getname(aitem));
                                                    }
                                                    if (isNaN(bnum) || isNaN(anum)) {
                                                        comp = compvals(aitem, bitem)
                                                    } else {
                                                        comp = bnum - anum;
                                                    }
                                                    break;
                                                case 'rank':
                                                    comp = Number(aitem) - Number(bitem);
                                                    break;
                                                default:
                                                    if (dtype(aitem, 'literal') && dtype(bitem, 'literal')) {
                                                        comp = compvals(aitem, bitem);
                                                    } else if (typeof aitem == 'object' && typeof bitem == 'object') {
                                                        let aval = getname(aitem);
                                                        let bval = getname(bitem);
                                                        if (aval != null && bval != null) {
                                                            comp = compvals(aval, bval);
                                                        } else {
                                                            aval = getid(aitem);
                                                            bval = getid(bitem);
                                                            if (aval != null && bval != null) {
                                                                comp = compvals(aval, bval);
                                                            } else {
                                                                comp = 0;
                                                            }
                                                        }
                                                    } else if (aitem != null && bitem == null) {
                                                        comp = -1;
                                                    } else if (aitem == null && bitem != null) {
                                                        comp = 1;
                                                    }
                                                    break;
                                            }
                                        }
                                        if (comp !== 0) break;
                                    }
                                }
                            }
                            if (comp === 0) {
                                if (alist && blist) {
                                    comp = blist.length - alist.length;
                                } else if (alist) {
                                    comp = -1;
                                } else if (blist) {
                                    comp = 1;
                                }
                            }
                            if (arg?.subop?.includes('-') && (arg.sortmode === 'literal' || !arg.sortmode)) {
                                comp = -comp;
                            }
                            if (comp !== 0) break;
                        }
                        return comp;
                    })
                    if (temped) outputlist = outputlist.map((vt) => vt._value);
                    outputlist.forEach((item, i) => {
                        delete item._sortindex;
                        if (op.args.length > 0 && op.args[0].label) item[op.args[0].label] = i + 1;
                    });
                    break;
                case '/': // group
                case '//': // merge
                    const groupindex = {};
                    let keylists = {};
                    let ofname = 'of';
                    let countname = 'count';
                    let sortgroups = true;
                    const groupargs = [];
                    const merge_ands = [];
                    for (const arg of op.args) {
                        if (arg.separator === ';' && arg.label == null && arg.value == null) {
                            sortgroups = false;
                        } else if (op.operator === '//' && (arg.separator === ';' || merge_ands.length > 0)) {
                            if (arg.separator === ';') {
                                merge_ands.push([arg.value]);
                            } else {
                                merge_ands[merge_ands.length - 1].push(arg.value)
                            }
                        } else {
                            groupargs.push(arg);
                        }
                    }
                    if (groupargs.length === 0) groupargs.push({value: null});
                    
                    groupargs.forEach((arg) => arg.groupcounter = 0);
                    
                    const grouptimer = {};
                    for (let ix=0; ix<currentlist.length; ix++) {
                        this.timecheck(grouptimer, ix, currentlist.length, op);
                        const item = currentlist[ix];
                        let keys = null;
                        let keyi = 0;
                        for (const arg of groupargs) {
                            if (arg.label === 'of') {
                                ofname = arg.value;
                                continue;
                            } else if (arg.label === 'count') {
                                countname = arg.value;
                                continue;
                            }
                            keyi++;
                            let groupnumber = null;
                            if (op.operator === '/' && dtype(arg.value, 'number')) {
                                if (arg.subop?.endsWith('@')) {
                                    arg.divisor = Number(arg.value);
                                } else {
                                    arg.divisor = currentlist.length / Number(arg.value);   
                                }
                                groupnumber = Math.floor(ix / arg.divisor) + 1;
                            } else if (arg.value != null && arg.subop?.endsWith('@@')) {
                                const groupval = step(item, arg.value, null, labeled);
                                if (ix === 0 || groupval?.length > 0) arg.groupcounter += 1;
                                groupnumber = arg.groupcounter;
                            }
                            const label = arg.label ?? (typeof arg.value === 'string' ? arg.value : null) ?? keyi;
                            const newkeyitems = groupnumber != null ? [groupnumber] : arg.value ? step(item, arg.value, arg.subop, labeled) : [null];
                            const newkeys = newkeyitems.map((newkey) => ([{arglabel: arg.label, label: label, keyitem: this.resolve(arg.value, newkey, labeled)}]));
                            if (keys) {
                                keys = keys.flatMap((oldkeys) => newkeys.filter((newkey) => arg.separator === ',' || oldkeys.filter((oldkey) => compvals(getname(oldkey.keyitem) || oldkey.keyitem, getname(newkey[0].keyitem) || newkey[0].keyitem) >= 0).length === 0).map((newkey) => oldkeys.concat(newkey)));
                            } else {
                                keys = newkeys;
                            }
                            if (arg.subop?.endsWith('@')) {
                                keys.forEach((key) => {
                                    const testkey = JSON.stringify(key.slice(0, -1));
                                    const newkeytest = JSON.stringify(key[key.length - 1]);
                                    if (testkey in keylists) {
                                        const lastkey = keylists[testkey][keylists[testkey].length - 1];
                                        if (newkeytest !== lastkey) keylists[testkey].push(newkeytest);
                                    } else {
                                        keylists[testkey] = [newkeytest];
                                    }
                                    key[key.length - 1].keyindex = keylists[testkey].length;
                                });
                            }
                        }
                        if (keys) {
                            for (const key of keys) {
                                const keystr = JSON.stringify(key);
                                (groupindex[keystr] ||= []).push(item);
                            }
                        }
                    }
                    for (const [keystr, items] of Object.entries(groupindex)) {
                        if (op.operator === '/') {
                            const newgroup = {};
                            const keydata = JSON.parse(keystr);
                            if (keydata.length === 1 && keydata[0].keyitem != null) {
                                let keyitemname = keydata[0].keyindex || getname(keydata[0].keyitem);
                                if (keyitemname != null) {
                                    newgroup.name = keyitemname;
                                }
                            }
                            let skip = false;
                            let keys = [];
                            for (const {arglabel, label, keyitem, keyindex} of keydata) {
                                if (keyitem != null) {
                                    const keyobj = (dtype(keyitem, 'object') || keyindex == null || false) ? keyitem : {name: keyitem};
                                    if (keyindex != null) {
                                        keyobj.index = keyindex;
                                    }
                                    keys.push(keyobj)
                                    if (label && dtype(label, 'string') && isNaN(label) && label !== '_') {
                                        newgroup[label] = [keyobj];
                                    }
                                }
                            }
                            newgroup[countname] = items.length;
                            if (keys) newgroup.key = keys;
                            newgroup[ofname] = items;
                            outputlist.push(newgroup);
                        } else if (op.operator === '//') {
                            if (merge_ands.length === 0 || !merge_ands.find((mand) => !mand.find((mr) => items.find((item) => mr in item)))) {
                                const newgroup = items.reduce((acc, item) => {
                                    for(const prop in item) {
                                        if (prop in acc) {
                                            if (Array.isArray(acc[prop])) {
                                                (Array.isArray(item[prop]) ? item[prop] : [item[prop]]).filter((val) => !acc[prop].includes(val)).forEach((val) => acc[prop].push(val));
                                            }
                                        } else {
                                            acc[prop] = item[prop];
                                        }
                                    }
                                    return acc;
                                }, {});
                                const keydata = JSON.parse(keystr);
                                for (const {arglabel, label, keyitem, keyindex} of keydata) {
                                    if (arglabel && typeof arglabel === 'string' && arglabel !== '_') {
                                        newgroup[arglabel] = [keyitem];
                                    }
                                }
                                outputlist.push(newgroup);
                            }
                        }
                    }
                    if (sortgroups) {
                        const sortquery = '#' + groupargs.map((arg, argx) => (arg.subop?.endsWith('@') || arg.divisor ? '+' : '') + '(..key:@' + (argx + 1) + (arg.subop?.endsWith('@') ? '.index;_' : '') + ')').join(',');
                        outputlist = this.execute(outputlist, this.assemble(this.tokenize(sortquery)), labeled);
                    }
                    break;
                case '...': // aggregate
                case '....': // aggregate to value
                    if (!(op?.args?.length > 0)) {
                        if (op.operator === '...') {
                            outputlist = [{of: currentlist.map((item) => dcopy(item))}];
                        } else {
                            outputlist = [currentlist.length]
                        }
                        break;
                    }
                    const tempitem = {};
                    if (op.operator === '...') {
                        tempitem.of = currentlist.map((item) => dcopy(item));
                    } else {
                        tempitem.of = currentlist;
                    }
                    const finalitem = {};
                    let aggregated = null;
                    let afteraggregated = 0;
                    let finalvalue = null;
                    op.args.forEach((arg, propx) => {
                        if (dtype(arg.value, 'string') && arg.subop === '~') {
                            tempitem[arg.label || '_' + (propx + 1).toString()] = [arg.value];
                        } else {
                            const prop = arg.label ?? (typeof arg.value === 'string' ? arg.value : null) ?? '_' + (propx + 1).toString();
                            const aggname = (arg.label == null && typeof arg.value == 'string' && arg.value) || (arg.value == null && arg.label);
                            const firstitem = currentlist[0];
                            const isprop = typeof firstitem == 'object' && prop in firstitem;
                            if ((arg.separator === ';' || !isprop) && aggname in this.aggregators) {
                                const aggval = this.aggregators[aggname](tempitem);
                                const agglabel = arg.label || aggname;
                                if (aggval != null) finalitem[agglabel] = aggval;
                                aggregated = agglabel;
                                finalvalue = dcopy(aggval);
                            } else if (!aggregated) {
                                let propitems;
                                if (op === operations[0] && op.operator === '....' && arg.label && !Array.isArray(arg.value) && !arg.value.startsWith('=')) {
                                    propitems = [arg.value]
                                } else {
                                    propitems = this.execute(currentlist, Array.isArray(arg.value) ? arg.value.map((x) => structuredClone(x)) : [{operator: '..', args: [{value: arg.value}]}], labeled)
                                }
                                tempitem[prop] = propitems;
                            } else if (aggregated in finalitem && Array.isArray(finalitem[aggregated]) && finalitem[aggregated].length > afteraggregated) {
                                tempitem[prop] = finalitem[aggregated][afteraggregated];
                                afteraggregated++;
                            }
                        }
                    });
                    if (op.operator === '....' && finalvalue != null) {
                        if (this.dtype(finalvalue, 'array')) {
                            outputlist = finalvalue;
                        } else {
                            outputlist = [finalvalue];
                        }
                    } else {
                        for (const prop in tempitem) {
                            if (!(prop in finalitem) && prop !== 'of') finalitem[prop] = tempitem[prop];
                        }
                        if (op.operator === '...') {
                            finalitem.of = tempitem.of;
                        } else {
                            for (const prop in finalitem) {
                                if (Array.isArray(finalitem[prop]) && finalitem[prop].length === 1) finalitem[prop] = finalitem[prop][0];
                            }
                        }
                        outputlist = [finalitem];
                    }
                    break;
                case '|': // annotate
                case '||': // annotate with values
                    outputlist = currentlist.slice(0).map((item) => dcopy(item));
                    op.args.filter((arg) => arg.subop?.includes('>') && arg.label != null && true && Array.isArray(arg.value))
                        .forEach((arg) => (labeled['=>'] ??= {})[arg.label] = arg.value);
                        
                    const annotatetimer = {};
                    for (const i in outputlist) {
                        this.timecheck(annotatetimer, i, outputlist.length, op);
                        const baseitem = outputlist[i];
                        if (typeof baseitem != 'object') {
                            outputlist[i] = {};
                            if (op.args?.[0]?.value !== '_') outputlist[i].name = baseitem;
                        }
                        const item = outputlist[i];
                        const newprops = op.args.filter((arg) => arg.subop !== '<' && (arg.label || !arg.subop?.includes('-'))).map((arg) => arg.label || arg.value);
                        if (typeof item == 'object') {
                            let argx = -1;
                            for (const arg of op.args) {
                                argx++;
                                if (arg?.subop === '<' && dtype(arg.value, 'string') && Array.isArray(item[arg.value]) && argx < op.args.length - 1) {
                                    const sublabeled = {...labeled};
                                    if (arg.label) sublabeled[arg.label] = [item];
                                    item[arg.value] = dactal.execute(item[arg.value], [{operator: op.operator, args: op.args.slice(argx + 1).map((arg) => structuredClone(arg))}], sublabeled);
                                    break;
                                } else if (arg.separator === ';' && argx === op.args.length - 1 && arg.label == null && arg.value == null && arg.subop == null) {
                                    for (const oldprop in item) {
                                        if (!(newprops.includes(oldprop))) {
                                            const tempval = item[oldprop];
                                            delete item[oldprop];
                                            item[oldprop] = tempval;
                                        }
                                    }
                                } else if (arg.subop?.endsWith('-')) {
                                    if (arg.value && typeof arg.value == 'string' && arg.value in item) {
                                        if (arg.label && typeof arg.label == 'string') item[arg.label] = item[arg.value];
                                        delete item[arg.value];
                                    } else if (arg.label && Array.isArray(arg.value)) {
                                        const toremoveids = new Set(step(item, arg.value, null, labeled).map((subitem) => getid(subitem)));
                                        item[arg.label] = item[arg.label].filter((subitem) => !toremoveids.has(getid(subitem)));                                  
                                    }
                                } else if (argx === 0 && arg.label != null && arg.value === '_') {
                                    item[arg.label] = op.operator === '||' ? baseitem : [baseitem];
                                } else {
                                    if (!arg.label && arg.value && typeof arg.value === 'string' && arg.value !== '') {
                                        const moveprop = arg.value;
                                        const value = structuredClone(item[moveprop]);
                                        delete item[moveprop];
                                        item[moveprop] = value;
                                    }
                                    if (arg.label) {
                                        let vals = [];
                                        if (arg.subop.endsWith('@')) {
                                            if (arg.subop.endsWith('@@')) {
                                                arg.counter ??= arg.value == null ? outputlist.length + 1 : 0;
                                            } else {
                                                arg.counter ??= 1;
                                                vals = [arg.counter];
                                            }
                                            if (Array.isArray(arg.value)) {
                                                arg.counter += step(item, arg.value, null, labeled).length;
                                            } else if (dtype(arg.value, 'string') && dtype(item[arg.value], 'number')) {
                                                arg.counter += Number(item[arg.value]);
                                            } else {
                                                arg.counter += arg.subop.endsWith('@@') ? -1 : 1;
                                            }
                                            if (arg.subop.endsWith('@@')) {
                                                vals = [arg.counter];
                                            }
                                        } else {
                                            vals = step(item, arg.value, arg.subop, labeled);
                                        }
                                        const base = (arg.subop?.endsWith('+') && arg.label in item && Array.isArray(item[arg.label])) ? item[arg.label] : [];
                                        if (this.dtype(vals, 'array')) {
                                            if (op.operator === '|') {
                                                item[arg.label] = base.concat(vals.map((v) => dcopy(v)));
                                            } else if (vals.length > 0 ) {
                                                if (typeof vals[0] === 'object') {
                                                    item[arg.label] = getname(vals[0]) != null ? getname(vals[0]) : vals[0].id != null ? vals[0].id : dcopy(vals[0]);
                                                } else {
                                                    item[arg.label] = vals[0];
                                                }
                                            }
                                        } else if (vals) {
                                            item[arg.label] = base.concat(vals);
                                        }
                                    }
                                }
                            }
                        }
                    }
                    break;
                case '???':
                    outputlist = currentlist;
                    const commentval = op.args?.[0]?.value;
                    if (commentval in labeled) console.log({[commentval]: labeled[commentval]});
                    if (commentval === 'end') return (this.debug && !inputlist) ? operations : outputlist;
                    break;
                default:
                    outputlist = []
            }
            currentlist = outputlist.slice(0);
            if (this.debug && !inputlist) op.results = outputlist.slice(0, typeof this.debug == 'number' ? this.debug : outputlist.length);
        }
        return (this.debug && !inputlist) ? operations : outputlist;
    }

    gettype(value, allow_relative=true) {
        if (value == null) return null;
        const trylist = [value];
        if (value.endsWith('s')) {
            trylist.push(value.slice(0, value.length - 1));
            trylist.push(value + 'es');
        } else {
            trylist.push(value + 's');
        }
        for (const tryval of trylist) {
            if (tryval in this.data) {
                return this.data[tryval];
            }
        }
        if (this.savedquerynames.has(value)) {
            const savedqueries = this.data.queries.filter((q) => q.name === value);
            if (savedqueries?.length === 1) {
                const sq = savedqueries[0];
                if (allow_relative && sq.query.startsWith('?')) {
                    return sq;
                } else {
                    if (!sq.results) sq.results = this.executeq(sq.query);
                    return sq.results.slice(0);
                }
            }
        }
        return null;
    }

    dcopy = (item) => {
        if (this.dtype(item, 'literal')) return item;
        if (this.dtype(item, 'array')) return item.slice(0);
        return Object.assign({}, item);
    }
    
    dtype(item, test=null) {
        let type = null;
        if (item && Array.isArray(item)) {
            type = 'array';
        } else if (item && typeof item === 'object') {
            type = 'object';
        } else if (typeof item === 'string' || typeof item === 'number' || typeof item === 'boolean') {
            type = 'literal';
        }
        if (test) {
            if (test === 'number') {
                return type === 'literal' && !isNaN(item);
            } else if (test === 'string') {
                return type === 'literal' && typeof item === 'string';
            } else if (test === 'boolean') {
                return type === 'literal' && typeof item === 'boolean';
            }
            return type === test;
        }
        return type;
    }

    getname(item) {
        if (typeof item === 'object' && item != null) {
            if (item?.name != null) {
                return item.name;
            } else if (item?.key?.length > 0) {
                return item.key.join(' / ');
            }
            const {id, ...nonidprops} = item;
            const names = Object.values(nonidprops).filter((val) => typeof val === 'string' || typeof val === 'number');
            if (names.length > 0) {
                return names[0];
            }
            for(const key in nonidprops) {
                if (Array.isArray(item[key]) && item[key].length === 1 && typeof item[key][0] == 'string') {
                    return item[key][0];
                }
            }
        } else if (typeof item === 'string' || typeof item === 'number') {
            return item;
        } else if (typeof item === 'boolean') {
            return item.toString();
        }
        return '';
    }
    
    getid(item) {
        if (typeof item === 'object' && item != null) {
            const iditems = (item.id || item.uri || item.name) ? [item] : item.key?.length > 0 ? item.key : Object.keys(item).length === 3 && item.of?.length > 0 ? item.of : [item];
            return iditems.map((item) => {
                if (item.id) {
                    if (Array.isArray(item.id)) {
                        return item.id[0];
                    } else {
                        return item.id;
                    }
                } else if (item.uri) {
                    if (Array.isArray(item.uri)) {
                        return item.uri[0];
                    } else {
                        return item.uri;
                    }
                } else if (item.name) {
                    return item.name;
                } else {
                    // if (stringifiedid.length > 128) console.warn({idstringify: item, idlength: stringifiedid.length});
                    return JSON.stringify(item);
                }
            }).join(',');
        } else if (typeof item === 'string' || typeof item === 'number') {
            return item;
        }
        return null;
    }
    
    step = (item, property, subop, labeled) => {
        // (this.data.trace ??= []).push({stepitem: item, property: property, subop: subop, labeled: labeled});
        const dtype = this.dtype;
        const dcopy = this.dcopy;
        const resolve = this.resolve;
        const escapeRegExp = this.escapeRegExp;
        const vals = [];
        if (subop?.includes('~') && dtype(property, 'literal')) {
            vals.push(property);
        } else if (Array.isArray(property)) {
            this.execute([item], property.map((x) => structuredClone(x)), labeled).forEach((val) => vals.push(val));
        } else if (property === '_') {
            vals.push(dcopy(item));
        } else if (item?.toString()?.match(/^-?\d+$/) && property.match(/^\d+$/)) {
            vals.push((item === 0 && 1 / item === -Infinity ? -1 : 1) * Number(`${item}.${property}`));
        } else if (item?.of && dtype(property, 'number')) {
            const propval = Number(property);
            (subop === '-' || propval < 0 ? item.of.slice(-1 * Math.abs(Number(property))) : item.of.slice(0, Math.abs(Number(property)))).forEach((val) => vals.push(val));
        } else if (property === 'id' && dtype(item, 'literal')) {
            vals.push(item);
        } else if (property === 'name') {
            vals.push(this.getname(item));
        } else if ((dtype(item, 'number') || dtype(item, 'object')) && property.startsWith('=')) {
            let calculation = property.slice(1);
            const mathwords = Object.getOwnPropertyNames(Math).filter((mathword) => mathword.match(/^[a-z0-9]+$/)).sort((a, b) => b.length - a.length || a.localeCompare(b));
            const otherwords = ['split'];
            const variables = Object.entries(item).concat(Object.entries(labeled))
                .map(([k, v]) => k)
                .sort((a, b) => b.length - a.length || a.localeCompare(b));
            if (dtype(this.getname(item), 'number')) variables.push('_');
            const allowedwords = mathwords.concat(otherwords).concat(variables).sort((a, b) => b.length - a.length || a.localeCompare(b));
            const allowed = new RegExp(`^((\b(${allowedwords.map((w) => escapeRegExp(w)).join('|')})\b)|([0-9_\+\/\*\(\)\[\].%=,'" -]*))*$`);
            let val;
            if (calculation.match(allowed)) {
                for (const variable of variables) {
                    const variableex = new RegExp(`\\b${variable}\\b`, 'g');
                    if (calculation.match(variableex)) {
                        let vval = variable === '_' ? Number(this.getname(item)) : item[variable] ?? labeled[variable];
                        if (Array.isArray(vval) && vval.length === 1) vval = vval[0];
                        calculation = calculation.replaceAll(variableex, dtype(vval, 'number') ? vval : JSON.stringify(vval));
                    }
                }
                if (calculation.match(/[A-Za-z]/)) {
                    for (const mathword of mathwords) {
                        const mathwordex = new RegExp(`\\b${mathword}\\b`, 'g');
                        calculation = calculation.replaceAll(mathwordex, `Math.${mathword}`);
                        if (!calculation.match(/[A-Za-z]/)) break;
                    }
                }
                calculation = calculation.replaceAll(/\b=\b/g, '==');
                try {
                    val = eval(calculation);
                } catch (error) {
                    val = calculation;
                }
            } else {
                val = calculation;
            }
            if (val !== false) vals.push(val);
        } else if (item) {
            let found = false;
            if (property != null && !dtype(item, 'literal')) {
                const tryvals = [property];
                tryvals.push(property.endsWith('s') ? property.slice(0, -1) : property + 's');
                if (property.match(/ /)) tryvals.push(property.replaceAll(/ /g, '_'));
                for (const tryval of tryvals) {
                    if (tryval in item) {
                        const resolved = resolve(tryval, item[tryval], labeled);
                        if (Array.isArray(resolved)) {
                            resolved.forEach((x) => vals.push(x));
                        } else {
                            vals.push(resolved);
                        }
                        found = true;
                        break;
                    }
                }
            }
            if (!found && labeled?.['=>']?.[property]) {
                this.execute([item], labeled['=>'][property], labeled).forEach((val) => vals.push(val));
                found = true;
            }
            if (!found && (property in this.data || property in this.adapters || this.savedquerynames.has(property) || property in labeled)) {
                let typenav;
                if (!dtype(item, 'literal')) {
                    if (this.adapters?.[property]?.annotator) {
                        typenav = resolve(property, item, labeled);
                    // } else if (property in this.aggregators) {
                    //     typenav = this.aggregators[property](item);
                    } else if ('id' in item || 'uri' in item) {
                        typenav = resolve(property, this.getid(item), labeled);
                    } else if (this.savedquerynames.has(property)) {
                        const relative_query = dactal.data.queries.find((q) => q.relative && q.name === property);
                        if (relative_query) {
                            typenav = this.step(item, this.parse(relative_query.query), subop, labeled);
                        } else {
                            typenav = this.getname(item);
                        }
                    } else {
                        typenav = this.getname(item);
                    }
                    if (typenav) vals.push(typenav);
                } else {
                    typenav = resolve(property, item, labeled);
                    if (typenav) {
                        if (Array.isArray(typenav)) {
                            typenav.forEach((t) => vals.push(t));
                        } else {
                            vals.push(typenav);
                        }
                    }
                }
            }
        }
        return vals;
    }

    resolve = (property, item, labeled) => {
        const dtype = this.dtype;
        if (dtype(item, 'object')) {
            if (this.adapters?.[property]?.annotator) {
                if ((!this.recache || this.recache.has(property)) && property in this.index && this.getid(item) in this.index[property]) {
                    return this.index[property][this.getid(item)];
                } else {
                    this.adapters[property].queue.push(this.adapters[property].annotator ? item : this.getid(item));
                    return null;
                }
            } else {
                return item;
            }
        } else if (dtype(item, 'literal') && typeof property == 'string' && (property in labeled || this.destinations.has(property) || this.destinations.has(property + 's'))) {
            if ((!this.recache || this.recache.has(property)) && property in this.index && item in this.index[property]) {
                // this.cachehitcount++;
                return this.index[property][item];
            }
            const typeitems = labeled[property] ?? this.gettype(property);
            if (Array.isArray(typeitems)) {
                for (const lookupkey of ['id', 'uri', 'name', property, property + 's']) {
                    const found = typeitems.filter((typeitem) => {
                        return dtype(typeitem, 'object') && (lookupkey in typeitem) && (typeitem[lookupkey] === item || (Array.isArray(typeitem[lookupkey]) && typeitem[lookupkey].length === 1 && typeitem[lookupkey][0] === item));
                    });
                    if (found.length === 1) {
                        (this.index[property] ||= {})[item] = found[0];
                        return found[0];
                    }
                }
                if (property in this.adapters && (!this.recache || !this.recache.has(property))) {
                    this.adapters[property].queue.push(this.adapters[property].annotator ? item : this.getid(item));
                }
                return null;
            } else if (typeitems && typeof typeitems == 'object' && 'query' in typeitems) {
                const relative_query = this.assemble(this.tokenize(typeitems.query));
                while (['?', ':'].includes(relative_query[0].operator)) {
                    relative_query.shift()
                }
                return this.execute([item], relative_query, labeled);
            }
            if (property in this.adapters) {
                this.adapters[property].queue.push(this.adapters[property].annotator ? item : this.getid(item));
                return null;
            }
        }
        return item;
    }
    
    samearray(a, b) {
        if ((a && !b) || (!a && b) || a.length !== b.length) return false;
        for(let x=0; x<a.length; x++) {
            if (this.getid(a[x]) !== this.getid(b[x])) return false;
        }
        return true;
    }
    
    async verify(i = null) {
        let toverify = this.data.queries;
        if (!isNaN(i)) toverify = toverify.slice(i - 1, i);
        let allsame = true;
        for (const savedq of toverify) {
            console.log('verifying ' + savedq.name);
            const testres = await this.query(savedq.query);
            if (testres.length !== savedq.results.length) {
                console.log('--x result count changed from ' + savedq.results.length + ' to ' + testres.length);
                allsame = false;
            } else {
                for (let i=0; i<testres.length; i++) {
                    const testrow = testres[i];
                    const savedrow = savedq.results[i];
                    if (typeof savedrow == 'object') {
                        for (const prop in savedrow) {
                            if (!(prop in testrow)) {
                                console.log('--x row ' + (i+1) + ': new results missing property ' + prop);
                                allsame = false;
                            } else {
                                const testval = JSON.stringify(testrow[prop]);
                                const savedval = JSON.stringify(savedrow[prop]);
                                if (testval !== savedval) {
                                    console.log('--x row ' + (i+1) + ': different value for property ' + prop);
                                    console.log({was: savedrow[prop], now: testrow[prop]})
                                    allsame = false;
                                }
                            }
                        }
                    } else {
                        if (testrow !== savedrow) {
                            console.log('--x row ' + (i+1) + ': different value');
                            console.log({was: savedrow, now: testrow});
                            allsame = false;
                        }
                    }
                }
            }
            if (allsame) console.log('--- results unchanged');
        }
    }
    
    index_check() {
        console.table(Object.entries(this.index).map(([key, vals]) => ({key: key, vals: Object.keys(vals).length, size: Object.keys(vals).length * JSON.stringify(Object.entries(vals).slice(0, 1)).length})).sort((a, b) => b.size - a.size || b.vals - a.vals || a.key.localeCompare(b.key)))
    }
    
    index_materialize() {
        this.load(Object.keys(this.index).flatMap((i) => Object.keys(this.index[i]).flatMap((k) => ({index: i, indexed: k, value: this.index[i][k]}))), 'index contents') 
    }
}