<commit_before>/**
 * Copyright (c) 2017 mol* contributors, licensed under MIT, See LICENSE file for more info.
 *
 * @author <NAME> <<EMAIL>>
 */

import * as B from 'benchmark'

import * as util from 'util'
import * as fs from 'fs'
import CIF from 'mol-io/reader/cif'

import { Structure, Model, Queries as Q, Atom, AtomGroup, AtomSet, Selection, Symmetry } from 'mol-model/structure'
import { Segmentation } from 'mol-data/int'

import to_mmCIF from 'mol-model/structure/export/mmcif'

require('util.promisify').shim();
const readFileAsync = util.promisify(fs.readFile);

async function readData(path: string) {
    if (path.match(/\.bcif$/)) {
        const input = await readFileAsync(path)
        const data = new Uint8Array(input.byteLength);
        for (let i = 0; i < input.byteLength; i++) data[i] = input[i];
        return data;
    } else {
        return readFileAsync(path, 'utf8');
    }
}

(Symbol as any).asyncIterator = (Symbol as any).asyncIterator || Symbol.for('Symbol.asyncIterator');

interface ProgressGenerator<T> extends AsyncIterableIterator<number | T> {
    next(cont?: boolean): Promise<IteratorResult<number | T>>
}

async function *test(): ProgressGenerator<boolean> {
    const r = yield await new Promise<number>(res => res(10));
    return r;
}

async function runIt(itP: () => ProgressGenerator<boolean>) {
    const it = itP();
    while (true) {
        const { value, done } = await it.next(true);
        if (done) return value;
    }
}

runIt(test).then(r => console.log('rerdasdasda', r))

export async function readCIF(path: string) {
    console.time('readData');
    const input = await readData(path)
    console.timeEnd('readData');

    console.time('parse');
    const comp = typeof input === 'string' ? CIF.parseText(input) : CIF.parseBinary(input);
    const parsed = await comp();
    console.timeEnd('parse');
    if (parsed.isError) {
        throw parsed;
    }

    const data = parsed.result.blocks[0];
    console.time('schema')
    const mmcif = CIF.schema.mmCIF(data);

    console.timeEnd('schema')
    console.time('buildModels')
    const models = Model.create({ kind: 'mmCIF', data: mmcif });
    console.timeEnd('buildModels')
    const structures = models.map(Structure.ofModel);

    return { mmcif, models, structures };
}

export namespace PropertyAccess {
    function baseline(model: Model) {
        const atom_site = model.sourceData.data.atom_site;
        const id = atom_site.id.value;
        let s = 0;
        for (let i = 0, _i = atom_site._rowCount; i < _i; i++) {
            s += id(i);
        }
        return s;
    }

    function sumProperty(structure: Structure, p: Atom.Property<number>) {
        const { atoms, units } = structure;
        const unitIds = AtomSet.unitIds(atoms);
        const l = Atom.Location();

        let s = 0;

        for (let i = 0, _i = unitIds.length; i < _i; i++) {
            l.unit = units[unitIds[i]];
            const set = AtomSet.unitGetByIndex(atoms, i);


            for (let j = 0, _j = AtomGroup.size(set); j < _j; j++) {
                l.atom = AtomGroup.getAt(set, j);
                s += p(l);
            }
        }

        return s;
    }

    function sumPropertySegmented(structure: Structure, p: Atom.Property<number>) {
        const { atoms, units } = structure;
        const unitIds = AtomSet.unitIds(atoms);
        const l = Atom.Location();

        let s = 0;

        let vA = 0, cC = 0, rC = 0;
        for (let i = 0, _i = unitIds.length; i < _i; i++) {
            const unit = units[unitIds[i]];
            l.unit = unit;
            const set = AtomSet.unitGetByIndex(atoms, i);

            const chainsIt = Segmentation.transientSegments(unit.hierarchy.chainSegments, set.atoms);
            const residues = unit.hierarchy.residueSegments;
            while (chainsIt.hasNext) {
                cC++;

                const chainSegment = chainsIt.move();
                const residuesIt = Segmentation.transientSegments(residues, set.atoms, chainSegment);
                while (residuesIt.hasNext) {
                    rC++;
                    const residueSegment = residuesIt.move();
                    // l.atom = OrdSet.getAt(set, residueSegment.start);
                    // console.log(unit.hierarchy.residues.auth_comp_id.value(unit.residueIndex[l.atom]), l.atom, OrdSet.getAt(set, residueSegment.end))
                    for (let j = residueSegment.start, _j = residueSegment.end; j < _j; j++) {
                        l.atom = AtomGroup.getAt(set, j);
                        vA++;
                        s += p(l);
                    }
                }
            }
        }

        console.log('seg atom count', vA, cC, rC);

        return s;
    }

    function sumPropertyResidue(structure: Structure, p: Atom.Property<number>) {
        const { atoms, units } = structure;
        const unitIds = AtomSet.unitIds(atoms);
        const l = Atom.Location();

        let s = 0;

        for (let i = 0, _i = unitIds.length; i < _i; i++) {
            const unit = units[unitIds[i]];
            l.unit = unit;
            const set = AtomSet.unitGetByIndex(atoms, i);
            const residuesIt = Segmentation.transientSegments(unit.hierarchy.residueSegments, set.atoms);
            while (residuesIt.hasNext) {
                l.atom = AtomGroup.getAt(set, residuesIt.move().start);
                s += p(l);
            }
        }

        return s;
    }

    function sumPropertyAtomSetIt(structure: Structure, p: Atom.Property<number>) {
        const { atoms, units } = structure;

        let s = 0;
        const atomsIt = AtomSet.atoms(atoms);
        const l = Atom.Location();
        while (atomsIt.hasNext) {
            const a = atomsIt.move();
            l.unit = units[Atom.unit(a)];
            l.atom = Atom.index(a);
            s += p(l);
        }
        return s;
    }

    // function sumPropertySegmentedMutable(structure: Structure, p: Property<number>) {
    //     const { atoms, units } = structure;
    //     const unitIds = AtomSet.unitIds(atoms);
    //     const l = Property.createLocation();

    //     let s = 0;

    //     for (let i = 0, _i = unitIds.length; i < _i; i++) {
    //         const unit = units[unitIds[i]];
    //         l.unit = unit;
    //         const set = AtomSet.unitGetByIndex(atoms, i);

    //         const chainsIt = Segmentation.transientSegments(unit.hierarchy.chainSegments, set);
    //         const residuesIt = Segmentation.transientSegments(unit.hierarchy.residueSegments, set);
    //         while (chainsIt.hasNext) {
    //             const chainSegment = chainsIt.move();
    //             residuesIt.updateRange(chainSegment);
    //             while (residuesIt.hasNext) {
    //                 const residueSegment = residuesIt.move();
    //                 for (let j = residueSegment.start, _j = residueSegment.end; j < _j; j++) {
    //                     l.atom = OrdSet.getAt(set, j);
    //                     s += p(l);
    //                 }
    //             }
    //         }
    //     }

    //     return s;
    // }

    function sumDirect(structure: Structure) {
        const { atoms, units } = structure;
        const unitIds = AtomSet.unitIds(atoms);

        let s = 0;

        for (let i = 0, _i = unitIds.length; i < _i; i++) {
            const unitId = unitIds[i];
            const unit = units[unitId];
            const set = AtomSet.unitGetByIndex(atoms, i);
            //const { residueIndex, chainIndex } = unit;
            const p = unit.conformation.atomId.value;
            for (let j = 0, _j = AtomGroup.size(set); j < _j; j++) {
                const aI = AtomGroup.getAt(set, j);
                s += p(aI);
            }
        }

        return s;
    }

    // function concatProperty(structure: Structure, p: Property<string>) {
    //     const { atoms, units } = structure;
    //     const unitIds = AtomSet.unitIds(atoms);
    //     const l = Property.createLocation(structure);

    //     let s = [];

    //     for (let i = 0, _i = unitIds.length; i < _i; i++) {
    //         const unitId = unitIds[i];
    //         l.unit = units[unitId];
    //         const set = AtomSet.unitGetByIndex(atoms, i);
    //         const { residueIndex, chainIndex } = l.unit;

    //         for (let j = 0, _j = OrdSet.size(set); j < _j; j++) {
    //             const aI = OrdSet.getAt(set, j);
    //             l.atom = aI;
    //             l.residueIndex = residueIndex[aI];
    //             l.chainIndex = chainIndex[aI];
    //             s[s.length] = p(l);
    //         }
    //     }

    //     return s;
    // }

    export function write(s: Structure) {
        console.log(to_mmCIF('test', s));
    }

    export function testAssembly(id: string, s: Structure) {
        console.time('assembly')
        const a = Symmetry.buildAssembly(s, '1');
        console.timeEnd('assembly')
        fs.writeFileSync(`e:/test/molstar/${id}_assembly.bcif`, to_mmCIF(id, a, true));
        console.log('exported');
        //write(a);
    }

    export async function run() {
        //const { structures, models, mmcif } = await readCIF('./examples/1cbs_full.bcif');
        const { structures, models } = await readCIF('e:/test/quick/3j3q_full.bcif');
        //const { structures, models, mmcif } = await readCIF('e:/test/quick/1cbs_updated.cif');
        //const { structures, models/*, mmcif*/ } = await readCIF('e:/test/quick/5j7v_updated.cif');

        //console.log(mmcif.pdbx_struct_oper_list.matrix.toArray());
        // console.log(mmcif.pdbx_struct_oper_list.vector.toArray());

        // testAssembly('5j7v', structures[0]);
        // throw '';

        // console.log(models[0].symmetry.assemblies);


        //const { structures, models } = await readCIF('e:/test/molstar/3j3q.bcif');

        // fs.writeFileSync('e:/test/molstar/3j3q.bcif', to_mmCIF('test', structures[0], true));
        // return;

        // console.log(toMmCIFString('test', structures[0]));

        // return;

        console.log('bs', baseline(models[0]));
        console.log('sp', sumProperty(structures[0], l => l.unit.model.conformation.atomId.value(l.atom)));
        console.log(sumPropertySegmented(structures[0], l => l.unit.model.conformation.atomId.value(l.atom)));

        //console.log(sumPropertySegmentedMutable(structures[0], l => l.unit.model.conformation.atomId.value(l.atom)));
        console.log(sumPropertyAtomSetIt(structures[0], l => l.unit.model.conformation.atomId.value(l.atom)));
        //console.log(sumProperty(structures[0], Property.cachedAtomColumn(m => m.conformation.atomId)));
        console.log(sumDirect(structures[0]));
        console.log('r', sumPropertyResidue(structures[0], l => l.unit.hierarchy.residues.auth_seq_id.value(l.unit.residueIndex[l.atom])));<commit_msg>added on-demand file loading to structure perf-test<commit_after>/**
 * Copyright (c) 2017 mol* contributors, licensed under MIT, See LICENSE file for more info.
 *
 * @author <NAME> <<EMAIL>>
 */

import * as B from 'benchmark'

import * as util from 'util'
import * as fs from 'fs'
import fetch from 'node-fetch'
import CIF from 'mol-io/reader/cif'

import { Structure, Model, Queries as Q, Atom, AtomGroup, AtomSet, Selection, Symmetry } from 'mol-model/structure'
import { Segmentation } from 'mol-data/int'

import to_mmCIF from 'mol-model/structure/export/mmcif'

require('util.promisify').shim();
const readFileAsync = util.promisify(fs.readFile);
const writeFileAsync = util.promisify(fs.writeFile);

async function readData(path: string) {
    if (path.match(/\.bcif$/)) {
        const input = await readFileAsync(path)
        const data = new Uint8Array(input.byteLength);
        for (let i = 0; i < input.byteLength; i++) data[i] = input[i];
        return data;
    } else {
        return readFileAsync(path, 'utf8');
    }
}

(Symbol as any).asyncIterator = (Symbol as any).asyncIterator || Symbol.for('Symbol.asyncIterator');

interface ProgressGenerator<T> extends AsyncIterableIterator<number | T> {
    next(cont?: boolean): Promise<IteratorResult<number | T>>
}

async function *test(): ProgressGenerator<boolean> {
    const r = yield await new Promise<number>(res => res(10));
    return r;
}

async function runIt(itP: () => ProgressGenerator<boolean>) {
    const it = itP();
    while (true) {
        const { value, done } = await it.next(true);
        if (done) return value;
    }
}

runIt(test).then(r => console.log('rerdasdasda', r))

export async function readCIF(path: string) {
    console.time('readData');
    const input = await readData(path)
    console.timeEnd('readData');

    console.time('parse');
    const comp = typeof input === 'string' ? CIF.parseText(input) : CIF.parseBinary(input);
    const parsed = await comp();
    console.timeEnd('parse');
    if (parsed.isError) {
        throw parsed;
    }

    const data = parsed.result.blocks[0];
    console.time('schema')
    const mmcif = CIF.schema.mmCIF(data);

    console.timeEnd('schema')
    console.time('buildModels')
    const models = Model.create({ kind: 'mmCIF', data: mmcif });
    console.timeEnd('buildModels')
    const structures = models.map(Structure.ofModel);

    return { mmcif, models, structures };
}

const DATA_DIR = './build/data';
if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR);

function getBcifUrl(pdbId: string) {
    return `http://www.ebi.ac.uk/pdbe/coordinates/${pdbId.toLowerCase()}/full?encoding=bcif`
}

function getBcifPath(pdbId: string) {
    return `${DATA_DIR}/${pdbId.toLowerCase()}_full.bcif`
}

async function ensureBcifAvailable(pdbId: string) {
    const bcifPath = getBcifPath(pdbId);
    if (!fs.existsSync(bcifPath)) {
        console.log(`downloading ${pdbId} bcif...`)
        const data = await fetch(getBcifUrl(pdbId))
        await writeFileAsync(bcifPath, await data.buffer())
        console.log(`done downloading ${pdbId} bcif`)
    }
}

async function getBcif(pdbId: string) {
    await ensureBcifAvailable(pdbId);
    return await readCIF(getBcifPath(pdbId));
}

export namespace PropertyAccess {
    function baseline(model: Model) {
        if (model.sourceData.kind !== 'mmCIF') throw new Error('Model must be mmCIF');
        const atom_site = model.sourceData.data.atom_site;
        const id = atom_site.id.value;
        let s = 0;
        for (let i = 0, _i = atom_site._rowCount; i < _i; i++) {
            s += id(i);
        }
        return s;
    }

    function sumProperty(structure: Structure, p: Atom.Property<number>) {
        const { atoms, units } = structure;
        const unitIds = AtomSet.unitIds(atoms);
        const l = Atom.Location();

        let s = 0;

        for (let i = 0, _i = unitIds.length; i < _i; i++) {
            l.unit = units[unitIds[i]];
            const set = AtomSet.unitGetByIndex(atoms, i);


            for (let j = 0, _j = AtomGroup.size(set); j < _j; j++) {
                l.atom = AtomGroup.getAt(set, j);
                s += p(l);
            }
        }

        return s;
    }

    function sumPropertySegmented(structure: Structure, p: Atom.Property<number>) {
        const { atoms, units } = structure;
        const unitIds = AtomSet.unitIds(atoms);
        const l = Atom.Location();

        let s = 0;

        let vA = 0, cC = 0, rC = 0;
        for (let i = 0, _i = unitIds.length; i < _i; i++) {
            const unit = units[unitIds[i]];
            l.unit = unit;
            const set = AtomSet.unitGetByIndex(atoms, i);

            const chainsIt = Segmentation.transientSegments(unit.hierarchy.chainSegments, set.atoms);
            const residues = unit.hierarchy.residueSegments;
            while (chainsIt.hasNext) {
                cC++;

                const chainSegment = chainsIt.move();
                const residuesIt = Segmentation.transientSegments(residues, set.atoms, chainSegment);
                while (residuesIt.hasNext) {
                    rC++;
                    const residueSegment = residuesIt.move();
                    // l.atom = OrdSet.getAt(set, residueSegment.start);
                    // console.log(unit.hierarchy.residues.auth_comp_id.value(unit.residueIndex[l.atom]), l.atom, OrdSet.getAt(set, residueSegment.end))
                    for (let j = residueSegment.start, _j = residueSegment.end; j < _j; j++) {
                        l.atom = AtomGroup.getAt(set, j);
                        vA++;
                        s += p(l);
                    }
                }
            }
        }

        console.log('seg atom count', vA, cC, rC);

        return s;
    }

    function sumPropertyResidue(structure: Structure, p: Atom.Property<number>) {
        const { atoms, units } = structure;
        const unitIds = AtomSet.unitIds(atoms);
        const l = Atom.Location();

        let s = 0;

        for (let i = 0, _i = unitIds.length; i < _i; i++) {
            const unit = units[unitIds[i]];
            l.unit = unit;
            const set = AtomSet.unitGetByIndex(atoms, i);
            const residuesIt = Segmentation.transientSegments(unit.hierarchy.residueSegments, set.atoms);
            while (residuesIt.hasNext) {
                l.atom = AtomGroup.getAt(set, residuesIt.move().start);
                s += p(l);
            }
        }

        return s;
    }

    function sumPropertyAtomSetIt(structure: Structure, p: Atom.Property<number>) {
        const { atoms, units } = structure;

        let s = 0;
        const atomsIt = AtomSet.atoms(atoms);
        const l = Atom.Location();
        while (atomsIt.hasNext) {
            const a = atomsIt.move();
            l.unit = units[Atom.unit(a)];
            l.atom = Atom.index(a);
            s += p(l);
        }
        return s;
    }

    // function sumPropertySegmentedMutable(structure: Structure, p: Property<number>) {
    //     const { atoms, units } = structure;
    //     const unitIds = AtomSet.unitIds(atoms);
    //     const l = Property.createLocation();

    //     let s = 0;

    //     for (let i = 0, _i = unitIds.length; i < _i; i++) {
    //         const unit = units[unitIds[i]];
    //         l.unit = unit;
    //         const set = AtomSet.unitGetByIndex(atoms, i);

    //         const chainsIt = Segmentation.transientSegments(unit.hierarchy.chainSegments, set);
    //         const residuesIt = Segmentation.transientSegments(unit.hierarchy.residueSegments, set);
    //         while (chainsIt.hasNext) {
    //             const chainSegment = chainsIt.move();
    //             residuesIt.updateRange(chainSegment);
    //             while (residuesIt.hasNext) {
    //                 const residueSegment = residuesIt.move();
    //                 for (let j = residueSegment.start, _j = residueSegment.end; j < _j; j++) {
    //                     l.atom = OrdSet.getAt(set, j);
    //                     s += p(l);
    //                 }
    //             }
    //         }
    //     }

    //     return s;
    // }

    function sumDirect(structure: Structure) {
        const { atoms, units } = structure;
        const unitIds = AtomSet.unitIds(atoms);

        let s = 0;

        for (let i = 0, _i = unitIds.length; i < _i; i++) {
            const unitId = unitIds[i];
            const unit = units[unitId];
            const set = AtomSet.unitGetByIndex(atoms, i);
            //const { residueIndex, chainIndex } = unit;
            const p = unit.conformation.atomId.value;
            for (let j = 0, _j = AtomGroup.size(set); j < _j; j++) {
                const aI = AtomGroup.getAt(set, j);
                s += p(aI);
            }
        }

        return s;
    }

    // function concatProperty(structure: Structure, p: Property<string>) {
    //     const { atoms, units } = structure;
    //     const unitIds = AtomSet.unitIds(atoms);
    //     const l = Property.createLocation(structure);

    //     let s = [];

    //     for (let i = 0, _i = unitIds.length; i < _i; i++) {
    //         const unitId = unitIds[i];
    //         l.unit = units[unitId];
    //         const set = AtomSet.unitGetByIndex(atoms, i);
    //         const { residueIndex, chainIndex } = l.unit;

    //         for (let j = 0, _j = OrdSet.size(set); j < _j; j++) {
    //             const aI = OrdSet.getAt(set, j);
    //             l.atom = aI;
    //             l.residueIndex = residueIndex[aI];
    //             l.chainIndex = chainIndex[aI];
    //             s[s.length] = p(l);
    //         }
    //     }

    //     return s;
    // }

    export function write(s: Structure) {
        console.log(to_mmCIF('test', s));
    }

    export function testAssembly(id: string, s: Structure) {
        console.time('assembly')
        const a = Symmetry.buildAssembly(s, '1');
        console.timeEnd('assembly')
        fs.writeFileSync(`${DATA_DIR}/${id}_assembly.bcif`, to_mmCIF(id, a, true));
        console.log('exported');
    }

    export async function run() {
        const { structures, models/*, mmcif*/ } = await getBcif('1cbs');
        // const { structures, models } = await getBcif('3j3q');

        //const { structures, models, mmcif } = await readCIF('e:/test/quick/1cbs_updated.cif');
        //const { structures, models/*, mmcif*/ } = await readCIF('e:/test/quick/5j7v_updated.cif');

        //console.log(mmcif.pdbx_struct_oper_list.matrix.toArray());
        // console.log(mmcif.pdbx_struct_oper_list.vector.toArray());

        // testAssembly('5j7v', structures[0]);
        // throw '';

        // console.log(models[0].symmetry.assemblies);


        //const { structures, models } = await readCIF('e:/test/molstar/3j3q.bcif');

        // fs.writeFileSync('e:/test/molstar/3j3q.bcif', to_mmCIF('test', structures[0], true));
        // return;

        // console.log(toMmCIFString('test', structures[0]));

        // return;

        console.log('bs', baseline(models[0]));
        console.log('sp', sumProperty(structures[0], l => l.unit.model.conformation.atomId.value(l.atom)));
        console.log(sumPropertySegmented(structures[0], l => l.unit.model.conformation.atomId.value(l.atom)));

        //console.log(sumPropertySegmentedMutable(structures[0], l => l.unit.model.conformation.atomId.value(l.atom)));
        console.log(sumPropertyAtomSetIt(structures[0], l => l.unit.model.conformation.atomId.value(l.atom)));
        //console.log(sumProperty(structures[0], Property.cachedAtomColumn(m => m.conformation.atomId)));
        console.log(sumDirect(structures[0]));
        console.log('r', sumPropertyResidue(structures[0], l => l.unit.hierarchy.residues.auth_seq_id.value(l.unit.residueIndex[l.atom])));
{'fraction_non_alphanumeric': 0.14353750496097367, 'fraction_numerical': 0.00485072981434934, 'mean_word_length': 2.0951276102088165, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '896452', 'n_tokens_mistral': 7432, 'n_tokens_neox': 7056, 'n_words': 1527}