<commit_before>import * as LocalForage from 'localforage';
import { IDatabasePersistor, ICollectionPersistor } from '../';

/**
 * LocalForageDatabasePersistor
 * ...
 */
export class LocalForageDatabasePersistor implements IDatabasePersistor {
    /**
     * _registry
     * ...
     */
    protected _metadata: any;

    /**
     * constructor
     * ...
     */
    constructor( protected databaseName: string ) {
        this._metadata = this._createLocalForageForMetadata();
    }

    /**
     * create
     * ...
     */
    public create( collectionName: string ): ICollectionPersistor {
        // FIXME: collection registering is run async!!
        this._registerCollection( collectionName );
        const localforage = this._createLocalForageForCollection( collectionName );
        return new LocalForageCollectionPersistor( localforage );
    }

    /**
     * drop
     * ...
     */
    public drop(): Promise<any> {
        return this._metadata.getItem( 'collections' )
            .then( collections => {
                if ( !collections || !collections.length ) {
                    return;
                }
                return Promise.all( collections.map( collectionName => {
                    const localforage = this._createLocalForageForCollection( collectionName );
                    return localforage.clear();
                }));
            })
            .then(() => this._metadata.setItem( 'collections', []));

    }

    /**
     * _registerCollection
     * ...
     */
    protected _registerCollection( name: string ): Promise<any> {
        return this._metadata.getItem( 'collections' )
            .then( collections => {
                if ( !Array.isArray( collections )) {
                    return this._metadata.setItem( 'collections', [ name ]);
                }
                const set = new Set<string>( collections.concat( name ));
                return this._metadata.setItem( 'collections', Array.from( set ));
            });
    }

    /**
     * _createLocalForageForCollection
     * ...
     */
    protected _createLocalForageForCollection( collectionName: string ): any {
        const name = this._createCollectionName( collectionName );
        return this._createLocalForage( name );
    }

    /**
     * _createLocalForageForMetadata
     * ...
     */
    protected _createLocalForageForMetadata(): any {
        const name = this._createMetadataCollectionName( 'metadata' );
        return this._createLocalForage( name );
    }

    /**
     * _createLocalForage
     * ...
     */
    protected _createLocalForage( name: string ): any {
        return LocalForage.createInstance({ name });
    }

    /**
     * _createCollectionName
     * ...
     */
     protected _createCollectionName( collectionName: string ): string {
         return `${this.databaseName}:${collectionName}`;
     }

    /**
     * _createMetadataCollectionName
     * ...
     */
     protected _createMetadataCollectionName( collectionName: string ): string {
         return `${this.databaseName}.${collectionName}`;
     }
}

/**
 * LocalForageCollectionPersistor
 * ...
 */
export class LocalForageCollectionPersistor implements ICollectionPersistor {
    constructor( protected localforage: any ) {
        // ...
    }

    /**
     * load
     * ...
     */
    public load(): Promise<any[]> {
        const documents = [];
        return this.localforage
            .iterate( value  => {
                // If a non-undefined value is returned, the iterator will stop
                documents.push( value );
                return undefined;
            })
            .then(() => documents );
    }

    /**
     * store
     * ...
     */
    public store( docs: any[]): Promise<any> {
        return Promise.all( docs.map( doc => this.localforage.setItem( doc.id, doc )));
    }

    /**
     * remove
     * ...
     */
    public remove( docs: any[]): Promise<any> {
        return Promise.all( docs.map( doc => this.localforage.removeItem( doc.id )));
    }

    /**
     * drop
     * ...
     */
    public drop(): Promise<any> {
        return this.localforage.clear();
    }
}
<commit_msg>Make collection registration async friendly<commit_after>import * as LocalForage from 'localforage';
import { IDatabasePersistor, ICollectionPersistor } from '../';

/**
 * LocalForageDatabasePersistor
 * ...
 */
export class LocalForageDatabasePersistor implements IDatabasePersistor {
    /**
     * _registry
     * ...
     */
    protected _metadata: any;

    /**
     * _collections
     * ...
     */
    protected _collections: Set<string>;
    protected _loadCollectionsPromise: Promise<Set<string>>;

    /**
     * constructor
     * ...
     */
    constructor( protected databaseName: string ) {
        this._metadata = this._createLocalForageForMetadata();
        this._collections = new Set<string>();
    }

    /**
     * create
     * ...
     */
    public create( collectionName: string ): ICollectionPersistor {
        // FIXME: collection registering is run async!!
        this._registerCollection( collectionName );
        const localforage = this._createLocalForageForCollection( collectionName );
        return new LocalForageCollectionPersistor( localforage );
    }

    /**
     * drop
     * ...
     */
    public drop(): Promise<any> {
        return this._metadata.getItem( 'collections' )
            .then( collections => {
                if ( !collections || !collections.length ) {
                    return;
                }
                return Promise.all( collections.map( collectionName => {
                    const localforage = this._createLocalForageForCollection( collectionName );
                    return localforage.clear();
                }));
            })
            .then(() => this._metadata.setItem( 'collections', []));

    }

    /**
     * _loadInitialCollections
     * ...
     */
    protected _loadInitialCollections(): Promise<Set<string>> {
        if ( !this._loadCollectionsPromise ) {
            this._loadCollectionsPromise = this._metadata.getItem( 'collections' )
                .then( collections => {
                    if ( collections ) {
                        collections.forEach( name => this._collections.add( name ));
                    }
                });
        }
        return this._loadCollectionsPromise;
    }

    /**
     * _registerCollection
     * ...
     */
    protected _registerCollection( name: string ): Promise<any> {
        return this._loadInitialCollections()
            .then(() => {
                if ( this._collections.has( name )) {
                    return;
                }
                this._collections.add( name );
                const collections = Array.from( this._collections );
                this._metadata.setItem( 'collections', collections );
            });
    }

    /**
     * _createLocalForageForCollection
     * ...
     */
    protected _createLocalForageForCollection( collectionName: string ): any {
        const name = this._createCollectionName( collectionName );
        return this._createLocalForage( name );
    }

    /**
     * _createLocalForageForMetadata
     * ...
     */
    protected _createLocalForageForMetadata(): any {
        const name = this._createMetadataCollectionName( 'metadata' );
        return this._createLocalForage( name );
    }

    /**
     * _createLocalForage
     * ...
     */
    protected _createLocalForage( name: string ): any {
        return LocalForage.createInstance({ name });
    }

    /**
     * _createCollectionName
     * ...
     */
     protected _createCollectionName( collectionName: string ): string {
         return `${this.databaseName}:${collectionName}`;
     }

    /**
     * _createMetadataCollectionName
     * ...
     */
     protected _createMetadataCollectionName( collectionName: string ): string {
         return `${this.databaseName}.${collectionName}`;
     }
}

/**
 * LocalForageCollectionPersistor
 * ...
 */
export class LocalForageCollectionPersistor implements ICollectionPersistor {
    constructor( protected localforage: any ) {
        // ...
    }

    /**
     * load
     * ...
     */
    public load(): Promise<any[]> {
        const documents = [];
        return this.localforage
            .iterate( value  => {
                // If a non-undefined value is returned, the iterator will stop
                documents.push( value );
                return undefined;
            })
            .then(() => documents );
    }

    /**
     * store
     * ...
     */
    public store( docs: any[]): Promise<any> {
        return Promise.all( docs.map( doc => this.localforage.setItem( doc.id, doc )));
    }

    /**
     * remove
     * ...
     */
    public remove( docs: any[]): Promise<any> {
        return Promise.all( docs.map( doc => this.localforage.removeItem( doc.id )));
    }

    /**
     * drop
     * ...
     */
    public drop(): Promise<any> {
        return this.localforage.clear();
    }
}

{'fraction_non_alphanumeric': 0.12498618631893027, 'fraction_numerical': 0.0, 'mean_word_length': 1.8149300155520995, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1855992', 'n_tokens_mistral': 2408, 'n_tokens_neox': 2327, 'n_words': 464}