<commit_before>import {Action, Document} from 'idai-components-2/core';
import {ChangeHistoryUtil} from '../../model/change-history-util';
import {ResultSets} from '../../../util/result-sets';
import {IndexItem} from './index-item';

/**
 * @author <NAME>
 * @author <NAME>
 */
export class FulltextIndexer {

    private fieldsToIndex = ['identifier', 'shortDescription'];

    private index: {
        [resourceType: string]: {
            [term: string]: {
                [resourceId: string]: IndexItem
            }
        }
    };


    constructor() {

        this.setUp();
    }


    public clear() {

        this.setUp();
    }


    public put(doc: Document, skipRemoval: boolean = false) {

        const indexItem = IndexItem.from(doc);
        if (!indexItem) return;

        if (!skipRemoval) this.remove(doc);
        if (!this.index[doc.resource.type]) {
            this.index[doc.resource.type] = {'*' : { } };
        }
        this.index[doc.resource.type]['*'][doc.resource.id as any] = indexItem;

        this.fieldsToIndex.
            filter(field => doc.resource[field] && doc.resource[field] !== '').
            forEach(field =>
                doc.resource[field].split(' ').forEach((token: string) =>
                    this.indexToken(doc.resource.id as any, token,
                        doc.resource.type, indexItem)
                )
            );
    }


    private indexToken(id: string, token: string, type: string, indexItem: IndexItem) {

        let accumulator = '';
        for (let letter of token.toLowerCase()) {
            accumulator += letter;
            if (!this.index[type][accumulator]) this.index[type][accumulator] = {};
            this.index[type][accumulator][id] = indexItem;
        }
    }


    public remove(doc: any) {

        if (Object.keys(this.index).length == 0) return;
        for (let type of Object.keys(this.index)) {
            for (let term of Object.keys(this.index[type])) {
                if (this.index[type][term][doc.resource.id]) {
                    delete this.index[type][term][doc.resource.id];
                }
            }
        }
    }


    /**
     * @param s search string, which gets tokenized, so that the result will include
     *   search hits for any of the tokens. If s is "hello world", all items which are
     *   indexed under either "hello" or "world" will be included in the result. The
     *   result will be a set in the sense that it will include each item only once.
     * @param types if undefined, searches in all types. If defined, only search hits
     *   indexed under the specified types will be included in the results.
     * @returns {any} array of items
     */
    public get(s: string, types: string[]): Array<any> {<commit_msg>Use collection methods<commit_after>import {Action, Document} from 'idai-components-2/core';
import {ChangeHistoryUtil} from '../../model/change-history-util';
import {ResultSets} from '../../../util/result-sets';
import {IndexItem} from './index-item';

/**
 * @author <NAME>
 * @author <NAME>
 */
export class FulltextIndexer {

    private fieldsToIndex = ['identifier', 'shortDescription'];

    private index: {
        [resourceType: string]: {
            [term: string]: {
                [resourceId: string]: IndexItem
            }
        }
    };


    constructor() {

        this.setUp();
    }


    public clear() {

        this.setUp();
    }


    public put(doc: Document, skipRemoval: boolean = false) {

        const indexItem = IndexItem.from(doc);
        if (!indexItem) return;

        if (!skipRemoval) this.remove(doc);
        if (!this.index[doc.resource.type]) {
            this.index[doc.resource.type] = {'*' : { } };
        }
        this.index[doc.resource.type]['*'][doc.resource.id as any] = indexItem;

        this.fieldsToIndex.
            filter(field => doc.resource[field] && doc.resource[field] !== '').
            forEach(field =>
                doc.resource[field].split(' ').forEach((token: string) =>
                    this.indexToken(doc.resource.id as any, token,
                        doc.resource.type, indexItem)
                )
            );
    }


    private indexToken(id: string, token: string, type: string, indexItem: IndexItem) {

        Array.from(token.toLowerCase()).
            reduce((accumulator: string, letter: string) => {
                accumulator += letter;
                if (!this.index[type][accumulator]) this.index[type][accumulator] = {};
                this.index[type][accumulator][id] = indexItem;
                return accumulator;
            }, '');
    }


    public remove(doc: any) {

        if (Object.keys(this.index).length == 0) return;
        for (let type of Object.keys(this.index)) {
            for (let term of Object.keys(this.index[type])) {
                if (this.index[type][term][doc.resource.id]) {
                    delete this.index[type][term][doc.resource.id];
                }
            }
        }
    }


    /**
     * @param s search string, which gets tokenized, so that the result will include
     *   search hits for any of the tokens. If s is "hello world", all items which are
     *   indexed under either "hello" or "world" will be included in the result. The
     *   result will be a set in the sense that it will include each item only once.
     * @param types if undefined, searches in all types. If defined, only search hits
     *   indexed under the specified types will be included in the results.
     * @returns {any} array of items
     */
    public get(s: string, types: string[]): Array<any> {
{'fraction_non_alphanumeric': 0.13560525381611643, 'fraction_numerical': 0.0007099751508697196, 'mean_word_length': 2.04430037817396, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1204095', 'n_tokens_mistral': 1614, 'n_tokens_neox': 1548, 'n_words': 432}