<commit_before>import {Map, Either} from 'tsfun';
import {Document} from 'idai-components-2';
import {ImportValidator} from './process/import-validator';
import {DocumentDatastore} from '../../datastore/document-datastore';
import {Find, Get} from './types';
import {complementInverseRelationsBetweenImportDocs, makeSureRelationStructuresExists, preprocessRelations} from './preprocess-relations';
import {preprocessFields} from './preprocess-fields';
import {ImportErrors as E} from './import-errors';
import {HierarchicalRelations, PARENT} from '../../model/relation-constants';
import LIES_WITHIN = HierarchicalRelations.LIESWITHIN;
import RECORDED_IN = HierarchicalRelations.RECORDEDIN;
import {InverseRelationsMap} from '../../configuration/inverse-relations-map';
import {processDocuments} from './process/process-documents';
import {processRelations} from './process/process-relations';
import {Settings} from '../../settings/settings';


export interface ImportOptions {

    mergeMode?: boolean;
    permitDeletions?: boolean;
    operationId?: string;
    useIdentifiersInRelations?: boolean;
}


export interface ImportHelpers {

    generateId: () => string;
    preprocessDocument: (_: Document) => Document;
    postprocessDocument: (_: Document) => Document;
}


export interface ImportServices {

    validator: ImportValidator,
    datastore: DocumentDatastore
}


export interface ImportContext {

    operationCategoryNames: string[];
    inverseRelationsMap: InverseRelationsMap;
    settings: Settings;
}


export interface ImportResult {

    createDocuments: Array<Document>;
    updateDocuments: Array<Document>;
    targetDocuments: Array<Document>;
    ignoredIdentifiers: string[];
}


export type ErrWithParams = Array<string>;
export type CreateDocuments = Array<Document>;
export type UpdateDocuments = Array<Document>;
export type TargetDocuments = Array<Document>;

export type Result = Either<ErrWithParams, ImportResult>;
export type Documents2Result = (_: Array<Document>) => Promise<Result>;


/**
 * @author <NAME>
 * @author <NAME>
 *
 * @returns Documents2Result, which expects
 *   documents with the field resource.identifier set to a non empty string.
 *   If resource.id is set, it will be taken as document.id, otherwise a new id gets generated.
 *   The relations map is assumed to be at least existent, but can be empty. // TODO REVIEW, than we can omit creation of it and only assert that it is there
 *   The resource.category field may be empty.
 */
export function buildImportDocuments(services: ImportServices,
                                     context: ImportContext,
                                     helpers: ImportHelpers,
                                     options: ImportOptions = {}): Documents2Result {

    if (options.mergeMode) {
        if (!options.useIdentifiersInRelations) {
            throw 'FATAL - illegal arguments - '
            + 'useIdentifiersInRelations must also be set when merge mode selected';
        }
        if (options.operationId) {
            throw 'FATAL - illegal argument combination '
            + '- mergeMode and operationId must not be both truthy';
        }
    }

    const get  = (resourceId: string) => services.datastore.get(resourceId);
    const find = findByIdentifier(services.datastore);

    return (documents: Array<Document>) => importDocuments(
            services, context, helpers, options,find, get, documents);
}


async function importDocuments(services: ImportServices,
                               context: ImportContext,
                               helpers: ImportHelpers,
                               options: ImportOptions,
                               find: Find,
                               get: Get,
                               documents: Array<Document>): Promise<Result> {

    makeSureRelationStructuresExists(documents);
    complementInverseRelationsBetweenImportDocs(context, options, documents); // TODO now that we have that here, we could simplify later steps probably

    try {
        const existingImportDocuments = await makeExistingDocumentsMap(find, options, documents);
        const { documentsToImport, documentsToIgnore } = getDocumentsToImport(existingImportDocuments, options, documents);
        preprocessFields(documentsToImport, options);
        await preprocessRelations(documentsToImport, helpers, get, find, options);
        const mergeDocs = preprocessDocuments(existingImportDocuments, helpers, options, documentsToImport);
        const processedDocuments = processDocuments(documentsToImport, mergeDocs, services.validator);
        const targetDocuments = await processRelations(
            processedDocuments,
            services.validator,
            context.operationCategoryNames,
            get,
            context.inverseRelationsMap,
            options
        );
        const postprocessedDocuments: Array<Document> = processedDocuments.map(helpers.postprocessDocument);
        const ignoredIdentifiers: string[] = documentsToIgnore.map(document => document.resource.identifier);

        return options.mergeMode === true
            ? [undefined, {
                createDocuments: [],
                updateDocuments: postprocessedDocuments,
                targetDocuments,
                ignoredIdentifiers
            }]
            : [undefined, {
                createDocuments: postprocessedDocuments,
                updateDocuments: [],
                targetDocuments,
                ignoredIdentifiers
            }];

    } catch (errWithParams) {
        if (errWithParams[0] === E.TARGET_CATEGORY_RANGE_MISMATCH
                && ([LIES_WITHIN, RECORDED_IN].includes(errWithParams[2]))) errWithParams[2] = PARENT;
        return [errWithParams, undefined];
    }
}

<commit_msg>Simplify; use infered types<commit_after>import {Map, Either} from 'tsfun';
import {Document} from 'idai-components-2';
import {ImportValidator} from './process/import-validator';
import {DocumentDatastore} from '../../datastore/document-datastore';
import {Find, Get} from './types';
import {complementInverseRelationsBetweenImportDocs, makeSureRelationStructuresExists, preprocessRelations} from './preprocess-relations';
import {preprocessFields} from './preprocess-fields';
import {ImportErrors as E} from './import-errors';
import {HierarchicalRelations, PARENT} from '../../model/relation-constants';
import LIES_WITHIN = HierarchicalRelations.LIESWITHIN;
import RECORDED_IN = HierarchicalRelations.RECORDEDIN;
import {InverseRelationsMap} from '../../configuration/inverse-relations-map';
import {processDocuments} from './process/process-documents';
import {processRelations} from './process/process-relations';
import {Settings} from '../../settings/settings';


export interface ImportOptions {

    mergeMode?: boolean;
    permitDeletions?: boolean;
    operationId?: string;
    useIdentifiersInRelations?: boolean;
}


export interface ImportHelpers {

    generateId: () => string;
    preprocessDocument: (_: Document) => Document;
    postprocessDocument: (_: Document) => Document;
}


export interface ImportServices {

    validator: ImportValidator,
    datastore: DocumentDatastore
}


export interface ImportContext {

    operationCategoryNames: string[];
    inverseRelationsMap: InverseRelationsMap;
    settings: Settings;
}


export interface ImportResult {

    createDocuments: Array<Document>;
    updateDocuments: Array<Document>;
    targetDocuments: Array<Document>;
    ignoredIdentifiers: string[];
}


export type ErrWithParams = Array<string>;
export type CreateDocuments = Array<Document>;
export type UpdateDocuments = Array<Document>;
export type TargetDocuments = Array<Document>;

export type Result = Either<ErrWithParams, ImportResult>;
export type Documents2Result = (_: Array<Document>) => Promise<Result>;


/**
 * @author <NAME>
 * @author <NAME>
 *
 * @returns Documents2Result, which expects
 *   documents with the field resource.identifier set to a non empty string.
 *   If resource.id is set, it will be taken as document.id, otherwise a new id gets generated.
 *   The relations map is assumed to be at least existent, but can be empty. // TODO REVIEW, than we can omit creation of it and only assert that it is there
 *   The resource.category field may be empty.
 */
export function buildImportDocuments(services: ImportServices,
                                     context: ImportContext,
                                     helpers: ImportHelpers,
                                     options: ImportOptions = {}): Documents2Result {

    if (options.mergeMode) {
        if (!options.useIdentifiersInRelations) {
            throw 'FATAL - illegal arguments - '
            + 'useIdentifiersInRelations must also be set when merge mode selected';
        }
        if (options.operationId) {
            throw 'FATAL - illegal argument combination '
            + '- mergeMode and operationId must not be both truthy';
        }
    }

    const get  = (resourceId: string) => services.datastore.get(resourceId);
    const find = findByIdentifier(services.datastore);

    return (documents: Array<Document>) => importDocuments(
            services, context, helpers, options,find, get, documents);
}


async function importDocuments(services: ImportServices,
                               context: ImportContext,
                               helpers: ImportHelpers,
                               options: ImportOptions,
                               find: Find,
                               get: Get,
                               documents: Array<Document>): Promise<Result> {

    makeSureRelationStructuresExists(documents);
    complementInverseRelationsBetweenImportDocs(context, options, documents); // TODO now that we have that here, we could simplify later steps probably

    try {
        const existingImportDocuments = await makeExistingDocumentsMap(find, options, documents);
        const { documentsToImport, documentsToIgnore } = getDocumentsToImport(existingImportDocuments, options, documents);
        preprocessFields(documentsToImport, options);
        await preprocessRelations(documentsToImport, helpers, get, find, options);
        const mergeDocs = preprocessDocuments(existingImportDocuments, helpers, options, documentsToImport);
        const processedDocuments = processDocuments(documentsToImport, mergeDocs, services.validator);
        const targetDocuments = await processRelations(
            processedDocuments,
            services.validator,
            context.operationCategoryNames,
            get,
            context.inverseRelationsMap,
            options
        );
        const postprocessedDocuments = processedDocuments.map(helpers.postprocessDocument);
        const ignoredIdentifiers = documentsToIgnore.map(_ => _.resource.identifier);

        return options.mergeMode === true
            ? [undefined, {
                createDocuments: [],
                updateDocuments: postprocessedDocuments,
                targetDocuments,
                ignoredIdentifiers
            }]
            : [undefined, {
                createDocuments: postprocessedDocuments,
                updateDocuments: [],
                targetDocuments,
                ignoredIdentifiers
            }];

    } catch (errWithParams) {
        if (errWithParams[0] === E.TARGET_CATEGORY_RANGE_MISMATCH
                && ([LIES_WITHIN, RECORDED_IN].includes(errWithParams[2]))) errWithParams[2] = PARENT;
        return [errWithParams, undefined];
    }
}


{'fraction_non_alphanumeric': 0.08977793139203318, 'fraction_numerical': 0.0012097122612978485, 'mean_word_length': 2.915426251691475, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3745530', 'n_tokens_mistral': 2922, 'n_tokens_neox': 2748, 'n_words': 769}