<commit_before>import {ImportValidator} from './import-validator';
import {duplicates, hasNot, includedIn, isArray, isnt, isUndefinedOrEmpty, not, to} from 'tsfun';
import {asyncForEach} from 'tsfun-extra';
import {ImportErrors as E} from './import-errors';
import {Document, Relations} from 'idai-components-2';
import {RESOURCE_ID, RESOURCE_IDENTIFIER} from '../../../c';
import {HIERARCHICAL_RELATIONS, PARENT} from '../../model/relation-constants';
import {processRelations} from './process-relations';
import {Find, GenerateId, Get, GetInverseRelation, Id, Identifier, IdentifierMap, ProcessResult} from './types';
import {processDocuments} from './process-documents';
import LIES_WITHIN = HIERARCHICAL_RELATIONS.LIES_WITHIN;
import {assertLegalCombination} from './utils';


/**
 * @author <NAME>
 * @author <NAME>
 */
export function build(validator: ImportValidator,
                      operationTypeNames: string[],
                      generateId: GenerateId,
                      find: Find,
                      get: Get,
                      getInverseRelation: GetInverseRelation,
                      mergeMode: boolean,
                      allowOverwriteRelationsInMergeMode: boolean,
                      mainTypeDocumentId: Id,
                      useIdentifiersInRelations: boolean) {

    assertLegalCombination(mainTypeDocumentId, mergeMode);

    /**
     * ImportErrors (accessible via ProcessResult)
     *
     * [MUST_LIE_WITHIN_OTHER_NON_OPERATON_RESOURCE, resourceType, resourceIdentifier]
     *   - if a resource of
     *     a defined builtin type should be placed inside another resource of a legal LIES_WITHIN range type, but is placed
     *     directly below an operation.
     *
     * [BAD_INTERRELATION, sourceId]
     *   - if opposing relations are pointing to the same resource.
     *     For example IS_BEFORE and IS_AFTER pointing both from document '1' to '2'.
     *   - if mutually exluding relations are pointing to the same resource.
     *     For example IS_CONTEMPORARY_WITH and IS_AFTER both from document '1' to '2'.
     *
     * [TARGET_TYPE_RANGE_MISMATCH, resourceIdentifier, relationName, relationTargetResourceType]
     *   - if a resource points to another resource, however, the specified relation is not allowed between the
     *     types of the resources.
     *
     * [PARENT_ASSIGNMENT_TO_OPERATIONS_NOT_ALLOWED]
     *   - if mainTypeDocumentId is not '' and
     *     a resource references an operation as parent.
     *
     * [LIES_WITHIN_TARGET_NOT_MATCHES_ON_IS_RECORDED_IN, resourceIdentifier]
     *   - if the inferredRecordedIn
     *     differs from a possibly specified recordedIn.
     *
     * [EXEC_MISSING_RELATION_TARGET, targetId]
     *
     * @throws [EMPTY_RELATION, resourceId]
     *   - if relations empty for some relation is empty.
     *     For example relations: {isAbove: []}
     */
    return async function process(documents: Array<Document>): Promise<ProcessResult> {

        try {
            assertNoDuplicates(documents);

            const identifierMap: IdentifierMap = mergeMode ? {} : assignIds(documents, generateId);
            const rewriteIdentifiersInRels = rewriteIdentifiersInRelations(find, identifierMap);
            const assertNoMissingRelTargets = assertNoMissingRelationTargets(get);

            await preprocessAndValidateRelations(
                documents,
                mergeMode,
                allowOverwriteRelationsInMergeMode,
                useIdentifiersInRelations,
                assertNoMissingRelTargets,
                rewriteIdentifiersInRels);

            const processedDocuments = await processDocuments(
                validator, mergeMode, allowOverwriteRelationsInMergeMode, find)(documents);

            const relatedDocuments = await processRelations(
                processedDocuments,
                validator, operationTypeNames,
                mergeMode, allowOverwriteRelationsInMergeMode,
                getInverseRelation,
                get,
                mainTypeDocumentId);

            return [processedDocuments, relatedDocuments, undefined];

        } catch (errWithParams) {

            return [[],[], errWithParams];
        }
    }
}



function assertNoDuplicates(documents: Array<Document>) {

    const dups = duplicates(documents.map(to(RESOURCE_IDENTIFIER)));
    if (dups.length > 0) throw [E.DUPLICATE_IDENTIFIER, dups[0]];
}


function adjustRelations(document: Document, relations: Relations) {

    assertHasNoForbiddenRelations(document);
    const assertIsntArrayRelation = assertIsNotArrayRelation(document);

    Object.keys(document.resource.relations)
        .filter(isnt(PARENT))
        .forEach(assertIsntArrayRelation);

    assertParentNotArray(relations[PARENT], document.resource.identifier);
    if (relations[PARENT]) (relations[LIES_WITHIN] = [relations[PARENT] as any]) && delete relations[PARENT];
}


function assertParentNotArray(parentRelation: any, resourceIdentifier: string) {

    if (isArray(parentRelation)) throw [E.PARENT_MUST_NOT_BE_ARRAY, resourceIdentifier];
}


function assertHasNoForbiddenRelations(document: Document) {

    const foundForbiddenRelations = Object.keys(document.resource.relations)
        .filter(includedIn(HIERARCHICAL_RELATIONS.ALL))
        .join(', ');
    if (foundForbiddenRelations) throw [E.INVALID_RELATIONS, document.resource.type, foundForbiddenRelations];
}


function assertIsNotArrayRelation(document: Document) {

    return (name: string) => {

        if (not(isArray)(document.resource.relations[name])) throw [E.MUST_BE_ARRAY, document.resource.identifier];
    }
}


async function preprocessAndValidateRelations(
    documents: Array<Document>,
    mergeMode: boolean,
    allowOverwriteRelationsInMergeMode: boolean,
    useIdentifiersInRelations: boolean,
    assertNoMissingRelTargets: Function,
    rewriteIdentifiersInRelations: Function): Promise<void> {<commit_msg>Add comment<commit_after>import {ImportValidator} from './import-validator';
import {duplicates, hasNot, includedIn, isArray, isnt, isUndefinedOrEmpty, not, to} from 'tsfun';
import {asyncForEach} from 'tsfun-extra';
import {ImportErrors as E} from './import-errors';
import {Document, Relations} from 'idai-components-2';
import {RESOURCE_ID, RESOURCE_IDENTIFIER} from '../../../c';
import {HIERARCHICAL_RELATIONS, PARENT} from '../../model/relation-constants';
import {processRelations} from './process-relations';
import {Find, GenerateId, Get, GetInverseRelation, Id, Identifier, IdentifierMap, ProcessResult} from './types';
import {processDocuments} from './process-documents';
import LIES_WITHIN = HIERARCHICAL_RELATIONS.LIES_WITHIN;
import {assertLegalCombination} from './utils';


/**
 * @author <NAME>
 * @author <NAME>
 */
export function build(validator: ImportValidator,
                      operationTypeNames: string[],
                      generateId: GenerateId,
                      find: Find,
                      get: Get,
                      getInverseRelation: GetInverseRelation,
                      mergeMode: boolean,
                      allowOverwriteRelationsInMergeMode: boolean,
                      mainTypeDocumentId: Id,
                      useIdentifiersInRelations: boolean) {

    assertLegalCombination(mainTypeDocumentId, mergeMode);

    /**
     * ImportErrors (accessible via ProcessResult)
     *
     * [MUST_LIE_WITHIN_OTHER_NON_OPERATON_RESOURCE, resourceType, resourceIdentifier]
     *   - if a resource of
     *     a defined builtin type should be placed inside another resource of a legal LIES_WITHIN range type, but is placed
     *     directly below an operation.
     *
     * [BAD_INTERRELATION, sourceId]
     *   - if opposing relations are pointing to the same resource.
     *     For example IS_BEFORE and IS_AFTER pointing both from document '1' to '2'.
     *   - if mutually exluding relations are pointing to the same resource.
     *     For example IS_CONTEMPORARY_WITH and IS_AFTER both from document '1' to '2'.
     *
     * [TARGET_TYPE_RANGE_MISMATCH, resourceIdentifier, relationName, relationTargetResourceType]
     *   - if a resource points to another resource, however, the specified relation is not allowed between the
     *     types of the resources.
     *
     * [PARENT_ASSIGNMENT_TO_OPERATIONS_NOT_ALLOWED]
     *   - if mainTypeDocumentId is not '' and
     *     a resource references an operation as parent.
     *
     * [LIES_WITHIN_TARGET_NOT_MATCHES_ON_IS_RECORDED_IN, resourceIdentifier]
     *   - if the inferredRecordedIn
     *     differs from a possibly specified recordedIn.
     *
     * [EXEC_MISSING_RELATION_TARGET, targetId]
     *
     * @throws [EMPTY_RELATION, resourceId]
     *   - if relations empty for some relation is empty.
     *     For example relations: {isAbove: []}
     */
    return async function process(documents: Array<Document>): Promise<ProcessResult> {

        try {
            assertNoDuplicates(documents);

            const identifierMap: IdentifierMap = mergeMode ? {} : assignIds(documents, generateId);
            const rewriteIdentifiersInRels = rewriteIdentifiersInRelations(find, identifierMap);
            const assertNoMissingRelTargets = assertNoMissingRelationTargets(get);

            await preprocessAndValidateRelations(
                documents,
                mergeMode,
                allowOverwriteRelationsInMergeMode,
                useIdentifiersInRelations,
                assertNoMissingRelTargets,
                rewriteIdentifiersInRels);

            const processedDocuments = await processDocuments(
                validator, mergeMode, allowOverwriteRelationsInMergeMode, find)(documents);

            const relatedDocuments = await processRelations(
                processedDocuments,
                validator, operationTypeNames,
                mergeMode, allowOverwriteRelationsInMergeMode,
                getInverseRelation,
                get,
                mainTypeDocumentId);

            return [processedDocuments, relatedDocuments, undefined];

        } catch (errWithParams) {

            return [[],[], errWithParams];
        }
    }
}



function assertNoDuplicates(documents: Array<Document>) {

    const dups = duplicates(documents.map(to(RESOURCE_IDENTIFIER)));
    if (dups.length > 0) throw [E.DUPLICATE_IDENTIFIER, dups[0]];
}


function adjustRelations(document: Document, relations: Relations) {

    assertHasNoHierarchicalRelations(document);
    const assertIsntArrayRelation = assertIsNotArrayRelation(document);

    Object.keys(document.resource.relations)
        .filter(isnt(PARENT))
        .forEach(assertIsntArrayRelation);

    assertParentNotArray(relations[PARENT], document.resource.identifier);
    if (relations[PARENT]) (relations[LIES_WITHIN] = [relations[PARENT] as any]) && delete relations[PARENT];
}


function assertParentNotArray(parentRelation: any, resourceIdentifier: string) {

    if (isArray(parentRelation)) throw [E.PARENT_MUST_NOT_BE_ARRAY, resourceIdentifier];
}


/**
 * Hierarchical relations are not used directly but instead one uses PARENT.
 */
function assertHasNoHierarchicalRelations(document: Document) {

    const foundForbiddenRelations = Object.keys(document.resource.relations)
        .filter(includedIn(HIERARCHICAL_RELATIONS.ALL))
        .join(', ');
    if (foundForbiddenRelations) throw [E.INVALID_RELATIONS, document.resource.type, foundForbiddenRelations];
}


function assertIsNotArrayRelation(document: Document) {

    return (name: string) => {

        if (not(isArray)(document.resource.relations[name])) throw [E.MUST_BE_ARRAY, document.resource.identifier];
    }
}


async function preprocessAndValidateRelations(
    documents: Array<Document>,
    mergeMode: boolean,
    allowOverwriteRelationsInMergeMode: boolean,
    useIdentifiersInRelations: boolean,
    assertNoMissingRelTargets: Function,
    rewriteIdentifiersInRelations: Function): Promise<void> {
{'fraction_non_alphanumeric': 0.08564294631710362, 'fraction_numerical': 0.001165210153974199, 'mean_word_length': 3.1221269296740997, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6695260', 'n_tokens_mistral': 3378, 'n_tokens_neox': 3158, 'n_words': 770}