<commit_before>import {TypeUtil} from "../Utility/TypeUtil";
import {DocumentInfo} from "../Documents/Session/DocumentInfo";
import { DocumentsChanges, ChangeType } from "../Documents/Session/DocumentsChanges";
import { CONSTANTS } from "../Constants";
import { throwError } from "../Exceptions";

export class JsonOperation {

    public static entityChanged(
        newObj: object, 
        documentInfo: DocumentInfo, 
        changes: { [id: string]: DocumentsChanges[] }): boolean {

        const docChanges: DocumentsChanges[] = changes ? [] : null;

        if (!documentInfo.newDocument && documentInfo.document) {
            return JsonOperation._compareJson(documentInfo.id, documentInfo.document, newObj, changes, docChanges);
        }

        if (!changes) {
            return true;
        }

        JsonOperation._newChange(null, null, null, docChanges, "DocumentAdded");
        changes[documentInfo.id] = docChanges;
        return true;
    }

    private static _compareJson(
        id: string, 
        originalJson: object, 
        newJson: object, 
        changes: { [id: string]: DocumentsChanges[] }, 
        docChanges: DocumentsChanges[]): boolean {
        const newJsonProps: string[] = Object.keys(newJson);
        const oldJsonProps: string[] = Object.keys(originalJson);

        const newFields = newJsonProps.filter(x => !oldJsonProps.find(y => y === x));
        const removedFields = oldJsonProps.filter(x => !newJsonProps.find(y => y === x));

        for (const field of removedFields) {
            if (!changes) {
                return true;
            }

            JsonOperation._newChange(field, null, null, docChanges, "RemovedField");
        }

        for (const prop of newJsonProps) {

            if (CONSTANTS.Documents.Metadata.LAST_MODIFIED === prop ||
                    CONSTANTS.Documents.Metadata.COLLECTION === prop ||
                    CONSTANTS.Documents.Metadata.CHANGE_VECTOR === prop ||
                    CONSTANTS.Documents.Metadata.ID === prop ||
                    CONSTANTS.Documents.Metadata.KEY === prop) {
                continue;
            }

            if (newFields.find(x => x === prop)) {
                if (!changes) {
                    return true;
                }

                JsonOperation._newChange(prop, newJson[prop], null, docChanges, "NewField");
                continue;
            }

            const newProp = newJson[prop];
            const oldProp = originalJson[prop];

            let changed: boolean;
            const typeOfNewProp = typeof newProp;
            if (typeOfNewProp === "number"
                || typeOfNewProp === "boolean"
                || typeOfNewProp === "string") {

                if (newProp === oldProp || JsonOperation._compareValues(oldProp, newProp)) {
                    continue;
                }
                if (!changes) {
                    return true;
                }

                JsonOperation._newChange(prop, newProp, oldProp, docChanges, "FieldChanged");
            } else if (!newProp) {
                if (oldProp) {
                    if (!changes) {
                        return true;
                    }

                    JsonOperation._newChange(prop, null, oldProp, docChanges, "FieldChanged");
                }
            } else if (Array.isArray(newProp)) {
                if (!oldProp || !Array.isArray(oldProp)) {
                    if (!changes) {
                        return true;
                    }

                    JsonOperation._newChange(prop, newProp, oldProp, docChanges, "FieldChanged");
                }

                changed = JsonOperation._compareJsonArray(
                    id, oldProp as any[], newProp as any[], changes, docChanges, prop);
                if (!changes && changed) {
                    return true;
                }
            } else if (TypeUtil.isObject(newProp)) {
                    if (!oldProp) {<commit_msg>code cleanup sync with java: 2d2c788c5644a986db322d719f45452909d36348<commit_after>import {TypeUtil} from "../Utility/TypeUtil";
import {DocumentInfo} from "../Documents/Session/DocumentInfo";
import { DocumentsChanges, ChangeType } from "../Documents/Session/DocumentsChanges";
import { CONSTANTS } from "../Constants";
import { throwError } from "../Exceptions";

export class JsonOperation {

    public static entityChanged(
        newObj: object, 
        documentInfo: DocumentInfo, 
        changes: { [id: string]: DocumentsChanges[] }): boolean {

        const docChanges: DocumentsChanges[] = changes ? [] : null;

        if (!documentInfo.newDocument && documentInfo.document) {
            return JsonOperation._compareJson(documentInfo.id, documentInfo.document, newObj, changes, docChanges);
        }

        if (!changes) {
            return true;
        }

        JsonOperation._newChange(null, null, null, docChanges, "DocumentAdded");
        changes[documentInfo.id] = docChanges;
        return true;
    }

    private static _compareJson(
        id: string, 
        originalJson: object, 
        newJson: object, 
        changes: { [id: string]: DocumentsChanges[] }, 
        docChanges: DocumentsChanges[]): boolean {
        const newJsonProps: string[] = Object.keys(newJson);
        const oldJsonProps: string[] = Object.keys(originalJson);

        const newFields = newJsonProps.filter(x => !oldJsonProps.find(y => y === x));
        const removedFields = oldJsonProps.filter(x => !newJsonProps.find(y => y === x));

        for (const field of removedFields) {
            if (!changes) {
                return true;
            }

            JsonOperation._newChange(field, null, null, docChanges, "RemovedField");
        }

        for (const prop of newJsonProps) {

            if (CONSTANTS.Documents.Metadata.LAST_MODIFIED === prop ||
                    CONSTANTS.Documents.Metadata.COLLECTION === prop ||
                    CONSTANTS.Documents.Metadata.CHANGE_VECTOR === prop ||
                    CONSTANTS.Documents.Metadata.ID === prop ||
                    CONSTANTS.Documents.Metadata.KEY === prop) {
                continue;
            }

            if (newFields.find(x => x === prop)) {
                if (!changes) {
                    return true;
                }

                JsonOperation._newChange(prop, newJson[prop], null, docChanges, "NewField");
                continue;
            }

            const newProp = newJson[prop];
            const oldProp = originalJson[prop];

            let changed: boolean;
            const typeOfNewProp = typeof newProp;
            if (typeOfNewProp === "number"
                || typeOfNewProp === "boolean"
                || typeOfNewProp === "string") {

                if (newProp === oldProp || JsonOperation._compareValues(oldProp, newProp)) {
                    continue;
                }
                if (!changes) {
                    return true;
                }

                JsonOperation._newChange(prop, newProp, oldProp, docChanges, "FieldChanged");
            } else if (!newProp) {
                if (oldProp) {
                    if (!changes) {
                        return true;
                    }

                    JsonOperation._newChange(prop, null, oldProp, docChanges, "FieldChanged");
                }
            } else if (Array.isArray(newProp)) {
                if (!Array.isArray(oldProp)) {
                    if (!changes) {
                        return true;
                    }

                    JsonOperation._newChange(prop, newProp, oldProp, docChanges, "FieldChanged");
                }

                changed = JsonOperation._compareJsonArray(
                    id, oldProp as any[], newProp as any[], changes, docChanges, prop);
                if (!changes && changed) {
                    return true;
                }
            } else if (TypeUtil.isObject(newProp)) {
                    if (!oldProp) {
{'fraction_non_alphanumeric': 0.11027756939234809, 'fraction_numerical': 0.003875968992248062, 'mean_word_length': 1.67346256684492, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5274893', 'n_tokens_mistral': 2088, 'n_tokens_neox': 1924, 'n_words': 464}