<commit_before>'use strict';

const debug = require('ghost-ignition').debug('importer:base'),
    common = require('../../../../lib/common'),
    models = require('../../../../models'),
    _ = require('lodash'),
    Promise = require('bluebird');

class Base {
    constructor(options) {
        let self = this;

        this.modelName = options.modelName;

        this.problems = [];

        this.errorConfig = {
            allowDuplicates: true,
            returnDuplicates: true,
            showNotFoundWarning: true
        };

        this.legacyKeys = {};
        this.legacyMapper = function legacyMapper(item) {
            return _.mapKeys(item, function matchLegacyKey(value, key) {
                return self.legacyKeys[key] || key;
            });
        };

        this.dataKeyToImport = options.dataKeyToImport;
        this.dataToImport = _.cloneDeep(options[this.dataKeyToImport] || []);
        this.importedData = [];

        // NOTE: e.g. properties are removed or properties are added/changed before importing
        _.each(options, function (obj, key) {
            if (options.requiredData.indexOf(key) !== -1) {
                self[key] = _.cloneDeep(obj);
            }
        });

        if (!this.users) {
            this.users = _.cloneDeep(options.users);
        }
    }

    /**
     * Never ever import these attributes!
     */
    stripProperties(properties) {
        _.each(this.dataToImport, function (obj) {
            _.each(properties, function (property) {
                delete obj[property];
            });
        });
    }

    /**
     * Clean invalid values.
     */
    sanitizeValues() {
        let temporaryDate, self = this;

        _.each(this.dataToImport, function (obj) {
            _.each(_.pick(obj, ['updated_at', 'created_at', 'published_at']), function (value, key) {
                temporaryDate = new Date(value);

                if (isNaN(temporaryDate)) {
                    self.problems.push({
                        message: 'Date is in a wrong format and invalid. It was replaced with the current timestamp.',
                        help: self.modelName,
                        context: JSON.stringify(obj)
                    });

                    obj[key] = new Date().toISOString();
                }
            });
        });
    }

    beforeImport() {
        this.stripProperties(['id']);
        this.sanitizeValues();
        return Promise.resolve();
    }

    handleError(errs, obj) {
        let self = this, errorsToReject = [], problems = [];

        // CASE: validation errors, see models/base/events.js onValidate
        if (!_.isArray(errs)) {
            errs = [errs];
        }

        _.each(errs, function (err) {
            if (err.code && err.message.toLowerCase().indexOf('unique') !== -1) {
                if (self.errorConfig.allowDuplicates) {
                    if (self.errorConfig.returnDuplicates) {
                        problems.push({
                            message: 'Entry was not imported and ignored. Detected duplicated entry.',
                            help: self.modelName,
                            context: JSON.stringify(obj),
                            err: err
                        });
                    }
                } else {
                    errorsToReject.push(new common.errors.DataImportError({
                        message: 'Detected duplicated entry.',
                        help: self.modelName,
                        context: JSON.stringify(obj),
                        err: err
                    }));
                }
            } else if (err instanceof common.errors.NotFoundError) {
                if (self.showNotFoundWarning) {
                    problems.push({
                        message: 'Entry was not imported and ignored. Could not find entry.',
                        help: self.modelName,
                        context: JSON.stringify(obj),
                        err: err
                    });
                }
            } else {
                if (!common.errors.utils.isIgnitionError(err)) {
                    err = new common.errors.DataImportError({
                        message: err.message,
                        context: JSON.stringify(obj),
                        help: self.modelName,
                        errorType: err.errorType,
                        err: err
                    });
                } else {
                    err.context = JSON.stringify(obj);
                }

                errorsToReject.push(err);
            }
        });

        if (!errorsToReject.length) {
            this.problems = this.problems.concat(problems);
            debug('detected problem/warning', problems);

            return Promise.resolve();
        }

        debug('err', errorsToReject, obj);
        return Promise.reject(errorsToReject);
    }

    doImport(options) {
        debug('doImport', this.modelName, this.dataToImport.length);

        let self = this, ops = [];

        _.each(this.dataToImport, function (obj) {
            ops.push(models[self.modelName].add(obj, options)
                .then(function (importedModel) {
                    obj.model = importedModel.toJSON();
                    self.importedData.push(obj.model);
                    return importedModel;
                })
                .catch(function (err) {
                    return self.handleError(err, obj);
                })
                .reflect()
            );
        });

        return Promise.all(ops);
    }

    /**
     * Update all user reference fields e.g. published_by
     *
     * Background:
     *  - we never import the id field
     *  - almost each imported model has a reference to a user reference
     *  - we update all fields after the import (!)
     */
    afterImport(options) {
        let self = this, dataToEdit = {}, oldUser, context;

        debug('afterImport', this.modelName);

        return models.User.getOwnerUser(options)<commit_msg>🐛  Fixed large imports<commit_after>'use strict';

const debug = require('ghost-ignition').debug('importer:base'),
    common = require('../../../../lib/common'),
    sequence = require('../../../../lib/promise/sequence'),
    models = require('../../../../models'),
    _ = require('lodash'),
    Promise = require('bluebird');

class Base {
    constructor(options) {
        let self = this;

        this.modelName = options.modelName;

        this.problems = [];

        this.errorConfig = {
            allowDuplicates: true,
            returnDuplicates: true,
            showNotFoundWarning: true
        };

        this.legacyKeys = {};
        this.legacyMapper = function legacyMapper(item) {
            return _.mapKeys(item, function matchLegacyKey(value, key) {
                return self.legacyKeys[key] || key;
            });
        };

        this.dataKeyToImport = options.dataKeyToImport;
        this.dataToImport = _.cloneDeep(options[this.dataKeyToImport] || []);
        this.importedData = [];

        // NOTE: e.g. properties are removed or properties are added/changed before importing
        _.each(options, function (obj, key) {
            if (options.requiredData.indexOf(key) !== -1) {
                self[key] = _.cloneDeep(obj);
            }
        });

        if (!this.users) {
            this.users = _.cloneDeep(options.users);
        }
    }

    /**
     * Never ever import these attributes!
     */
    stripProperties(properties) {
        _.each(this.dataToImport, function (obj) {
            _.each(properties, function (property) {
                delete obj[property];
            });
        });
    }

    /**
     * Clean invalid values.
     */
    sanitizeValues() {
        let temporaryDate, self = this;

        _.each(this.dataToImport, function (obj) {
            _.each(_.pick(obj, ['updated_at', 'created_at', 'published_at']), function (value, key) {
                temporaryDate = new Date(value);

                if (isNaN(temporaryDate)) {
                    self.problems.push({
                        message: 'Date is in a wrong format and invalid. It was replaced with the current timestamp.',
                        help: self.modelName,
                        context: JSON.stringify(obj)
                    });

                    obj[key] = new Date().toISOString();
                }
            });
        });
    }

    beforeImport() {
        this.stripProperties(['id']);
        this.sanitizeValues();
        return Promise.resolve();
    }

    handleError(errs, obj) {
        let self = this, errorsToReject = [], problems = [];

        // CASE: validation errors, see models/base/events.js onValidate
        if (!_.isArray(errs)) {
            errs = [errs];
        }

        _.each(errs, function (err) {
            if (err.code && err.message.toLowerCase().indexOf('unique') !== -1) {
                if (self.errorConfig.allowDuplicates) {
                    if (self.errorConfig.returnDuplicates) {
                        problems.push({
                            message: 'Entry was not imported and ignored. Detected duplicated entry.',
                            help: self.modelName,
                            context: JSON.stringify(obj),
                            err: err
                        });
                    }
                } else {
                    errorsToReject.push(new common.errors.DataImportError({
                        message: 'Detected duplicated entry.',
                        help: self.modelName,
                        context: JSON.stringify(obj),
                        err: err
                    }));
                }
            } else if (err instanceof common.errors.NotFoundError) {
                if (self.showNotFoundWarning) {
                    problems.push({
                        message: 'Entry was not imported and ignored. Could not find entry.',
                        help: self.modelName,
                        context: JSON.stringify(obj),
                        err: err
                    });
                }
            } else {
                if (!common.errors.utils.isIgnitionError(err)) {
                    err = new common.errors.DataImportError({
                        message: err.message,
                        context: JSON.stringify(obj),
                        help: self.modelName,
                        errorType: err.errorType,
                        err: err
                    });
                } else {
                    err.context = JSON.stringify(obj);
                }

                errorsToReject.push(err);
            }
        });

        if (!errorsToReject.length) {
            this.problems = this.problems.concat(problems);
            debug('detected problem/warning', problems);

            return Promise.resolve();
        }

        debug('err', errorsToReject, obj);
        return Promise.reject(errorsToReject);
    }

    doImport(options) {
        debug('doImport', this.modelName, this.dataToImport.length);

        let self = this, ops = [];

        _.each(this.dataToImport, function forEachDataToImport(obj) {
            ops.push(function addModel() {
                return models[self.modelName].add(obj, options)
                    .then(function (importedModel) {
                        obj.model = importedModel.toJSON();
                        self.importedData.push(obj.model);
                        return importedModel;
                    })
                    .catch(function (err) {
                        return self.handleError(err, obj);
                    })
                    .reflect();
            });
        });

        /**
         * NOTE: Do not run with Promise.all in this case. With a large import file, we run an enormous
         *       amount of queries in parallel. Node will very fast eat lot's of memory, because all queries start
         *       at the same time, but memory can only be released if the query finished.
         *
         *       Promise.map(.., {concurrency: Int}) was not really improving the end performance for me.
         */
        return sequence(ops);
    }

    /**
     * Update all user reference fields e.g. published_by
     *
     * Background:
     *  - we never import the id field
     *  - almost each imported model has a reference to a user reference
     *  - we update all fields after the import (!)
     */
    afterImport(options) {
        let self = this, dataToEdit = {}, oldUser, context;

        debug('afterImport', this.modelName);

        return models.User.getOwnerUser(options)
{'fraction_non_alphanumeric': 0.11289560699102504, 'fraction_numerical': 0.000314911037631869, 'mean_word_length': 1.3624697786870001, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7143040', 'n_tokens_mistral': 3242, 'n_tokens_neox': 3072, 'n_words': 714}