<commit_before>const _ = require('lodash');
const Promise = require('bluebird');
const fs = require('fs-extra');
const path = require('path');
const os = require('os');
const glob = require('glob');
const uuid = require('uuid');
const {extract} = require('@tryghost/zip');
const {pipeline, sequence} = require('@tryghost/promise');
const i18n = require('../../../shared/i18n');
const logging = require('@tryghost/logging');
const errors = require('@tryghost/errors');
const ImageHandler = require('./handlers/image');
const JSONHandler = require('./handlers/json');
const MarkdownHandler = require('./handlers/markdown');
const ImageImporter = require('./importers/image');
const DataImporter = require('./importers/data');

// Glob levels
const ROOT_ONLY = 0;

const ROOT_OR_SINGLE_DIR = 1;
const ALL_DIRS = 2;
let defaults;

defaults = {
    extensions: ['.zip'],
    contentTypes: ['application/zip', 'application/x-zip-compressed'],
    directories: []
};

class ImportManager {
    constructor() {
        this.importers = [ImageImporter, DataImporter];
        this.handlers = [ImageHandler, JSONHandler, MarkdownHandler];

        // Keep track of file to cleanup at the end
        this.fileToDelete = null;
    }

    /**
     * Get an array of all the file extensions for which we have handlers
     * @returns {string[]}
     */
    getExtensions() {
        return _.flatten(_.union(_.map(this.handlers, 'extensions'), defaults.extensions));
    }

    /**
     * Get an array of all the mime types for which we have handlers
     * @returns {string[]}
     */
    getContentTypes() {
        return _.flatten(_.union(_.map(this.handlers, 'contentTypes'), defaults.contentTypes));
    }

    /**
     * Get an array of directories for which we have handlers
     * @returns {string[]}
     */
    getDirectories() {
        return _.flatten(_.union(_.map(this.handlers, 'directories'), defaults.directories));
    }

    /**
     * Convert items into a glob string
     * @param {String[]} items
     * @returns {String}
     */
    getGlobPattern(items) {
        return '+(' + _.reduce(items, function (memo, ext) {
            return memo !== '' ? memo + '|' + ext : ext;
        }, '') + ')';
    }

    /**
     * @param {String[]} extensions
     * @param {Number} level
     * @returns {String}
     */
    getExtensionGlob(extensions, level) {
        const prefix = level === ALL_DIRS ? '**/*' :
            (level === ROOT_OR_SINGLE_DIR ? '{*/*,*}' : '*');

        return prefix + this.getGlobPattern(extensions);
    }

    /**
     *
     * @param {String[]} directories
     * @param {Number} level
     * @returns {String}
     */
    getDirectoryGlob(directories, level) {
        const prefix = level === ALL_DIRS ? '**/' :
            (level === ROOT_OR_SINGLE_DIR ? '{*/,}' : '');

        return prefix + this.getGlobPattern(directories);
    }

    /**
     * Remove files after we're done (abstracted into a function for easier testing)
     * @returns {Function}
     */
    cleanUp() {
        const self = this;

        if (self.fileToDelete === null) {
            return;
        }

        fs.remove(self.fileToDelete, function (err) {
            if (err) {
                logging.error(new errors.GhostError({
                    err: err,
                    context: i18n.t('errors.data.importer.index.couldNotCleanUpFile.error'),
                    help: i18n.t('errors.data.importer.index.couldNotCleanUpFile.context')
                }));
            }

            self.fileToDelete = null;
        });
    }

    /**
     * Return true if the given file is a Zip
     * @returns Boolean
     */
    isZip(ext) {
        return _.includes(defaults.extensions, ext);
    }

    /**
     * Checks the content of a zip folder to see if it is valid.
     * Importable content includes any files or directories which the handlers can process
     * Importable content must be found either in the root, or inside one base directory
     *
     * @param {String} directory
     * @returns {Promise}
     */
    isValidZip(directory) {
        // Globs match content in the root or inside a single directory
        const extMatchesBase = glob.sync(this.getExtensionGlob(this.getExtensions(), ROOT_OR_SINGLE_DIR), {cwd: directory});

        const extMatchesAll = glob.sync(
            this.getExtensionGlob(this.getExtensions(), ALL_DIRS), {cwd: directory}
        );

        const dirMatches = glob.sync(
            this.getDirectoryGlob(this.getDirectories(), ROOT_OR_SINGLE_DIR), {cwd: directory}
        );

        const oldRoonMatches = glob.sync(this.getDirectoryGlob(['drafts', 'published', 'deleted'], ROOT_OR_SINGLE_DIR),
            {cwd: directory});

        // This is a temporary extra message for the old format roon export which doesn't work with Ghost
        if (oldRoonMatches.length > 0) {
            throw new errors.UnsupportedMediaTypeError({message: i18n.t('errors.data.importer.index.unsupportedRoonExport')});
        }

        // If this folder contains importable files or a content or images directory
        if (extMatchesBase.length > 0 || (dirMatches.length > 0 && extMatchesAll.length > 0)) {
            return true;
        }

        if (extMatchesAll.length < 1) {
            throw new errors.UnsupportedMediaTypeError({message: i18n.t('errors.data.importer.index.noContentToImport')});
        }

        throw new errors.UnsupportedMediaTypeError({message: i18n.t('errors.data.importer.index.invalidZipStructure')});
    }

    /**
     * Use the extract module to extract the given zip file to a temp directory & return the temp directory path
     * @param {String} filePath
     * @returns {Promise[]} Files
     */
    extractZip(filePath) {
        const tmpDir = path.join(os.tmpdir(), uuid.v4());
        this.fileToDelete = tmpDir;

        return extract(filePath, tmpDir).then(function () {
            return tmpDir;
        });
    }

    /**
     * Use the handler extensions to get a globbing pattern, then use that to fetch all the files from the zip which
     * are relevant to the given handler, and return them as a name and path combo
     * @param {Object} handler
     * @param {String} directory
     * @returns [] Files
     */
    getFilesFromZip(handler, directory) {
        const globPattern = this.getExtensionGlob(handler.extensions, ALL_DIRS);
        return _.map(glob.sync(globPattern, {cwd: directory}), function (file) {
            return {name: file, path: path.join(directory, file)};
        });
    }

    /**
     * Get the name of the single base directory if there is one, else return an empty string
     * @param {String} directory
     * @returns {String}
     */
    getBaseDirectory(directory) {
        // Globs match root level only
        const extMatches = glob.sync(this.getExtensionGlob(this.getExtensions(), ROOT_ONLY), {cwd: directory});

        const dirMatches = glob.sync(this.getDirectoryGlob(this.getDirectories(), ROOT_ONLY), {cwd: directory});
        let extMatchesAll;

        // There is no base directory
        if (extMatches.length > 0 || dirMatches.length > 0) {
            return;
        }
        // There is a base directory, grab it from any ext match
        extMatchesAll = glob.sync(
            this.getExtensionGlob(this.getExtensions(), ALL_DIRS), {cwd: directory}
        );
        if (extMatchesAll.length < 1 || extMatchesAll[0].split('/') < 1) {
            throw new errors.ValidationError({message: i18n.t('errors.data.importer.index.invalidZipFileBaseDirectory')});
        }

        return extMatchesAll[0].split('/')[0];
    }

    /**
     * Process Zip
     * Takes a reference to a zip file, extracts it, sends any relevant files from inside to the right handler, and
     * returns an object in the importData format: {data: {}, images: []}
     * The data key contains JSON representing any data that should be imported
     * The image key contains references to images that will be stored (and where they will be stored)
     * @param {File} file
     * @returns {Promise(ImportData)}
     */
    processZip(file) {
        const self = this;

        return this.extractZip(file.path).then(function (zipDirectory) {
            const ops = [];
            const importData = {};
            let baseDir;

            self.isValidZip(zipDirectory);
            baseDir = self.getBaseDirectory(zipDirectory);

            _.each(self.handlers, function (handler) {
                if (Object.prototype.hasOwnProperty.call(importData, handler.type)) {
                    // This limitation is here to reduce the complexity of the importer for now
                    return Promise.reject(new errors.UnsupportedMediaTypeError({
                        message: i18n.t('errors.data.importer.index.zipContainsMultipleDataFormats')
                    }));
                }

                const files = self.getFilesFromZip(handler, zipDirectory);

                if (files.length > 0) {
                    ops.push(function () {
                        return handler.loadFile(files, baseDir).then(function (data) {
                            importData[handler.type] = data;
                        });
                    });
                }
            });

            if (ops.length === 0) {
                return Promise.reject(new errors.UnsupportedMediaTypeError({
                    message: i18n.t('errors.data.importer.index.noContentToImport')
                }));
            }
<commit_msg>Replaced i18n.t w/ tpl helper in import-manager.js refs: TryGhost#13380<commit_after>const _ = require('lodash');
const Promise = require('bluebird');
const fs = require('fs-extra');
const path = require('path');
const os = require('os');
const glob = require('glob');
const uuid = require('uuid');
const {extract} = require('@tryghost/zip');
const {pipeline, sequence} = require('@tryghost/promise');
const tpl = require('@tryghost/tpl');
const logging = require('@tryghost/logging');
const errors = require('@tryghost/errors');
const ImageHandler = require('./handlers/image');
const JSONHandler = require('./handlers/json');
const MarkdownHandler = require('./handlers/markdown');
const ImageImporter = require('./importers/image');
const DataImporter = require('./importers/data');

// Glob levels
const ROOT_ONLY = 0;

const ROOT_OR_SINGLE_DIR = 1;
const ALL_DIRS = 2;
let defaults;

const messages = {
    couldNotCleanUpFile: {
        error: 'Import could not clean up file ',
        context: 'Your site will continue to work as expected'
    },
    unsupportedRoonExport: 'Your zip file looks like an old format Roon export, please re-export your Roon blog and try again.',
    noContentToImport: 'Zip did not include any content to import.',
    invalidZipStructure: 'Invalid zip file structure.',
    invalidZipFileBaseDirectory: 'Invalid zip file: base directory read failed',
    zipContainsMultipleDataFormats: 'Zip file contains multiple data formats. Please split up and import separately.'
};

defaults = {
    extensions: ['.zip'],
    contentTypes: ['application/zip', 'application/x-zip-compressed'],
    directories: []
};

class ImportManager {
    constructor() {
        this.importers = [ImageImporter, DataImporter];
        this.handlers = [ImageHandler, JSONHandler, MarkdownHandler];

        // Keep track of file to cleanup at the end
        this.fileToDelete = null;
    }

    /**
     * Get an array of all the file extensions for which we have handlers
     * @returns {string[]}
     */
    getExtensions() {
        return _.flatten(_.union(_.map(this.handlers, 'extensions'), defaults.extensions));
    }

    /**
     * Get an array of all the mime types for which we have handlers
     * @returns {string[]}
     */
    getContentTypes() {
        return _.flatten(_.union(_.map(this.handlers, 'contentTypes'), defaults.contentTypes));
    }

    /**
     * Get an array of directories for which we have handlers
     * @returns {string[]}
     */
    getDirectories() {
        return _.flatten(_.union(_.map(this.handlers, 'directories'), defaults.directories));
    }

    /**
     * Convert items into a glob string
     * @param {String[]} items
     * @returns {String}
     */
    getGlobPattern(items) {
        return '+(' + _.reduce(items, function (memo, ext) {
            return memo !== '' ? memo + '|' + ext : ext;
        }, '') + ')';
    }

    /**
     * @param {String[]} extensions
     * @param {Number} level
     * @returns {String}
     */
    getExtensionGlob(extensions, level) {
        const prefix = level === ALL_DIRS ? '**/*' :
            (level === ROOT_OR_SINGLE_DIR ? '{*/*,*}' : '*');

        return prefix + this.getGlobPattern(extensions);
    }

    /**
     *
     * @param {String[]} directories
     * @param {Number} level
     * @returns {String}
     */
    getDirectoryGlob(directories, level) {
        const prefix = level === ALL_DIRS ? '**/' :
            (level === ROOT_OR_SINGLE_DIR ? '{*/,}' : '');

        return prefix + this.getGlobPattern(directories);
    }

    /**
     * Remove files after we're done (abstracted into a function for easier testing)
     * @returns {Function}
     */
    cleanUp() {
        const self = this;

        if (self.fileToDelete === null) {
            return;
        }

        fs.remove(self.fileToDelete, function (err) {
            if (err) {
                logging.error(new errors.GhostError({
                    err: err,
                    context: tpl(messages.couldNotCleanUpFile.error),
                    help: tpl(messages.couldNotCleanUpFile.context)
                }));
            }

            self.fileToDelete = null;
        });
    }

    /**
     * Return true if the given file is a Zip
     * @returns Boolean
     */
    isZip(ext) {
        return _.includes(defaults.extensions, ext);
    }

    /**
     * Checks the content of a zip folder to see if it is valid.
     * Importable content includes any files or directories which the handlers can process
     * Importable content must be found either in the root, or inside one base directory
     *
     * @param {String} directory
     * @returns {Promise}
     */
    isValidZip(directory) {
        // Globs match content in the root or inside a single directory
        const extMatchesBase = glob.sync(this.getExtensionGlob(this.getExtensions(), ROOT_OR_SINGLE_DIR), {cwd: directory});

        const extMatchesAll = glob.sync(
            this.getExtensionGlob(this.getExtensions(), ALL_DIRS), {cwd: directory}
        );

        const dirMatches = glob.sync(
            this.getDirectoryGlob(this.getDirectories(), ROOT_OR_SINGLE_DIR), {cwd: directory}
        );

        const oldRoonMatches = glob.sync(this.getDirectoryGlob(['drafts', 'published', 'deleted'], ROOT_OR_SINGLE_DIR),
            {cwd: directory});

        // This is a temporary extra message for the old format roon export which doesn't work with Ghost
        if (oldRoonMatches.length > 0) {
            throw new errors.UnsupportedMediaTypeError({message: tpl(messages.unsupportedRoonExport)});
        }

        // If this folder contains importable files or a content or images directory
        if (extMatchesBase.length > 0 || (dirMatches.length > 0 && extMatchesAll.length > 0)) {
            return true;
        }

        if (extMatchesAll.length < 1) {
            throw new errors.UnsupportedMediaTypeError({message: tpl(messages.noContentToImport)});
        }

        throw new errors.UnsupportedMediaTypeError({message: tpl(messages.invalidZipStructure)});
    }

    /**
     * Use the extract module to extract the given zip file to a temp directory & return the temp directory path
     * @param {String} filePath
     * @returns {Promise[]} Files
     */
    extractZip(filePath) {
        const tmpDir = path.join(os.tmpdir(), uuid.v4());
        this.fileToDelete = tmpDir;

        return extract(filePath, tmpDir).then(function () {
            return tmpDir;
        });
    }

    /**
     * Use the handler extensions to get a globbing pattern, then use that to fetch all the files from the zip which
     * are relevant to the given handler, and return them as a name and path combo
     * @param {Object} handler
     * @param {String} directory
     * @returns [] Files
     */
    getFilesFromZip(handler, directory) {
        const globPattern = this.getExtensionGlob(handler.extensions, ALL_DIRS);
        return _.map(glob.sync(globPattern, {cwd: directory}), function (file) {
            return {name: file, path: path.join(directory, file)};
        });
    }

    /**
     * Get the name of the single base directory if there is one, else return an empty string
     * @param {String} directory
     * @returns {String}
     */
    getBaseDirectory(directory) {
        // Globs match root level only
        const extMatches = glob.sync(this.getExtensionGlob(this.getExtensions(), ROOT_ONLY), {cwd: directory});

        const dirMatches = glob.sync(this.getDirectoryGlob(this.getDirectories(), ROOT_ONLY), {cwd: directory});
        let extMatchesAll;

        // There is no base directory
        if (extMatches.length > 0 || dirMatches.length > 0) {
            return;
        }
        // There is a base directory, grab it from any ext match
        extMatchesAll = glob.sync(
            this.getExtensionGlob(this.getExtensions(), ALL_DIRS), {cwd: directory}
        );
        if (extMatchesAll.length < 1 || extMatchesAll[0].split('/') < 1) {
            throw new errors.ValidationError({message: tpl(messages.invalidZipFileBaseDirectory)});
        }

        return extMatchesAll[0].split('/')[0];
    }

    /**
     * Process Zip
     * Takes a reference to a zip file, extracts it, sends any relevant files from inside to the right handler, and
     * returns an object in the importData format: {data: {}, images: []}
     * The data key contains JSON representing any data that should be imported
     * The image key contains references to images that will be stored (and where they will be stored)
     * @param {File} file
     * @returns {Promise(ImportData)}
     */
    processZip(file) {
        const self = this;

        return this.extractZip(file.path).then(function (zipDirectory) {
            const ops = [];
            const importData = {};
            let baseDir;

            self.isValidZip(zipDirectory);
            baseDir = self.getBaseDirectory(zipDirectory);

            _.each(self.handlers, function (handler) {
                if (Object.prototype.hasOwnProperty.call(importData, handler.type)) {
                    // This limitation is here to reduce the complexity of the importer for now
                    return Promise.reject(new errors.UnsupportedMediaTypeError({
                        message: tpl(messages.zipContainsMultipleDataFormats)
                    }));
                }

                const files = self.getFilesFromZip(handler, zipDirectory);

                if (files.length > 0) {
                    ops.push(function () {
                        return handler.loadFile(files, baseDir).then(function (data) {
                            importData[handler.type] = data;
                        });
                    });
                }
            });

            if (ops.length === 0) {
                return Promise.reject(new errors.UnsupportedMediaTypeError({
                    message: tpl(messages.noContentToImport)
                }));
            }

{'fraction_non_alphanumeric': 0.1183650443612493, 'fraction_numerical': 0.003230934919739474, 'mean_word_length': 2.3219761499148213, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5703679', 'n_tokens_mistral': 5510, 'n_tokens_neox': 5331, 'n_words': 1497}