<commit_before>const moment = require('moment-timezone');
const path = require('path');
const fs = require('fs-extra');
const membersCSV = require('@tryghost/members-csv');
const errors = require('@tryghost/errors');
const tpl = require('@tryghost/tpl');

const emailTemplate = require('./email-template');

const messages = {
    filenameCollision: 'Filename already exists, please try again.',
    jobAlreadyComplete: 'Job is already complete.'
};

module.exports = class MembersCSVImporter {
    /**
     * @param {Object} options
     * @param {string} options.storagePath - The path to store CSV's in before importing
     * @param {Function} options.getTimezone - function returning currently configured timezone
     * @param {() => Object} options.getMembersApi
     * @param {Function} options.sendEmail - function sending an email
     * @param {(string) => boolean} options.isSet - Method checking if specific feature is enabled
     * @param {({job, offloaded}) => void} options.addJob - Method registering an async job
     * @param {Object} options.knex - An instance of the Ghost Database connection
     * @param {Function} options.urlFor - function generating urls
     * @param {Object} options.context
     */
    constructor({storagePath, getTimezone, getMembersApi, sendEmail, isSet, addJob, knex, urlFor, context}) {
        this._storagePath = storagePath;
        this._getTimezone = getTimezone;
        this._getMembersApi = getMembersApi;
        this._sendEmail = sendEmail;
        this._isSet = isSet;
        this._addJob = addJob;
        this._knex = knex;
        this._urlFor = urlFor;
        this._context = context;
    }

    /**
     * @typedef {string} JobID
     */

    /**
     * @typedef {Object} Job
     * @prop {string} filename
     * @prop {JobID} id
     * @prop {string} status
     */

    /**
     * Get the Job for a jobCode
     * @param {JobID} jobId
     * @returns {Promise<Job>}
     */
    async getJob(jobId) {
        return {
            id: jobId,
            filename: jobId,
            status: 'pending'
        };
    }

    /**
     * Prepares a CSV file for import
     * - Maps headers based on headerMapping, this allows for a non standard CSV
     *   to be imported, so long as a mapping exists between it and a standard CSV
     * - Stores the CSV to be imported in the storagePath
     * - Creates a MemberImport Job and associated MemberImportBatch's
     *
     * @param {string} inputFilePath - The path to the CSV to prepare
     * @param {Object.<string, string>} headerMapping - An object whose keys are headers in the input CSV and values are the header to replace it with
     * @param {Array<string>} defaultLabels - A list of labels to apply to every member
     *
     * @returns {Promise<{id: JobID, batches: number, metadata: Object.<string, any>}>} - A promise resolving to the id of the MemberImport Job
     */
    async prepare(inputFilePath, headerMapping, defaultLabels) {
        // @NOTE: investigate why is it "1" and do we even need this concept anymore?
        const batchSize = 1;

        const siteTimezone = this._getTimezone();
        const currentTime = moment().tz(siteTimezone).format('YYYY-MM-DD HH:mm:ss.SSS');
        const outputFileName = `Members Import ${currentTime}.csv`;
        const outputFilePath = path.join(this._storagePath, '/', outputFileName);

        const pathExists = await fs.pathExists(outputFilePath);

        if (pathExists) {
            throw new errors.DataImportError({message: tpl(messages.filenameCollision)});
        }

        const inputMapping = Object.assign({}, headerMapping, {
            subscribed_to_emails: 'subscribed'
        });
        const rows = await membersCSV.parse(inputFilePath, inputMapping, defaultLabels);
        const columns = Object.keys(rows[0]);
        const numberOfBatches = Math.ceil(rows.length / batchSize);
        const mappedCSV = membersCSV.unparse(rows, columns);

        const hasStripeData = rows.find(function rowHasStripeData(row) {
            return !!row.stripe_customer_id || !!row.complimentary_plan;
        });

        await fs.writeFile(outputFilePath, mappedCSV);

        return {
            id: outputFilePath,
            batches: numberOfBatches,
            metadata: {
                hasStripeData
            }
        };
    }

    /**
     * Performs an import of a CSV file
     *
     * @param {JobID} id - The id of the job to perform
     */
    async perform(id) {
        const job = await this.getJob(id);

        if (job.status === 'complete') {
            throw new errors.BadRequestError({message: tpl(messages.jobAlreadyComplete)});
        }

        const rows = membersCSV.parse(job.filename);

        const membersApi = await this._getMembersApi();

        const defaultProductPage = await membersApi.productRepository.list({
            filter: 'type:paid',
            limit: 1
        });

        const defaultProduct = defaultProductPage.data[0];

        const result = await rows.reduce(async (resultPromise, row) => {<commit_msg>Removed unused concept of "status" in importer job<commit_after>const moment = require('moment-timezone');
const path = require('path');
const fs = require('fs-extra');
const membersCSV = require('@tryghost/members-csv');
const errors = require('@tryghost/errors');
const tpl = require('@tryghost/tpl');

const emailTemplate = require('./email-template');

const messages = {
    filenameCollision: 'Filename already exists, please try again.',
    jobAlreadyComplete: 'Job is already complete.'
};

module.exports = class MembersCSVImporter {
    /**
     * @param {Object} options
     * @param {string} options.storagePath - The path to store CSV's in before importing
     * @param {Function} options.getTimezone - function returning currently configured timezone
     * @param {() => Object} options.getMembersApi
     * @param {Function} options.sendEmail - function sending an email
     * @param {(string) => boolean} options.isSet - Method checking if specific feature is enabled
     * @param {({job, offloaded}) => void} options.addJob - Method registering an async job
     * @param {Object} options.knex - An instance of the Ghost Database connection
     * @param {Function} options.urlFor - function generating urls
     * @param {Object} options.context
     */
    constructor({storagePath, getTimezone, getMembersApi, sendEmail, isSet, addJob, knex, urlFor, context}) {
        this._storagePath = storagePath;
        this._getTimezone = getTimezone;
        this._getMembersApi = getMembersApi;
        this._sendEmail = sendEmail;
        this._isSet = isSet;
        this._addJob = addJob;
        this._knex = knex;
        this._urlFor = urlFor;
        this._context = context;
    }

    /**
     * @typedef {string} JobID
     */

    /**
     * @typedef {Object} Job
     * @prop {string} filename
     * @prop {JobID} id
     */

    /**
     * Get the Job for a jobCode
     * @param {JobID} jobId
     * @returns {Promise<Job>}
     */
    async getJob(jobId) {
        return {
            id: jobId,
            filename: jobId
        };
    }

    /**
     * Prepares a CSV file for import
     * - Maps headers based on headerMapping, this allows for a non standard CSV
     *   to be imported, so long as a mapping exists between it and a standard CSV
     * - Stores the CSV to be imported in the storagePath
     * - Creates a MemberImport Job and associated MemberImportBatch's
     *
     * @param {string} inputFilePath - The path to the CSV to prepare
     * @param {Object.<string, string>} headerMapping - An object whose keys are headers in the input CSV and values are the header to replace it with
     * @param {Array<string>} defaultLabels - A list of labels to apply to every member
     *
     * @returns {Promise<{id: JobID, batches: number, metadata: Object.<string, any>}>} - A promise resolving to the id of the MemberImport Job
     */
    async prepare(inputFilePath, headerMapping, defaultLabels) {
        // @NOTE: investigate why is it "1" and do we even need this concept anymore?
        const batchSize = 1;

        const siteTimezone = this._getTimezone();
        const currentTime = moment().tz(siteTimezone).format('YYYY-MM-DD HH:mm:ss.SSS');
        const outputFileName = `Members Import ${currentTime}.csv`;
        const outputFilePath = path.join(this._storagePath, '/', outputFileName);

        const pathExists = await fs.pathExists(outputFilePath);

        if (pathExists) {
            throw new errors.DataImportError({message: tpl(messages.filenameCollision)});
        }

        const inputMapping = Object.assign({}, headerMapping, {
            subscribed_to_emails: 'subscribed'
        });
        const rows = await membersCSV.parse(inputFilePath, inputMapping, defaultLabels);
        const columns = Object.keys(rows[0]);
        const numberOfBatches = Math.ceil(rows.length / batchSize);
        const mappedCSV = membersCSV.unparse(rows, columns);

        const hasStripeData = rows.find(function rowHasStripeData(row) {
            return !!row.stripe_customer_id || !!row.complimentary_plan;
        });

        await fs.writeFile(outputFilePath, mappedCSV);

        return {
            id: outputFilePath,
            batches: numberOfBatches,
            metadata: {
                hasStripeData
            }
        };
    }

    /**
     * Performs an import of a CSV file
     *
     * @param {JobID} id - The id of the job to perform
     */
    async perform(id) {
        const job = await this.getJob(id);

        const rows = membersCSV.parse(job.filename);

        const membersApi = await this._getMembersApi();

        const defaultProductPage = await membersApi.productRepository.list({
            filter: 'type:paid',
            limit: 1
        });

        const defaultProduct = defaultProductPage.data[0];

        const result = await rows.reduce(async (resultPromise, row) => {
{'fraction_non_alphanumeric': 0.10503106835037082, 'fraction_numerical': 0.0010022048506714773, 'mean_word_length': 2.596036036036036, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '352309', 'n_tokens_mistral': 2792, 'n_tokens_neox': 2697, 'n_words': 845}