<commit_before>/*
Passport-Live is a modern web app for schools that helps them manage passes.
    Copyright (C) 2017  <NAME>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

email: <EMAIL>
*/

/**
* A set of Apis for importing large amounts of data into passport
* @module js/import
*/
const convertExcel = require("excel-as-json").processFile;
const accountAPI = require("./accounts.js");
var r = require("rethinkdb");
var db = require("../../modules/db/index.js");
var r_ = db.dash();
var utils = require("../passport-utils/index.js");
var typeCheck = require("type-check").typeCheck;
var moment = require("moment");
var emailJS = require("./email.js");

/**
* Creates a bulk import job log 
* @function newBulkLog
* @link module:js/import
* @param {String} name - A Non unique human readable ID for the job.
* @param {String} importType - The type of import it is.  includes "account" and "schedule"
* @returns {Promise}
*/
function newBulkLog(name, importType) {
    //return new Promise((resolve, reject) => {
    return r.table("bulkImports").insert({name: name, date: r.now(), importType: importType}).run(db.conn());
    //})
}
/**
* Updates the bulk log with new info about the import
* @function updateBulkLog
* @link module:js/import
* @param {String} id - Bulk Import ID
* @param {Number} totalTried - Total number of attempted document imports
* @param {Number} totalImported - The total number of documents successfully imported
* @param {Object[]} loggedErrors - Any error logged when processing the documents
* @param {Object} properties - any special properties that each importType might want to include.
* @param {(Number|undefined)} properties.totalInitialized - (For importType "account") Total number of accounts imported with passwords set.
* @returns {Promise}
*/
function updateBulkLog(id, totalTried, totalImported, loggedErrors, properties) { 
    return r.table("bulkImports").get(id).update({totalTried: totalTried, totalImported: totalImported, loggedErrors: loggedErrors, properties: properties}).run(db.conn());
}
/**
* Deletes the bulk log
* @function deleteBulkLog
* @link module:js/import
* @param {String} id - Bulk Import ID
* @returns {Promise}
*/
function deleteBulkLog(id) { 
    return r.table("bulkImports").get(id).delete().run(db.conn());
}

/*
* Searches the bulk log database 
* @link module:js/import
* @param {Object} queries
* @param {(String|undefined)} queries.name - Bulk Log Name
* @param {(String|undefined)} queries.type - importType. Current values: "account", "schedule" 
* @param {(Object|undefined)} queries.date
* @param {(String|Date|undefined)} queries.date.from - ISO Strng or date Low end.  inclusive
* @param {(String|Date|undefined)} queries.date.to - ISO Strng High end. inclusive
* @returns {Promise} - array
*/
exports.searchBulkLogs = (queries) => {
    return new Promise((resolve, reject) => {
        if(!typeCheck("{name: Maybe String, date: Maybe {from: Maybe ISODate | Date, to: Maybe ISODate | Date}, type: Maybe String}", queries, utils.typeCheck)) {
            var err = new TypeError("queries must be an object with format: \"{name: Maybe String, date: Maybe {from: Maybe ISODate | Date, to: Maybe ISODate | Date}, type: Maybe String}\"");
            err.status = 400;
            return reject(err);
        }
        //            date: r.ISO8601(queries.date),
        if(queries.date && typeCheck("Date", queries.date.from)) {
            queries.date.from = moment(queries.date.from).toISOString();
        }
        if(queries.date && typeCheck("Date", queries.date.to)) {
            queries.date.to = moment(queries.date.to).toISOString();
        }
        return r_.table("bulkImports")
            .filter((date) => {
                if(queries.date && queries.date.from && queries.date.to) {
                    return date("date").during(r_.ISO8601(queries.date.from), r_.ISO8601(queries.date.to), {leftBound: "closed", rightBound: "closed"});
                } else if(queries.date && queries.date.from) {
                    return date("date").ge(r_.ISO8601(queries.date.from));
                } else if(queries.date && queries.date.to) {
                    return date("date").le(r_.ISO8601(queries.date.to));
                } else {
                    return true;
                }
            })
            .filter((row) => {
                if(queries.name != undefined) {
                    return row("name").match("(?i)"+queries.name.replace(/\s/g,""));
                } else {
                    return true;
                }
            })
            .filter((row) => {
                if(queries.type != undefined) {
                    return row("importType").eq(queries.type);
                } else {
                    return true;
                }
            })
            .run().then(resolve).catch(reject);
    });
};



/**
 * Functions for manupulating imported accunts. Assumes account importType
 * @name accounts
 * @inner
 * @private
 * @memberof js/import
 * @property {Object} accounts
 * @property {function} accounts.rollback - Deletes all accounts linked to a given import job
 */
var accounts = {};
/**
 * Deletes all accounts linked to a given import job ID
 * @function
 * @memberof js/import
 * @param {String} bulkID - Id of the bulk import.
 * @param {Boolean} ignoreActivated - If true, activated accounts will be skipped.
 * @returns {Promise} - Transaction Statement  
 */
accounts.rollback = (bulkID, ignoreVerified) => {
    return new Promise((resolve, reject) => {
        return r_.table("accounts").filter((row) => {
            return row("flags")("bulkImportID").eq(bulkID);
        }).filter((row) => {
            if(ignoreVerified) {
                return row("isVerified").eq(false);
            } else {
                return true;
            }
        }).delete().run().then((trans) => {
            if(trans.deleted > 0) {
                r_.table("bulkImports").get(bulkID).update({rollback: {on: r_.now(), deleted: trans.deleted}}).run().then(() => {
                    return resolve(trans);
                }).catch((err)=>{return reject(err);});
            } else {
                return resolve(trans);
            }
        }).catch(reject);
    });
};

//accounts.rollback("8e4a8ff9-5503-4e6f-920c-7b07f1109601", true).then((res)=>{console.log(res)}).catch((err)=>{console.error(err)});

/**
 * Sends athe activation email to all unverified users.  
 * @function
 * @memberof js/import
 * @param {String} bulkID - Id of the bulk import.
 * @returns {Promise} - accounts: Accounts, cursor: sendActivationEmail Cursor
 */
accounts.sendActivation = (bulkID) => {
    return new Promise((resolve, reject) => {
        return r_.table("accounts")
            .filter((row) => {
                return row("flags")("bulkImportID").eq(bulkID);
            })
            .filter((row) => {
                return row("isVerified").eq(false);
            })
            .withFields("id", "email", {name: "first"})
            .map((doc) => {
                return doc.merge({accountID: doc("id")}).without("id");
            })
            .map((doc) => {
                return doc.merge({to: doc("email")}).without("email");
            })
            .run().then((accounts) => {
                if(accounts.length < 1) {
                    let err = new Error("No accounts linked to that bulkID");
                    err.status = 404;
                    return reject(err);
                } else {
                    emailJS.sendActivationEmail(accounts).then((cur) => {
                        return resolve({accounts: accounts, cursor: cur});
                    }).catch(reject);
                }
            
            }).catch(reject);
    });
};
/*
setTimeout(() => {
    accounts.sendActivation("61b8ea46-c329-4b2f-b0c7-88be7718cd4c").then((res) => {
        console.log(res)
    }).catch((err) => {
        console.error(err)
    })
})*/


/** 
* Takes in an array of account json objects and imports them
* NOTE: Email domains are still must follow userGroup settings.
* If the json object lacks the nessessary values to create an account, the row is skipped 
* @link module:js/import
* @param {accountImport[]} accounts - The accountImport objects 
* @param {String} importName - a (non unique) name for this import job
* @returns {Promise} - Array of objects with key "account" containing the user imported, and key "error" with an error that occured during import for that user 
*/

accounts.json = (accounts, importName) => {
    console.log(accounts);
    return new Promise((resolve, reject) => {
        let typeDef = `[{
            name: {
                first: String, 
                last: String,
                salutation: String
            },
            schoolID: Maybe String,
            email: String,
            userGroup: userGroup,
            isVerified: Maybe Boolean,
            password: Maybe String,
            graduationYear: Maybe Number
        }]`;
        //check type
        if(!typeCheck(typeDef, accounts, utils.typeCheck) || accounts.length < 1) {
            let err = new TypeError("\"accounts\" expected an array with structure: " + typeDef);
            err.status = 400;
            return reject(err);
        }
        if(typeof importName !== "string") {
            let err = new TypeError("\"importName\" expected a string");
            err.status = 400;
            return reject(err);
        }
        //declare 
        let loopPromice = [];
        let errors = [];
        let imported = 0;
        let initialized = 0;
        let defaults = {
            schoolID: null,
            isVerified: false,
            password: null,
            graduationYear: null
        };
        //loop through accounts
        newBulkLog(importName, "account").then((jResp) => {
            if(jResp.inserted == 1) {
                for(let x = 0; x < accounts.length; x++) {
                    loopPromice.push(new Promise((lRes, lRej) => {
                        //set default values
                        let account = Object.assign(defaults, accounts[x]);
                        
                        //Import each account
                        accountAPI.createAccount({
                            userGroup: account.userGroup, 
                            name: account.name, 
                            email: account.email, 
                            password: <PASSWORD>, 
                            schoolID: account.schoolID, 
                            graduationYear: account.graduationYear, 
                            flags: {
                                bulkImportID: jResp.generated_keys[0]
                            }
                        }, {
                            skipEmail: true,
                            allowNullPassword: true,
                            generatePassword: false
                        }).then(function(transSummery) {
                            //success 
                            if(transSummery.transaction.inserted > 0) {
                                imported += transSummery.transaction.inserted;
                                if(account.password) {initialized++;}
                            }
                            console.log(account, transSummery);
                            lRes({account: account, error: null, transaction: transSummery.transaction});
                        }).catch((err) => {
                            if(err.status == 500) {
                                return lRej(err); 
                            } else {
                                errors.push({status: err.status, message: err.message});
                                //failed, but continue import
                                return lRes({account: account, error: {status: err.status, message: err.message}});
                            }
                        });
                    }));
                }

                // wrap up transaction 
                Promise.all(loopPromice).then((sumArray) => {
                    let finalLog = [];
                    let bulkID = jResp.generated_keys[0];
                    if(imported == 0) {
                        //deletes log if theere was nothing that could be imported
                        finalLog.push(deleteBulkLog(bulkID));
                        bulkID = null;
                    } else {
                        //updates the log with more job info
                        finalLog.push(updateBulkLog(bulkID, accounts.length, imported, errors, {totalInitialized: initialized}));
                    }
                    Promise.all(finalLog).then(() => {
                        resolve({bulkID: bulkID, summary: sumArray, totalTried: accounts.length, totalImported: imported, totalInitialized: initialized});
                    }).catch((err) => {
                        return reject(err);
                    });
                });
            } else {
                let err = new Error("Failed to log import job");
                err.status = 500;
                return reject(err);
            }
        });


        <commit_msg>Trying to kill the import loop promise forgetting the response value.<commit_after>/*
Passport-Live is a modern web app for schools that helps them manage passes.
    Copyright (C) 2017  <NAME>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

email: <EMAIL>
*/

/**
* A set of Apis for importing large amounts of data into passport
* @module js/import
*/
const convertExcel = require("excel-as-json").processFile;
const accountAPI = require("./accounts.js");
var r = require("rethinkdb");
var db = require("../../modules/db/index.js");
var r_ = db.dash();
var utils = require("../passport-utils/index.js");
var typeCheck = require("type-check").typeCheck;
var moment = require("moment");
var emailJS = require("./email.js");

/**
* Creates a bulk import job log 
* @function newBulkLog
* @link module:js/import
* @param {String} name - A Non unique human readable ID for the job.
* @param {String} importType - The type of import it is.  includes "account" and "schedule"
* @returns {Promise}
*/
function newBulkLog(name, importType) {
    //return new Promise((resolve, reject) => {
    return r.table("bulkImports").insert({name: name, date: r.now(), importType: importType}).run(db.conn());
    //})
}
/**
* Updates the bulk log with new info about the import
* @function updateBulkLog
* @link module:js/import
* @param {String} id - Bulk Import ID
* @param {Number} totalTried - Total number of attempted document imports
* @param {Number} totalImported - The total number of documents successfully imported
* @param {Object[]} loggedErrors - Any error logged when processing the documents
* @param {Object} properties - any special properties that each importType might want to include.
* @param {(Number|undefined)} properties.totalInitialized - (For importType "account") Total number of accounts imported with passwords set.
* @returns {Promise}
*/
function updateBulkLog(id, totalTried, totalImported, loggedErrors, properties) { 
    return r.table("bulkImports").get(id).update({totalTried: totalTried, totalImported: totalImported, loggedErrors: loggedErrors, properties: properties}).run(db.conn());
}
/**
* Deletes the bulk log
* @function deleteBulkLog
* @link module:js/import
* @param {String} id - Bulk Import ID
* @returns {Promise}
*/
function deleteBulkLog(id) { 
    return r.table("bulkImports").get(id).delete().run(db.conn());
}

/*
* Searches the bulk log database 
* @link module:js/import
* @param {Object} queries
* @param {(String|undefined)} queries.name - Bulk Log Name
* @param {(String|undefined)} queries.type - importType. Current values: "account", "schedule" 
* @param {(Object|undefined)} queries.date
* @param {(String|Date|undefined)} queries.date.from - ISO Strng or date Low end.  inclusive
* @param {(String|Date|undefined)} queries.date.to - ISO Strng High end. inclusive
* @returns {Promise} - array
*/
exports.searchBulkLogs = (queries) => {
    return new Promise((resolve, reject) => {
        if(!typeCheck("{name: Maybe String, date: Maybe {from: Maybe ISODate | Date, to: Maybe ISODate | Date}, type: Maybe String}", queries, utils.typeCheck)) {
            var err = new TypeError("queries must be an object with format: \"{name: Maybe String, date: Maybe {from: Maybe ISODate | Date, to: Maybe ISODate | Date}, type: Maybe String}\"");
            err.status = 400;
            return reject(err);
        }
        //            date: r.ISO8601(queries.date),
        if(queries.date && typeCheck("Date", queries.date.from)) {
            queries.date.from = moment(queries.date.from).toISOString();
        }
        if(queries.date && typeCheck("Date", queries.date.to)) {
            queries.date.to = moment(queries.date.to).toISOString();
        }
        return r_.table("bulkImports")
            .filter((date) => {
                if(queries.date && queries.date.from && queries.date.to) {
                    return date("date").during(r_.ISO8601(queries.date.from), r_.ISO8601(queries.date.to), {leftBound: "closed", rightBound: "closed"});
                } else if(queries.date && queries.date.from) {
                    return date("date").ge(r_.ISO8601(queries.date.from));
                } else if(queries.date && queries.date.to) {
                    return date("date").le(r_.ISO8601(queries.date.to));
                } else {
                    return true;
                }
            })
            .filter((row) => {
                if(queries.name != undefined) {
                    return row("name").match("(?i)"+queries.name.replace(/\s/g,""));
                } else {
                    return true;
                }
            })
            .filter((row) => {
                if(queries.type != undefined) {
                    return row("importType").eq(queries.type);
                } else {
                    return true;
                }
            })
            .run().then(resolve).catch(reject);
    });
};



/**
 * Functions for manupulating imported accunts. Assumes account importType
 * @name accounts
 * @inner
 * @private
 * @memberof js/import
 * @property {Object} accounts
 * @property {function} accounts.rollback - Deletes all accounts linked to a given import job
 */
var accounts = {};
/**
 * Deletes all accounts linked to a given import job ID
 * @function
 * @memberof js/import
 * @param {String} bulkID - Id of the bulk import.
 * @param {Boolean} ignoreActivated - If true, activated accounts will be skipped.
 * @returns {Promise} - Transaction Statement  
 */
accounts.rollback = (bulkID, ignoreVerified) => {
    return new Promise((resolve, reject) => {
        return r_.table("accounts").filter((row) => {
            return row("flags")("bulkImportID").eq(bulkID);
        }).filter((row) => {
            if(ignoreVerified) {
                return row("isVerified").eq(false);
            } else {
                return true;
            }
        }).delete().run().then((trans) => {
            if(trans.deleted > 0) {
                r_.table("bulkImports").get(bulkID).update({rollback: {on: r_.now(), deleted: trans.deleted}}).run().then(() => {
                    return resolve(trans);
                }).catch((err)=>{return reject(err);});
            } else {
                return resolve(trans);
            }
        }).catch(reject);
    });
};

//accounts.rollback("8e4a8ff9-5503-4e6f-920c-7b07f1109601", true).then((res)=>{console.log(res)}).catch((err)=>{console.error(err)});

/**
 * Sends athe activation email to all unverified users.  
 * @function
 * @memberof js/import
 * @param {String} bulkID - Id of the bulk import.
 * @returns {Promise} - accounts: Accounts, cursor: sendActivationEmail Cursor
 */
accounts.sendActivation = (bulkID) => {
    return new Promise((resolve, reject) => {
        return r_.table("accounts")
            .filter((row) => {
                return row("flags")("bulkImportID").eq(bulkID);
            })
            .filter((row) => {
                return row("isVerified").eq(false);
            })
            .withFields("id", "email", {name: "first"})
            .map((doc) => {
                return doc.merge({accountID: doc("id")}).without("id");
            })
            .map((doc) => {
                return doc.merge({to: doc("email")}).without("email");
            })
            .run().then((accounts) => {
                if(accounts.length < 1) {
                    let err = new Error("No accounts linked to that bulkID");
                    err.status = 404;
                    return reject(err);
                } else {
                    emailJS.sendActivationEmail(accounts).then((cur) => {
                        return resolve({accounts: accounts, cursor: cur});
                    }).catch(reject);
                }
            
            }).catch(reject);
    });
};
/*
setTimeout(() => {
    accounts.sendActivation("61b8ea46-c329-4b2f-b0c7-88be7718cd4c").then((res) => {
        console.log(res)
    }).catch((err) => {
        console.error(err)
    })
})*/


/** 
* Takes in an array of account json objects and imports them
* NOTE: Email domains are still must follow userGroup settings.
* If the json object lacks the nessessary values to create an account, the row is skipped 
* @link module:js/import
* @param {accountImport[]} accounts - The accountImport objects 
* @param {String} importName - a (non unique) name for this import job
* @returns {Promise} - Array of objects with key "account" containing the user imported, and key "error" with an error that occured during import for that user 
*/

accounts.json = (accounts, importName) => {
    console.log(accounts);
    return new Promise((resolve, reject) => {
        let typeDef = `[{
            name: {
                first: String, 
                last: String,
                salutation: String
            },
            schoolID: Maybe String,
            email: String,
            userGroup: userGroup,
            isVerified: Maybe Boolean,
            password: Maybe String,
            graduationYear: Maybe Number
        }]`;
        //check type
        if(!typeCheck(typeDef, accounts, utils.typeCheck) || accounts.length < 1) {
            let err = new TypeError("\"accounts\" expected an array with structure: " + typeDef);
            err.status = 400;
            return reject(err);
        }
        if(typeof importName !== "string") {
            let err = new TypeError("\"importName\" expected a string");
            err.status = 400;
            return reject(err);
        }
        //declare 
        var loopPromice = [];
        var errors = [];
        var imported = 0;
        var initialized = 0;
        var defaults = {
            schoolID: null,
            isVerified: false,
            password: <PASSWORD>,
            graduationYear: null
        };
        //loop through accounts
        newBulkLog(importName, "account").then((jResp) => {
            if(jResp.inserted == 1) {
                for(let x = 0; x < accounts.length; x++) {
                    loopPromice.push(new Promise((lRes, lRej) => {
                        //set default values
                        let account = Object.assign(defaults, accounts[x]);
                        //Import each account
                        accountAPI.createAccount({
                            userGroup: account.userGroup, 
                            name: account.name, 
                            email: account.email, 
                            password: <PASSWORD>, 
                            schoolID: account.schoolID, 
                            graduationYear: account.graduationYear, 
                            flags: {
                                bulkImportID: jResp.generated_keys[0]
                            }
                        }, {
                            skipEmail: true,
                            allowNullPassword: true,
                            generatePassword: false
                        }).then((transSummery) => {
                            //success 
                            if(transSummery.transaction.inserted > 0) {
                                imported += transSummery.transaction.inserted;
                                if(account.password) {initialized++;}
                            }
                            //console.log(accountR,Object.assign(defaults, accountR), accounts[x], account, transSummery);
                            let resp = {account: account, error: null, transaction: transSummery.transaction, rand: Math.random()};
                            console.log(resp)
                            return lRes(resp);
                        }).catch((err) => {
                            if(err.status == 500) {
                                return lRej(err); 
                            } else {
                                errors.push({status: err.status, message: err.message});
                                //failed, but continue import
                                return lRes({account: account, error: {status: err.status, message: err.message}});
                            }
                        });
                    }));
                }

                // wrap up transaction 
                Promise.all(loopPromice).then((sumArray) => {
                    //console.log(sumArray)
                    let finalLog = [];
                    let bulkID = jResp.generated_keys[0];
                    if(imported == 0) {
                        //deletes log if theere was nothing that could be imported
                        finalLog.push(deleteBulkLog(bulkID));
                        bulkID = null;
                    } else {
                        //updates the log with more job info
                        finalLog.push(updateBulkLog(bulkID, accounts.length, imported, errors, {totalInitialized: initialized}));
                    }
                    Promise.all(finalLog).then(() => {
                        resolve({bulkID: bulkID, summary: sumArray, totalTried: accounts.length, totalImported: imported, totalInitialized: initialized});
                    }).catch((err) => {
                        return reject(err);
                    });
                });
            } else {
                let err = new Error("Failed to log import job");
                err.status = 500;
                return reject(err);
            }
        });


        
{'fraction_non_alphanumeric': 0.12416445051390786, 'fraction_numerical': 0.006900021562567383, 'mean_word_length': 1.81137603556274, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1982295', 'n_tokens_mistral': 7559, 'n_tokens_neox': 7185, 'n_words': 1997}