<commit_before>'use strict'

var fs 			= require('fs');
var util       	= require('util');
var oracledb    = require('oracledb');
var xlsx       	= require('xlsx');
var async      	= require('async');
var ini 		= require('ini');
var __ 			= require('lodash');

global.argv = require('optimist').argv;

var config = ini.parse(fs.readFileSync(global.argv.cfg, 'utf-8'));
if (!config.oracle) {
	console.warn('no configuration!');
	process.exit(0);
}
var schema = global.argv.schema;
var SheetNames = [];
var tables = {};

async.series([
	function(callback) {
		tables = loadExcel2Json(config.xlsx.fileDir + '/' + schema + '.xlsx', '');
		callback();
	},
	function(callback) {
		loadJson2DB(tables, callback);
	}
], function(err){
	if (err) console.error(err.message);
	process.exit(0);
});

function loadJson2DB(tables, callback) {
	oracledb.getConnection({
		user : config.oracle.user,
		password : <PASSWORD>,
		connectString : config.oracle.connectString
	}, function (err, connection){
		if (err) { 
			console.warn(err.message); 
			cb(err);
			return; 
		}

		var insertQry = [];
		SheetNames.forEach(function(sheet){
			insertQry.push(util.format("TRUNCATE TABLE %s.%s", schema, sheet));
			insertQry = insertQry.concat(getInsertQry(schema, sheet, tables[sheet]));
		});
		
		async.eachSeries(insertQry, function(qry, cbk){
			connection.execute(qry, {}, { autoCommit: false }, cbk);
		}, function(err){
			if (err){
				console.error(err.message);
				connection.release(callback);
				return;
			}
			
			connection.commit(function(err){
				if (err){
					console.error(err.message);
				}else {
					console.log('commit!');
				}
				connection.release(callback);
			});
		});
	});
}

function loadExcel2Json(filePath, ignores) {
	try {
		var XL = require('../bin/xlsx_utils');
		var iFile = xlsx.readFile(filePath);
		SheetNames = iFile.SheetNames;
		var tables = {};
		var zLen = iFile.SheetNames.length;
		for(var z=0; z<zLen; z++) {
			var name = iFile.SheetNames[z];
			if (name[0] === '#')
                continue;
            var sheet = iFile.Sheets[name];
            var dataKey = {};

            try {
                var csv = XL.utils.sheet_to_csv(sheet).split('\n');
                var obj = XL.utils.sheet_to_row_object_array(sheet);
                if (!csv[2] || !csv[1])
                    continue;
                var iName = splitData(csv[3], false);<commit_msg>accept multi schemas<commit_after>'use strict'

var fs 			= require('fs');
var util       	= require('util');
var oracledb    = require('oracledb');
var xlsx       	= require('xlsx');
var async      	= require('async');
var ini 		= require('ini');
var __ 			= require('lodash');

global.argv = require('optimist').argv;

var config = ini.parse(fs.readFileSync(global.argv.cfg, 'utf-8'));
if (!config.oracle) {
	console.warn('no configuration!');
	process.exit(0);
}

if (!global.argv.schemas) {
	console.warn('no schemas!');
	process.exit(0);	
}

var schemas = global.argv.schemas.split(',');
var SheetNames = [];
// var tables = {};

async.eachSeries(schemas, function(schema, callback){
	var tables = loadExcel2Json(config.xlsx.fileDir + '/' + schema + '.xlsx', '');
	loadJson2DB(tables, schema, callback);
}, function(err){
	if (err) console.error(err.message);
	process.exit(0);
});

function loadJson2DB(tables, schema, callback) {
	if (!tables) {
		callback(new Error(schema + ': no such file or directory'));
		return;
	}
	oracledb.getConnection({
		user : config.oracle.user,
		password : <PASSWORD>,
		connectString : config.oracle.connectString
	}, function (err, connection){
		if (err) { 
			console.warn(err.message); 
			cb(err);
			return; 
		}

		var insertQry = [];
		SheetNames.forEach(function(sheet){
			insertQry.push(util.format("TRUNCATE TABLE %s.%s", schema, sheet));
			insertQry = insertQry.concat(getInsertQry(schema, sheet, tables[sheet]));
		});

		async.eachSeries(insertQry, function(qry, cbk){
			connection.execute(qry, {}, { autoCommit: false }, cbk);
		}, function(err){
			if (err){
				console.error(err.message);
				connection.release(callback);
				return;
			}
			
			connection.commit(function(err){
				if (err){
					console.error(err.message);
				}else {
					console.log(schema + ': commit!');
				}
				connection.release(callback);
			});
		});
	});
}

function loadExcel2Json(filePath, ignores) {
	try {
		if (!fs.existsSync(filePath)){
			return null;
		}
		var XL = require('../bin/xlsx_utils');
		var iFile = xlsx.readFile(filePath);
		SheetNames = iFile.SheetNames;
		var tables = {};
		var zLen = iFile.SheetNames.length;
		for(var z=0; z<zLen; z++) {
			var name = iFile.SheetNames[z];
			if (name[0] === '#')
                continue;
            var sheet = iFile.Sheets[name];
            var dataKey = {};

            try {
                var csv = XL.utils.sheet_to_csv(sheet).split('\n');
                var obj = XL.utils.sheet_to_row_object_array(sheet);
                if (!csv[2] || !csv[1])
                    continue;
                var iName = splitData(csv[3], false);
{'fraction_non_alphanumeric': 0.16984126984126985, 'fraction_numerical': 0.00496031746031746, 'mean_word_length': 3.1017087062652564, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2186684', 'n_tokens_mistral': 1931, 'n_tokens_neox': 1714, 'n_words': 327}