<commit_before>'use strict'

const
	bodyParser = require('body-parser'),
	co = require('co'),
	express = require('express'),
	fs = require('fs-extra'),
	http = require('http'),
	merge = require('deepmerge'),
	path = require('path'),
	requestLogger = require('morgan'),
	wrap = require('co-express')

class Migrations {
	constructor(config, db) {
		this.config = config
		this.moduleConfig = config.migrations
		this.db = db
	}

	listen() {
		this.app = express()
		this.router = express.Router()
		this.paths = ['/seed', '/sync', '/generateSeed', '/generateBackup', '/insertStaticData']
		const {config, moduleConfig} = this
		let instance = this,
			app = this.app,
			router = this.router

		app.use(requestLogger(`[Migrations Module API] :method request to :url; result: :status; completed in: :response-time; :date`))
		app.use(bodyParser.json())


		router.get('/seed', wrap(function* (req, res, next) {
			try {
				res.json({data: yield instance.seed({
					seedFolder: req.query.seedFolder && decodeURIComponent(req.query.seedFolder) || instance.config.migrations.seedfilesFolder,
					seedFile: req.query.seedFile && decodeURIComponent(req.query.seedFile) || instance.config.migrations.defaultSeedfileName
				})})
			} catch (error) {
				req.locals = {error}
				next()
			}
		}))

		router.get('/sync', wrap(function* (req, res, next) {
			try {
				res.json({data: yield instance.sync()})
			} catch (error) {
				req.locals = {error}
				next()
			}
		}))

		router.get('/generateSeed', wrap(function* (req, res, next) {
			try {
				res.json({data: yield instance.generateSeed({
					seedFile: req.query.seedFile && decodeURIComponent(req.query.seedFile) || instance.config.migrations.defaultSeedfileName
				})})
			} catch (error) {
				req.locals = {error}
				next()
			}
		}))

		router.get('/generateBackup', wrap(function* (req, res, next) {
			try {
				res.json({data: yield instance.generateBackup()})
			} catch (error) {
				req.locals = {error}
				next()
			}
		}))

		router.get('/insertStaticData', wrap(function* (req, res, next) {
			try {
				res.json({data: yield instance.insertStaticData()})
			} catch (error) {
				req.locals = {error}
				next()
			}
		}))

		app.use('/', router)
		app.use(this.paths, (req, res) => {
			if (req.locals && req.locals.error) {
				console.log(req.locals.error)
				return res.status(500).end()
			}
			res.status(200).end()
		})

		let migrationsApiServer = http.createServer(app)
		migrationsApiServer.listen(moduleConfig.serverPort, () => {
			console.log(`[Migrations Module API] Server started.`)
			console.log(`[Migrations Module API] Port:`, moduleConfig.serverPort)
			console.log(`[Migrations Module API] Configuration profile:`, config.name)
		})
	}

	getFullTableData() {
		try {
			const instance = this
			return co(function*() {
				let data = {},
					schemas = yield instance.db.sequelize.query('SELECT "t"."table_name" FROM "information_schema"."tables" AS "t" ' +
						'WHERE "t"."table_schema"=\'' + instance.config.db.schema + '\'')
				for (let i = 0; i < schemas[0].length; i++) {
					data[schemas[0][i].table_name] = (yield instance.db.sequelize.query('SELECT * FROM "' +
						instance.config.db.schema + '"."' + schemas[0][i].table_name + '"'))[0]
				}
				return data
			})
		} catch (e) {
			console.log(e)
		}
	}

	getTableLayout(t) {
		try {
			const instance = this
			return co(function*() {
				let data = {},
					schemas = yield instance.db.sequelize.query('SELECT "t"."table_name","c"."column_name" FROM "information_schema"."tables" AS "t" ' +
						'INNER JOIN "information_schema"."columns" AS "c" ON "t"."table_name"="c"."table_name"' +
						'WHERE "t"."table_schema"=\'' + instance.config.db.schema + '\'', {transaction: t})
				for (let i = 0; i < schemas[0].length; i++) {
					if (typeof(data[schemas[0][i].table_name]) === 'undefined') {
						data[schemas[0][i].table_name] = []
					}
					data[schemas[0][i].table_name].push(schemas[0][i].column_name)
				}
				return data
			})
		} catch (e) {
			console.log(e)
		}
	}

	runQueryFromColumnData(instance, tableName, inserts, queryInterface, deleteTableContents, t, dontSetIdSequence) {
		return co(function*() {
			if (inserts.values.length > 0) {
				console.log('Query series starting, rows to execute:', inserts.values.length)
				if (deleteTableContents.indexOf(tableName) !== -1) {
					yield instance.db.sequelize.query('DELETE FROM "' + tableName + '";', {transaction: t})
				}

				let queryTemplate = 'INSERT INTO "' + tableName + '" (',
					query = ''
				for (let i in inserts.columns) {
					queryTemplate += '"' + (queryInterface.escape(inserts.columns[i])).replace(/'/g, '') + '",'
				}
				queryTemplate = queryTemplate.substr(0, queryTemplate.length - 1)
				queryTemplate += ') VALUES '
				query += queryTemplate

				for (let i in inserts.values) {
					let valuesItem = inserts.values[i]
					query += ' ('
					for (let j in valuesItem) {
						query += valuesItem[j] + ','
					}
					query = query.substr(0, query.length - 1)
					query += ')'
					if (parseInt(i, 10) % 100 === 0) {
						query += ';'
						yield instance.db.sequelize.query(query, {transaction: t})
						query = '' + queryTemplate
						continue;
					}
					query += ','
				}
				if (query[query.length - 1] === ',') {
					query = query.substr(0, query.length - 1)
					query += ';'
					yield instance.db.sequelize.query(query, {transaction: t})
				}
				console.log('All queries executed successfully.')

				if (!dontSetIdSequence) {
					yield instance.db.sequelize.query('SELECT setval(\'"' + tableName + '_id_seq"\'::regclass, (SELECT "id" FROM "' + tableName + '" ORDER BY "id" DESC LIMIT 1))', {transaction: t})
				}
			}
		})
	}

	prepareDataObjectForQuery(dataObject, tableLayout, queryInterface) {
		let columns = [],
			values = []
		for (let column in dataObject) {
			if (tableLayout.indexOf(column) !== -1) {
				columns.push(column)
				let columnValue = dataObject[column]
				if ((typeof columnValue === 'object') && (columnValue !== null) && (typeof columnValue.length === 'undefined')) {
					values.push(queryInterface.escape(JSON.stringify(columnValue)))
					continue
				}
				values.push(queryInterface.escape(columnValue))
			}
		}
		return {columns, values}
	}<commit_msg>urgent hotfix for the migrations module<commit_after>'use strict'

const
	bodyParser = require('body-parser'),
	co = require('co'),
	express = require('express'),
	fs = require('fs-extra'),
	http = require('http'),
	merge = require('deepmerge'),
	path = require('path'),
	requestLogger = require('morgan'),
	wrap = require('co-express')

class Migrations {
	constructor(config, db) {
		this.config = config
		this.moduleConfig = config.migrations
		this.db = db
	}

	listen() {
		this.app = express()
		this.router = express.Router()
		this.paths = ['/seed', '/sync', '/generateSeed', '/generateBackup', '/insertStaticData']
		const {config, moduleConfig} = this
		let instance = this,
			app = this.app,
			router = this.router

		app.use(requestLogger(`[Migrations Module API] :method request to :url; result: :status; completed in: :response-time; :date`))
		app.use(bodyParser.json())


		router.get('/seed', wrap(function* (req, res, next) {
			try {
				res.json({data: yield instance.seed({
					seedFolder: req.query.seedFolder && decodeURIComponent(req.query.seedFolder) || instance.config.migrations.seedfilesFolder,
					seedFile: req.query.seedFile && decodeURIComponent(req.query.seedFile) || instance.config.migrations.defaultSeedfileName
				})})
			} catch (error) {
				req.locals = {error}
				next()
			}
		}))

		router.get('/sync', wrap(function* (req, res, next) {
			try {
				res.json({data: yield instance.sync()})
			} catch (error) {
				req.locals = {error}
				next()
			}
		}))

		router.get('/generateSeed', wrap(function* (req, res, next) {
			try {
				res.json({data: yield instance.generateSeed({
					seedFile: req.query.seedFile && decodeURIComponent(req.query.seedFile) || instance.config.migrations.defaultSeedfileName
				})})
			} catch (error) {
				req.locals = {error}
				next()
			}
		}))

		router.get('/generateBackup', wrap(function* (req, res, next) {
			try {
				res.json({data: yield instance.generateBackup()})
			} catch (error) {
				req.locals = {error}
				next()
			}
		}))

		router.get('/insertStaticData', wrap(function* (req, res, next) {
			try {
				res.json({data: yield instance.insertStaticData()})
			} catch (error) {
				req.locals = {error}
				next()
			}
		}))

		app.use('/', router)
		app.use(this.paths, (req, res) => {
			if (req.locals && req.locals.error) {
				console.log(req.locals.error)
				return res.status(500).end()
			}
			res.status(200).end()
		})

		let migrationsApiServer = http.createServer(app)
		migrationsApiServer.listen(moduleConfig.serverPort, () => {
			console.log(`[Migrations Module API] Server started.`)
			console.log(`[Migrations Module API] Port:`, moduleConfig.serverPort)
			console.log(`[Migrations Module API] Configuration profile:`, config.name)
		})
	}

	getFullTableData() {
		try {
			const instance = this
			return co(function*() {
				let data = {},
					schemas = yield instance.db.sequelize.query('SELECT "t"."table_name" FROM "information_schema"."tables" AS "t" ' +
						'WHERE "t"."table_schema"=\'' + instance.config.db.schema + '\'')
				for (let i = 0; i < schemas[0].length; i++) {
					data[schemas[0][i].table_name] = (yield instance.db.sequelize.query('SELECT * FROM "' +
						instance.config.db.schema + '"."' + schemas[0][i].table_name + '"'))[0]
				}
				return data
			})
		} catch (e) {
			console.log(e)
		}
	}

	getTableLayout(t) {
		try {
			const instance = this
			return co(function*() {
				let data = {},
					schemas = yield instance.db.sequelize.query('SELECT "t"."table_name","c"."column_name" FROM "information_schema"."tables" AS "t" ' +
						'INNER JOIN "information_schema"."columns" AS "c" ON "t"."table_name"="c"."table_name"' +
						'WHERE "t"."table_schema"=\'' + instance.config.db.schema + '\'', {transaction: t})
				for (let i = 0; i < schemas[0].length; i++) {
					if (typeof(data[schemas[0][i].table_name]) === 'undefined') {
						data[schemas[0][i].table_name] = []
					}
					data[schemas[0][i].table_name].push(schemas[0][i].column_name)
				}
				return data
			})
		} catch (e) {
			console.log(e)
		}
	}

	runQueryFromColumnData(instance, tableName, inserts, queryInterface, deleteTableContents, t, dontSetIdSequence) {
		return co(function*() {
			if (inserts.values.length > 0) {
				console.log('Query series starting, rows to execute:', inserts.values.length, ' for table ', tableName)
				if (deleteTableContents.indexOf(tableName) !== -1) {
					yield instance.db.sequelize.query('DELETE FROM "' + tableName + '";', {transaction: t})
				}

				let queryTemplate = 'INSERT INTO "' + tableName + '" (',
					query = ''
				for (let i in inserts.columns) {
					queryTemplate += '"' + (queryInterface.escape(inserts.columns[i])).replace(/'/g, '') + '",'
				}
				queryTemplate = queryTemplate.substr(0, queryTemplate.length - 1)
				queryTemplate += ') VALUES '
				query += queryTemplate

				for (let i in inserts.values) {
					let valuesItem = inserts.values[i]
					query += ' ('
					for (let j in valuesItem) {
						query += valuesItem[j] + ','
					}
					query = query.substr(0, query.length - 1)
					query += ')'
					if (parseInt(i, 10) % 100 === 0) {
						query += ';'
						yield instance.db.sequelize.query(query, {transaction: t})
						query = '' + queryTemplate
						continue;
					}
					query += ','
				}
				if (query[query.length - 1] === ',') {
					query = query.substr(0, query.length - 1)
					query += ';'
					yield instance.db.sequelize.query(query, {transaction: t})
				}
				console.log('All queries executed successfully.')

				if (!dontSetIdSequence) {
					yield instance.db.sequelize.query('SELECT setval(\'"' + tableName + '_id_seq"\'::regclass, (SELECT "id" FROM "' + tableName + '" ORDER BY "id" DESC LIMIT 1))', {transaction: t})
				}
			}
		})
	}

	escapeRecursively(queryInterface, value) {
		if (value instanceof Array) {
			let escapedObject = []
			value.forEach((item, index) => escapedObject.push(this.escapeRecursively(queryInterface, item)))
			return escapedObject
		}
		if ((typeof value === 'object') && (value !== null)) {
			let escapedObject = {}
			for (const key in value) {
				escapedObject[key] = this.escapeRecursively(queryInterface, value[key])
			}
			return escapedObject
		}
		return queryInterface.escape(value)
	}

	prepareDataObjectForQuery(dataObject, tableLayout, queryInterface) {
		let columns = [],
			values = []
		for (let column in dataObject) {
			if (tableLayout.indexOf(column) !== -1) {
				columns.push(column)
				let columnValue = dataObject[column]
				if ((typeof columnValue === 'object') && (columnValue !== null)) {
					values.push(queryInterface.escape(JSON.stringify(columnValue)))
					// values.push(JSON.stringify(this.escapeRecursively(queryInterface, columnValue)))
					continue
				}
				values.push(queryInterface.escape(columnValue))
			}
		}
		return {columns, values}
	}
{'fraction_non_alphanumeric': 0.17262901507078704, 'fraction_numerical': 0.005175825848683209, 'mean_word_length': 3.3738348868175767, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5366671', 'n_tokens_mistral': 5296, 'n_tokens_neox': 4343, 'n_words': 877}