<commit_before>var statusCodes = require('../statusCodes');
var restError = require('./errors');
var log = require('czagenda-log').from(__filename);
var shuntingYard = require('../shuntingYard');

exports.searchFields = {};
exports.fulltextFields = null;

exports.search = function(req, res) {

	this._getQueryFromRequest(req, function(err, query) {

				if (err !== null) {

					if (err instanceof restError.BadRequest) {
						res.statusCode = statusCodes.BAD_REQUEST;
						res.end(err.message)
					} else {
						res.statusCode = statusCodes.INTERNAL_ERROR;
						res.end('Internal error')
					}

				} else {
					this._search(req, res, this._setPermsOnQuery(req, query));
				}
			}.bind(this));
}

exports.unescape = function(str) {

	var re = /\\([\+\-\&\|\!\(\)\{\}\[\]\^\"\~\*\?\:\\])/

	return str.replace(re, '\1')

}

exports.getSearchQuery = function(req) {

	var qString = null;
	var rawData = null;

	if (req.method === 'POST') {
		rawData = req.body;
	} else {
		rawData = req.query;
	}

	if (typeof(rawData.q) !== 'undefined') {
		qString = rawData.q.trim();
		return shuntingYard.parseSearchString(qString);
	}

	return {};

	/*
	var q = {};

	if (typeof(rawData.q) !== 'undefined') {
		qString = rawData.q.trim();

		// parse query : text field:.. field:...

		var re = new RegExp(/[a-z0-9\.]+:/gi);
		var fields = [];

		if (re.test(qString) === true) {
			qString.match(re).forEach(function(v) {
						fields.push(v.substr(0, v.length - 1))
					});
			var qArray = qString.split(re);

			// first element is fulltext query
			if (qString.search(re) !== 0) {
				q.fulltext = qArray[0].trim();
				qArray = qArray.splice(1);
			} else {
				qArray = qArray.splice(1);
			}

			for (var i = 0, l = qArray.length; i < l; i++) {
				if (fields.length !== qArray.length) {
					log.warning('getSearchQuery error ' + qString, JSON
									.stringify(fields), JSON.stringify(qArray))
					req.res.statusCode = statusCodes.BAD_REQUEST;
					req.res.end(qString);
					return;
				}

				if (typeof(q[fields[i]]) === 'undefined') {
					q[fields[i]] = [];
				}

				q[fields[i]].push(qArray[i].trim());
			}

		} else {
			q.fulltext = qString;
		}

	}

	return q;*/

}

exports.getDatetimeSearchPart = function(field, args) {

	var q = null;
	if (args.length > 1) {
		q = {
			"or" : []
		}
		args.forEach(function(arg) {
					q.or.push(this.getDatetimeSearchPart(field, [arg]));
				}.bind(this))

	} else {
		var dateRe = '[0-9]{4}-[0-9]{2}-[0-9]{2}';
		var dateReExact = new RegExp('^' + dateRe + '$');
		var dateReRange = /^\[[ ]*([^ ]+)[ ]+TO[ ]+([^ ]+)[ ]*\]$/g
		var range = null;

		if ((range = dateReRange.exec(args[0])) !== null) {

			var from = range[1];
			var to = range[2];

			if (to == 'NOW') {
				to = (new Date()).toISOString();
			}

			if (from == 'NOW') {
				from = (new Date()).toISOString();
			}
			q = {
				range : {}
			};

			if (to == '*') {
				q.range[field] = {
					"gte" : from.replace(
							/([0-9]{4}-[0-9]{2}-[0-9]{2}(?!T[^ ]+))/g,
							'$1T00:00:00.000Z')
				}
			} else if (from == '*') {
				q.range[field] = {
					"lte" : to.replace(
							/([0-9]{4}-[0-9]{2}-[0-9]{2}(?!T[^ ]+))/g,
							'$1T23:59:59.999Z')
				}
			} else {
				q.range[field] = {
					"include_lower" : true,
					"include_upper" : true,
					"from" : from.replace(
							/([0-9]{4}-[0-9]{2}-[0-9]{2}(?!T[^ ]+))/g,
							'$1T00:00:00.000Z'),
					"to" : to.replace(
							/([0-9]{4}-[0-9]{2}-[0-9]{2}(?!T[^ ]+))/g,
							'$1T23:59:59.999Z')
				}
			}

		} else if (dateReExact.test(args[0])) {
			q = {
				term : {}
			};
			q.term[field] = args[0];
		} else {
			throw new restError.BadRequest(field + ':' + args[0])
		}

	}

	return q;

}

exports.getTermSearchPart = function(field, args) {

	var q = null;
	if (args.length > 1) {
		q = {
			terms : {}
		};
		q.terms[field] = args;
	} else {
		q = {
			term : {}
		};
		q.term[field] = args[0];
	}

	return q;

}

exports.getIdsSearchPart = function(field, args) {

	var q = null;
<commit_msg>dateTimeSearch: transform exact date search to date range that represent the day from 00h00 to 23h59<commit_after>var statusCodes = require('../statusCodes');
var restError = require('./errors');
var log = require('czagenda-log').from(__filename);
var shuntingYard = require('../shuntingYard');

exports.searchFields = {};
exports.fulltextFields = null;

exports.search = function(req, res) {

	this._getQueryFromRequest(req, function(err, query) {

				if (err !== null) {

					if (err instanceof restError.BadRequest) {
						res.statusCode = statusCodes.BAD_REQUEST;
						res.end(err.message)
					} else {
						res.statusCode = statusCodes.INTERNAL_ERROR;
						res.end('Internal error')
					}

				} else {
					this._search(req, res, this._setPermsOnQuery(req, query));
				}
			}.bind(this));
}

exports.unescape = function(str) {

	var re = /\\([\+\-\&\|\!\(\)\{\}\[\]\^\"\~\*\?\:\\])/

	return str.replace(re, '\1')

}

exports.getSearchQuery = function(req) {

	var qString = null;
	var rawData = null;

	if (req.method === 'POST') {
		rawData = req.body;
	} else {
		rawData = req.query;
	}

	if (typeof(rawData.q) !== 'undefined') {
		qString = rawData.q.trim();
		return shuntingYard.parseSearchString(qString);
	}

	return {};

	/*
	var q = {};

	if (typeof(rawData.q) !== 'undefined') {
		qString = rawData.q.trim();

		// parse query : text field:.. field:...

		var re = new RegExp(/[a-z0-9\.]+:/gi);
		var fields = [];

		if (re.test(qString) === true) {
			qString.match(re).forEach(function(v) {
						fields.push(v.substr(0, v.length - 1))
					});
			var qArray = qString.split(re);

			// first element is fulltext query
			if (qString.search(re) !== 0) {
				q.fulltext = qArray[0].trim();
				qArray = qArray.splice(1);
			} else {
				qArray = qArray.splice(1);
			}

			for (var i = 0, l = qArray.length; i < l; i++) {
				if (fields.length !== qArray.length) {
					log.warning('getSearchQuery error ' + qString, JSON
									.stringify(fields), JSON.stringify(qArray))
					req.res.statusCode = statusCodes.BAD_REQUEST;
					req.res.end(qString);
					return;
				}

				if (typeof(q[fields[i]]) === 'undefined') {
					q[fields[i]] = [];
				}

				q[fields[i]].push(qArray[i].trim());
			}

		} else {
			q.fulltext = qString;
		}

	}

	return q;*/

}

exports.getDatetimeSearchPart = function(field, args) {

	var q = null;
	if (args.length > 1) {
		q = {
			"or" : []
		}
		args.forEach(function(arg) {
					q.or.push(this.getDatetimeSearchPart(field, [arg]));
				}.bind(this))

	} else {
		var dateRe = '[0-9]{4}-[0-9]{2}-[0-9]{2}';
		var dateReExact = new RegExp('^' + dateRe + '$');
		var dateReRange = /^\[[ ]*([^ ]+)[ ]+TO[ ]+([^ ]+)[ ]*\]$/g
		var range = null;

		if ((range = dateReRange.exec(args[0])) !== null) {

			var from = range[1];
			var to = range[2];

			if (to == 'NOW') {
				to = (new Date()).toISOString();
			}

			if (from == 'NOW') {
				from = (new Date()).toISOString();
			}
			q = {
				range : {}
			};

			if (to == '*') {
				q.range[field] = {
					"gte" : from.replace(
							/([0-9]{4}-[0-9]{2}-[0-9]{2}(?!T[^ ]+))/g,
							'$1T00:00:00.000Z')
				}
			} else if (from == '*') {
				q.range[field] = {
					"lte" : to.replace(
							/([0-9]{4}-[0-9]{2}-[0-9]{2}(?!T[^ ]+))/g,
							'$1T23:59:59.999Z')
				}
			} else {
				q.range[field] = {
					"include_lower" : true,
					"include_upper" : true,
					"from" : from.replace(
							/([0-9]{4}-[0-9]{2}-[0-9]{2}(?!T[^ ]+))/g,
							'$1T00:00:00.000Z'),
					"to" : to.replace(
							/([0-9]{4}-[0-9]{2}-[0-9]{2}(?!T[^ ]+))/g,
							'$1T23:59:59.999Z')
				}
			}
		
		} else if (dateReExact.test(args[0])) {
			
			q.range[field] = {
					"include_lower" : true,
					"include_upper" : true,
					"from" : args[0].replace(
							/([0-9]{4}-[0-9]{2}-[0-9]{2}(?!T[^ ]+))/g,
							'$1T00:00:00.000Z'),
					"to" : args[0].replace(
							/([0-9]{4}-[0-9]{2}-[0-9]{2}(?!T[^ ]+))/g,
							'$1T23:59:59.999Z')
				}
			
		} else  {
			
			q = {
				term : {}
			};
			q.term[field] = args[0];
		} 

	}

	return q;

}

exports.getTermSearchPart = function(field, args) {

	var q = null;
	if (args.length > 1) {
		q = {
			terms : {}
		};
		q.terms[field] = args;
	} else {
		q = {
			term : {}
		};
		q.term[field] = args[0];
	}

	return q;

}

exports.getIdsSearchPart = function(field, args) {

	var q = null;

{'fraction_non_alphanumeric': 0.2300035975536635, 'fraction_numerical': 0.03057920613982492, 'mean_word_length': 2.502729945401092, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5492616', 'n_tokens_mistral': 4174, 'n_tokens_neox': 3362, 'n_words': 578}