<commit_before>var request = require('superagent').agent();

var init = require('../main/init');
var config = require('../main/config');

var opt = {};

exports.options = function (_opt) {
	for(var p in _opt) {
		opt[p] = _opt[p];
	}
	return exports;
};

init.add(function (next) {

	var url = config.data.esUrl + '/' + config.data.esIndexName;

	console.log('elasticsearch: ' + url);

	initExports();
	checkDrop(next);

	function checkDrop(next) {
		if (opt.dropIndex) {
			exports.dropIndex(next);
		} else {
			setSchema(next)
		}
	}

	function setSchema(next) {
		var schema = {
			settings: {
				index: {
					number_of_shards : 1,
					number_of_replicas : 1,
					analysis: { analyzer: { 'default': { type: 'cjk' } } }
				}
			},
			mappings: {
				'post': {
					properties: {
						threadId: { type: 'integer', index: 'no', include_in_all: false },
						categoryId: { type: 'integer', index: 'no', include_in_all: false },
						created: { type: 'date', index: 'not_analyzed', include_in_all: false },
						title: { type: 'string', index: 'no', include_in_all: false },
						titlei: { type: 'string', index: 'no', include_in_all: true },
						writer: { type: 'string', index: 'no', include_in_all: true },
						text: { type: 'string', index: 'no', include_in_all: true }
					}
				}
			}
		};
		request.post(url).send(schema).end(function (err, res) {
			if (err) return next(err);
			if (res.error) return next(res.error);
			next(null, res);
		});
	};

	function initExports() {
		exports.dropIndex = function (next) {
			request.del(url, function (err, res) {
				if (err) return next(err);
				if (res.error) return next(res.error);
				setSchema(next);
			});
		};

		exports.flush = function (next) {
			request.post(url + '/_flush', function (err, res) {
				if (res.error) return next(res.error);
				next(err, res);
			});
		};

		exports.updatePost = function (thread, post, next) {
			request.put(url + '/post/' + post._id).send({
				threadId: thread._id,
				categoryId: thread.categoryId,
				created: post.created,
				title: thread.title,
				titlei: thread.created.getTime() === post.created.getTime() ? thread.title : '',
				writer: post.writer,
				text: post.text,
				visible: post.visible
			}).end(function (err, res) {
					if (res.error) return next(res.error);
					next(err, res);
				});
		};

		exports.getPost = function (postId, next) {
			request.get(url + '/post/' + postId, function (err, res) {
				if (err) return next(err);
				if (res.error) return next(res.error);
				res.body._id = parseInt(res.body._id);
				res.body._source.created = new Date(res.body._source.created);
				next(err, res);
			});
		};

		exports.searchPost = function (body, next) {
			request.post(url + '/post/_search').send(body).end(function (err, res) {
				if (err) return next(err);
				if (res.error) return next(res.error);
				if (res.body.hits) {
					var hits = res.body.hits.hits;
					var len = hits.length;
					var i;
					for (i = 0; i < len; i++) {
						var hit = hits[i];
						hit._id = parseInt(hit._id);
						hit._source.created = new Date(hit._source.created);
					}
				}
				next(err, res);
			});
		};
	}

});

<commit_msg>es.js, if (res.error) return next(res.body);<commit_after>var request = require('superagent').agent();

var init = require('../main/init');
var config = require('../main/config');

var opt = {};

exports.options = function (_opt) {
	for(var p in _opt) {
		opt[p] = _opt[p];
	}
	return exports;
};

init.add(function (next) {

	var url = config.data.esUrl + '/' + config.data.esIndexName;
	var log = 'elasticsearch: ' + url;

	initExports();

	checkDrop(function (err) {
		console.log(log);
		next(err);
	});

	function checkDrop(next) {
		if (opt.dropIndex) {
			log += ' drop-index';
			exports.dropIndex(next);
		} else {
			setSchema(next)
		}
	}

	function setSchema(next) {
		var schema = {
			settings: {
				index: {
					number_of_shards : 1,
					number_of_replicas : 1,
					analysis: { analyzer: { 'default': { type: 'cjk' } } }
				}
			},
			mappings: {
				'post': {
					properties: {
						threadId: { type: 'integer', index: 'no', include_in_all: false },
						categoryId: { type: 'integer', index: 'no', include_in_all: false },
						created: { type: 'date', index: 'not_analyzed', include_in_all: false },
						title: { type: 'string', index: 'no', include_in_all: false },
						titlei: { type: 'string', index: 'no', include_in_all: true },
						writer: { type: 'string', index: 'no', include_in_all: true },
						text: { type: 'string', index: 'no', include_in_all: true }
					}
				}
			}
		};
		request.post(url).send(schema).end(function (err, res) {
			if (err) return next(err);
			if (res.error) return next(res.body);
			next(null, res);
		});
	};

	function initExports() {
		exports.dropIndex = function (next) {
			request.del(url, function (err, res) {
				if (err) return next(err);
				if (res.error) return next(res.body);
				setSchema(next);
			});
		};

		exports.flush = function (next) {
			request.post(url + '/_flush', function (err, res) {
				if (err) return next(err);
				if (res.error) return next(res.body);
				next(null, res);
			});
		};

		exports.update = function (thread, post, next) {
			var form = {
				threadId: thread._id,
				categoryId: thread.categoryId,
				created: post.created,
				title: thread.title,
				titlei: thread.created.getTime() === post.created.getTime() ? thread.title : '',
				writer: post.writer,
				text: post.text,
				visible: post.visible
			}
			request.put(url + '/post/' + post._id).send(form).end(function (err, res) {
				if (err) return next(err);
				if (res.error) return next(res.body);
				next(null, res);
			});
		};

		exports.getPost = function (postId, next) {
			request.get(url + '/post/' + postId, function (err, res) {
				if (err) return next(err);
				if (res.error) return next(res.body);
				res.body._id = parseInt(res.body._id);
				res.body._source.created = new Date(res.body._source.created);
				next(null, res);
			});
		};

		exports.search = function (form, next) {
			request.post(url + '/post/_search').send(form).end(function (err, res) {
				if (err) return next(err);
				if (res.error) return next(res.body);
				if (res.body.hits) {
					var hits = res.body.hits.hits;
					var len = hits.length;
					var i;
					for (i = 0; i < len; i++) {
						var hit = hits[i];
						hit._id = parseInt(hit._id);
						hit._source.created = new Date(hit._source.created);
					}
				}
				next(null, res);
			});
		};
	}

});


{'fraction_non_alphanumeric': 0.17838003368549993, 'fraction_numerical': 0.0009186954524575103, 'mean_word_length': 2.9090365050867746, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5585766', 'n_tokens_mistral': 2831, 'n_tokens_neox': 2322, 'n_words': 526}