<commit_before>const JlTransform = require('./JlTransform');
const Order = require('../Order');

/**
 * Тупейший алгоритм сортировки: сохраняем всё в памятьЮ а потом сортируем
 * TODO Переделать на нормальную схему
 */
class SorterInMemory extends JlTransform
{
	constructor(orders)
	{
		if (!orders.length) {
			throw new Error('Empty orders');
		}

		super(JlTransform.ARRAYS_OF_OBJECTS, JlTransform.ARRAYS_OF_OBJECTS);

		this.orders = orders;
		this.objects = [];
	}

	_transform(chunk, encoding, cb)
	{
		for (var i = 0; i < chunk.length; i++) {
			this.objects.push(chunk[i]);
		}

		cb();
	}

	_flush(cb)
	{
		this.objects.sort(this.sortingFunction());

		this.push(this.objects);
		this.objects = [];

		cb();
	}

	sortingFunction()
	{
		const compare = (row1, row2) => {
			for (let i = 0; i < this.orders.length; i++) {
				const order = this.orders[i];

				const v1 = order.valueFunction(row1);
				const v2 = order.valueFunction(row2);

				const direction = order.direction === Order.DIRECTION_DESC ? -1 : 1;

				if (v1 > v2) {
					return direction;
				} else if (v1 < v2) {
					return -direction;
				}
			}

			return 0;
		};

		return compare;
	}
}

module.exports = SorterInMemory;<commit_msg>In memory sort now compare values as strings to be consistent with external sort. Natural sorting to be implemented in future<commit_after>const JlTransform = require('./JlTransform');
const Order = require('../Order');

/**
 * Тупейший алгоритм сортировки: сохраняем всё в памятьЮ а потом сортируем
 * TODO Переделать на нормальную схему
 */
class SorterInMemory extends JlTransform
{
	constructor(orders)
	{
		if (!orders.length) {
			throw new Error('Empty orders');
		}

		super(JlTransform.ARRAYS_OF_OBJECTS, JlTransform.ARRAYS_OF_OBJECTS);

		this.orders = orders;
		this.objects = [];
	}

	_transform(chunk, encoding, cb)
	{
		for (var i = 0; i < chunk.length; i++) {
			this.objects.push(chunk[i]);
		}

		cb();
	}

	_flush(cb)
	{
		this.objects.sort(this.sortingFunction());

		this.push(this.objects);
		this.objects = [];

		cb();
	}

	sortingFunction()
	{
		const compare = (row1, row2) => {
			for (let i = 0; i < this.orders.length; i++) {
				const order = this.orders[i];

				let v1 = order.valueFunction(row1);
				let v2 = order.valueFunction(row2);

				v1 = v1 === undefined ? '' : JSON.stringify(v1);
				v2 = v2 === undefined ? '' : JSON.stringify(v2);

				const direction = order.direction === Order.DIRECTION_DESC ? -1 : 1;

				if (v1 > v2) {
					return direction;
				} else if (v1 < v2) {
					return -direction;
				}
			}

			return 0;
		};

		return compare;
	}
}

module.exports = SorterInMemory;
{'fraction_non_alphanumeric': 0.1448015122873346, 'fraction_numerical': 0.013610586011342156, 'mean_word_length': 3.1933438985736924, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5422347', 'n_tokens_mistral': 1108, 'n_tokens_neox': 976, 'n_words': 223}