<commit_before>var _ = require('underscore');
var Promise = require('bluebird');
var request = Promise.promisifyAll(require('request'));
request.async = Promise.promisify(request);
var retry = require('bluebird-retry');
var expect = require('chai').expect;
var util = require('util');

var test_utils = require('./elastic-test-utils');
var juttle_test_utils = require('juttle/test/runtime/specs/juttle-test-utils');
var check_juttle = juttle_test_utils.check_juttle;
var points = require('./apache-sample');

function format_juttle_result_like_es(pts) {
    pts.forEach(function(pt) {
        _.each(pt, function nullify_infinities(value, key) {
            if (value === Infinity || value === -Infinity) {
                pt[key] = null;
            }
        });
    });

    return pts;
}

var start = new Date(points[0].time).toISOString();
var end_ms = new Date(_.last(points).time).getTime() + 1;
var end = new Date(end_ms).toISOString();

// Register the adapter
require('./elastic-test-utils');

var modes = test_utils.modes;

describe('optimization', function() {
    this.timeout(300000);

    modes.forEach(function(type) {
        describe(type, function() {
            after(function() {
                return test_utils.clear_data(type);
            });

            before(function() {
                return test_utils.write(points, {id: type})
                .then(function() {
                    return test_utils.verify_import(points, type);
                });
            });

            it('optimizes head', function() {
                return test_utils.read({id: type}, '| head 3')
                .then(function(result) {
                    var expected = points.slice(0, 3);
                    test_utils.check_result_vs_expected_sorting_by(result.sinks.table, expected, 'bytes');
                    expect(result.prog.graph.es_opts.limit).equal(3);
                });
            });

            it('optimizes head with a nontrivial time filter', function() {
                var start = '2014-09-17T14:13:43.000Z';
                var end = '2014-09-17T14:13:46.000Z';
                return test_utils.read({from: start, to: end, id: type}, '| head 2')
                .then(function(result) {
                    var expected = points.filter(function(pt) {
                        return pt.time >= start && pt.time < end;
                    }).slice(0, 2);

                    test_utils.check_result_vs_expected_sorting_by(result.sinks.table, expected, 'bytes');
                    expect(result.prog.graph.es_opts.limit).equal(2);
                });
            });

            it('optimizes head with tag filter', function() {
                return test_utils.read({id: type}, 'clientip = "172.16.58.3" | head 2')
                .then(function(result) {
                    var expected = points.filter(function(pt) {
                        return pt.clientip === '172.16.58.3';
                    }).slice(0, 2);

                    test_utils.check_result_vs_expected_sorting_by(result.sinks.table, expected, 'bytes');
                    expect(result.prog.graph.es_opts.limit).equal(2);
                });
            });

            it('optimizes head 0 (returns nothing)', function() {
                return test_utils.read({id: type}, '| head 0')
                .then(function(result) {
                    expect(result.sinks.table).deep.equal([]);
                    expect(result.prog.graph.es_opts.limit).equal(0);
                });
            });

            describe('reduce', function() {<commit_msg>Describe block for head<commit_after>var _ = require('underscore');
var Promise = require('bluebird');
var request = Promise.promisifyAll(require('request'));
request.async = Promise.promisify(request);
var retry = require('bluebird-retry');
var expect = require('chai').expect;
var util = require('util');

var test_utils = require('./elastic-test-utils');
var juttle_test_utils = require('juttle/test/runtime/specs/juttle-test-utils');
var check_juttle = juttle_test_utils.check_juttle;
var points = require('./apache-sample');

function format_juttle_result_like_es(pts) {
    pts.forEach(function(pt) {
        _.each(pt, function nullify_infinities(value, key) {
            if (value === Infinity || value === -Infinity) {
                pt[key] = null;
            }
        });
    });

    return pts;
}

var start = new Date(points[0].time).toISOString();
var end_ms = new Date(_.last(points).time).getTime() + 1;
var end = new Date(end_ms).toISOString();

// Register the adapter
require('./elastic-test-utils');

var modes = test_utils.modes;

describe('optimization', function() {
    this.timeout(300000);

    modes.forEach(function(type) {
        describe(type, function() {
            after(function() {
                return test_utils.clear_data(type);
            });

            before(function() {
                return test_utils.write(points, {id: type})
                .then(function() {
                    return test_utils.verify_import(points, type);
                });
            });

            describe('head', function() {
                it('optimizes head', function() {
                    return test_utils.read({id: type}, '| head 3')
                    .then(function(result) {
                        var expected = points.slice(0, 3);
                        test_utils.check_result_vs_expected_sorting_by(result.sinks.table, expected, 'bytes');
                        expect(result.prog.graph.es_opts.limit).equal(3);
                    });
                });

                it('optimizes head with a nontrivial time filter', function() {
                    var start = '2014-09-17T14:13:43.000Z';
                    var end = '2014-09-17T14:13:46.000Z';
                    return test_utils.read({from: start, to: end, id: type}, '| head 2')
                    .then(function(result) {
                        var expected = points.filter(function(pt) {
                            return pt.time >= start && pt.time < end;
                        }).slice(0, 2);

                        test_utils.check_result_vs_expected_sorting_by(result.sinks.table, expected, 'bytes');
                        expect(result.prog.graph.es_opts.limit).equal(2);
                    });
                });

                it('optimizes head with tag filter', function() {
                    return test_utils.read({id: type}, 'clientip = "172.16.58.3" | head 2')
                    .then(function(result) {
                        var expected = points.filter(function(pt) {
                            return pt.clientip === '172.16.58.3';
                        }).slice(0, 2);

                        test_utils.check_result_vs_expected_sorting_by(result.sinks.table, expected, 'bytes');
                        expect(result.prog.graph.es_opts.limit).equal(2);
                    });
                });

                it('optimizes head 0 (returns nothing)', function() {
                    return test_utils.read({id: type}, '| head 0')
                    .then(function(result) {
                        expect(result.sinks.table).deep.equal([]);
                        expect(result.prog.graph.es_opts.limit).equal(0);
                    });
                });
            });

            describe('reduce', function() {
{'fraction_non_alphanumeric': 0.13867028493894165, 'fraction_numerical': 0.019810040705563095, 'mean_word_length': 1.7984054669703873, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7026842', 'n_tokens_mistral': 2200, 'n_tokens_neox': 1984, 'n_words': 369}