<commit_before>var Promise = require('bluebird');
var path = require('path');
var retry = require('bluebird-retry');
var expect = require('chai').expect;
var _ = require('underscore');
var Elasticsearch = require('elasticsearch');
var AmazonElasticsearchClient = require('aws-es2');
var uuid = require('uuid');
var util = require('util');

var juttle_test_utils = require('juttle/test').utils;

var check_juttle = juttle_test_utils.check_juttle;
var configureAdapter = juttle_test_utils.configureAdapter;

var AWS_HOST = 'search-dave-6jy2cskdfaye4ji6gfa6x375ve.us-west-2.es.amazonaws.com';
var AWS_REGION = 'us-west-2';
var TEST_RUN_ID = uuid.v4().substring(0, 8);

var LOCAL = 'local';
var AWS = 'aws';

var mode = process.env.TESTMODE;
var modes;
if (mode === 'all') {
    modes = [LOCAL, AWS];
} else if (mode === 'aws') {
    modes = [AWS];
} else {
    modes = [LOCAL];
}

var local_client = Promise.promisifyAll(new Elasticsearch.Client({
    host: 'localhost:9200'
}));
Promise.promisifyAll(local_client.indices);

var test_index = 'my_index';
var has_index_id = 'has_default_index';
var has_default_type_id = 'has_default_type';
var aws_has_default_type_id = 'aws_has_default_type';

var config = [
    {
        id: LOCAL,
        address: 'localhost',
        port: 9200
    },
    {
        id: 'b',
        address: 'localhost',
        port: 9999 // b's config is botched so we get errors reading from it
    },
    {
        id: has_index_id,
        address: 'localhost',
        port: 9200,
        index: test_index
    },
    {
        id: has_default_type_id,
        address: 'localhost',
        port: 9200,
        type: 'my_test_type'
    }
];

var aws_client;
if (_.contains(modes, AWS)) {
    aws_client = Promise.promisifyAll(new AmazonElasticsearchClient({
        accessKeyId: process.env.AWS_ACCESS_KEY_ID,
        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
        service: 'es',
        region: AWS_REGION,
        host: AWS_HOST
    }));

    Promise.promisifyAll(aws_client.indices);

    config.push({
        id: AWS,
        aws: true,
        endpoint: AWS_HOST,
        region: AWS_REGION
    });

    config.push({
        id: aws_has_default_type_id,
        aws: true,
        type: 'aws_default_type',
        endpoint: AWS_HOST,
        region: AWS_REGION
    });
}

config.path = path.resolve(__dirname, '..');
configureAdapter({elastic: config});

function _client_for_mode(mode) {
    return mode === 'aws' ? aws_client : local_client;
}

function _option_is_moment(key) {
    return key === 'from' || key === 'to' || key === 'lag';
}

function options_from_object(options) {
    return _.reduce(options, function(memo, value, key) {
        var str = _option_is_moment(key) ? ':%s:' : '"%s"';
        return memo + '-' + key + ' ' + util.format(str, value) + ' ';
    }, '');
}

function write(points, options, extra) {
    options = options || {};
    options = _.clone(options);
    options.index = options.index || TEST_RUN_ID;
    extra = extra || '';

    var opts = options_from_object(options);

    var program = 'emit -points %s | write elastic %s %s';
    var write_program = util.format(program, JSON.stringify(points), opts, extra);

    return check_juttle({
        program: write_program
    });
}

var DEFAULT_TEST_READ_OPTIONS = {
    from: '10 years ago',
    to: 'now',
    index: TEST_RUN_ID
};

function read(options, extra, deactivateAfter) {
    options = options || {};
    options = _.clone(options);
    _.defaults(options, DEFAULT_TEST_READ_OPTIONS);
    extra = extra || '';

    var opts = options_from_object(options);

    var program = 'read elastic %s %s';
    var read_program = util.format(program, opts, extra);

    return check_juttle({
        program: read_program,
        realtime: !!deactivateAfter
    }, deactivateAfter);
}

function clear_data(mode, indexes) {
    indexes = indexes || TEST_RUN_ID + '*';
    var client = _client_for_mode(mode);
    return client.indices.deleteAsync({index: indexes});
}

function verify_import(points, mode, indexes, options) {
    options = options || {};
    var client = _client_for_mode(mode);
    var request_body = {
        index: indexes || TEST_RUN_ID + '*',
        type: '',
        body: {
            size: 10000
        }
    };<commit_msg>test: fix deactivateAfter option<commit_after>var Promise = require('bluebird');
var path = require('path');
var retry = require('bluebird-retry');
var expect = require('chai').expect;
var _ = require('underscore');
var Elasticsearch = require('elasticsearch');
var AmazonElasticsearchClient = require('aws-es2');
var uuid = require('uuid');
var util = require('util');

var juttle_test_utils = require('juttle/test').utils;

var check_juttle = juttle_test_utils.check_juttle;
var configureAdapter = juttle_test_utils.configureAdapter;

var AWS_HOST = 'search-dave-6jy2cskdfaye4ji6gfa6x375ve.us-west-2.es.amazonaws.com';
var AWS_REGION = 'us-west-2';
var TEST_RUN_ID = uuid.v4().substring(0, 8);

var LOCAL = 'local';
var AWS = 'aws';

var mode = process.env.TESTMODE;
var modes;
if (mode === 'all') {
    modes = [LOCAL, AWS];
} else if (mode === 'aws') {
    modes = [AWS];
} else {
    modes = [LOCAL];
}

var local_client = Promise.promisifyAll(new Elasticsearch.Client({
    host: 'localhost:9200'
}));
Promise.promisifyAll(local_client.indices);

var test_index = 'my_index';
var has_index_id = 'has_default_index';
var has_default_type_id = 'has_default_type';
var aws_has_default_type_id = 'aws_has_default_type';

var config = [
    {
        id: LOCAL,
        address: 'localhost',
        port: 9200
    },
    {
        id: 'b',
        address: 'localhost',
        port: 9999 // b's config is botched so we get errors reading from it
    },
    {
        id: has_index_id,
        address: 'localhost',
        port: 9200,
        index: test_index
    },
    {
        id: has_default_type_id,
        address: 'localhost',
        port: 9200,
        type: 'my_test_type'
    }
];

var aws_client;
if (_.contains(modes, AWS)) {
    aws_client = Promise.promisifyAll(new AmazonElasticsearchClient({
        accessKeyId: process.env.AWS_ACCESS_KEY_ID,
        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
        service: 'es',
        region: AWS_REGION,
        host: AWS_HOST
    }));

    Promise.promisifyAll(aws_client.indices);

    config.push({
        id: AWS,
        aws: true,
        endpoint: AWS_HOST,
        region: AWS_REGION
    });

    config.push({
        id: aws_has_default_type_id,
        aws: true,
        type: 'aws_default_type',
        endpoint: AWS_HOST,
        region: AWS_REGION
    });
}

config.path = path.resolve(__dirname, '..');
configureAdapter({elastic: config});

function _client_for_mode(mode) {
    return mode === 'aws' ? aws_client : local_client;
}

function _option_is_moment(key) {
    return key === 'from' || key === 'to' || key === 'lag';
}

function options_from_object(options) {
    return _.reduce(options, function(memo, value, key) {
        var str = _option_is_moment(key) ? ':%s:' : '"%s"';
        return memo + '-' + key + ' ' + util.format(str, value) + ' ';
    }, '');
}

function write(points, options, extra) {
    options = options || {};
    options = _.clone(options);
    options.index = options.index || TEST_RUN_ID;
    extra = extra || '';

    var opts = options_from_object(options);

    var program = 'emit -points %s | write elastic %s %s';
    var write_program = util.format(program, JSON.stringify(points), opts, extra);

    return check_juttle({
        program: write_program
    });
}

var DEFAULT_TEST_READ_OPTIONS = {
    from: '10 years ago',
    to: 'now',
    index: TEST_RUN_ID
};

function read(options, extra, deactivateAfter) {
    options = options || {};
    options = _.clone(options);
    _.defaults(options, DEFAULT_TEST_READ_OPTIONS);
    extra = extra || '';

    var opts = options_from_object(options);

    var program = 'read elastic %s %s';
    var read_program = util.format(program, opts, extra);

    return check_juttle({
        program: read_program,
        realtime: !!deactivateAfter,
        deactivateAfter: deactivateAfter
    });
}

function clear_data(mode, indexes) {
    indexes = indexes || TEST_RUN_ID + '*';
    var client = _client_for_mode(mode);
    return client.indices.deleteAsync({index: indexes});
}

function verify_import(points, mode, indexes, options) {
    options = options || {};
    var client = _client_for_mode(mode);
    var request_body = {
        index: indexes || TEST_RUN_ID + '*',
        type: '',
        body: {
            size: 10000
        }
    };
{'fraction_non_alphanumeric': 0.13865059123579876, 'fraction_numerical': 0.00950614421516346, 'mean_word_length': 3.0905642484589855, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3714125', 'n_tokens_mistral': 2901, 'n_tokens_neox': 2830, 'n_words': 636}