<commit_before>var reader = require('..');
var tape = require('tape');
var fs = require('fs');
var split = require('split');
var AWS = require('aws-sdk-mock');
var path = require('path');

AWS.mock('S3', 'getObject', function (params, callback) {
    console.log('parameter', params);
    callback(null, fs.readFileSync(path.join(__dirname, 'fixtures/getObjectResponse.txt')).toString());
});

function testFunc(r, f, expected, t) {
    var sample = reader.SampleStream({rate: (r * 0.1), filter: f});

    var getStream = require('s3scan').Get('./fixtures/AAAAAAAAAAAAAA.2015-10-19-17.e5b6526a.gz');

    var transformedStream = getStream
    .pipe(split())
    .pipe(sample);

    var count = 0;
    sample.on('data', function (data) {
        count++;
    });
    sample.on('finish', function () {
        t.equals(count, expected, 'got expected records (' + expected + ')');
        t.end();
    });

    for (var k = 0; k < 1000; k++) {
    // each of these is 9 records long
        transformedStream.write({Body: fs.readFileSync(path.join(__dirname + '/fixtures/AAAAAAAAAAAAAA.2015-10-19-17.e5b6526a.gz'))});
    }
    transformedStream.end();
}

var expectedUnfiltered = [949, 1879, 2820, 3728, 4592, 5449, 6323, 7220, 8136];
for (var rate = 1; rate < 10; rate++) {
    tape('unfiltered, sample rate ' + (rate * 0.1).toFixed(1), testFunc.bind(null, rate, false, expectedUnfiltered[rate - 1]));
}

var expectedFiltered = [299, 607, 904, 1218, 1509, 1788, 2085, 2384, 2716];
for (var rate = 1; rate < 10; rate++) {
    tape('filtered, sample rate ' + (rate * 0.1).toFixed(1), testFunc.bind(null, rate, 'a\.json', expectedFiltered[rate - 1]));
}

tape('[s3 getObject] restore', function (assert) {
    AWS.restore('S3');
    assert.end();
});<commit_msg>Adjust sample test to write to sampler stream directly<commit_after>var reader = require('..');
var tape = require('tape');
var fs = require('fs');
var split = require('split');
var AWS = require('aws-sdk-mock');
var path = require('path');

function testFunc(r, f, expected, t) {
    var sample = reader.SampleStream({rate: (r * 0.1), filter: f});

    var transformedStream = split()
    .pipe(sample);

    var count = 0;
    sample.on('data', function (data) {
        count++;
    });
    sample.on('finish', function () {
        t.equals(count, expected, 'got expected records (' + expected + ')');
        t.end();
    });

    for (var k = 0; k < 1000; k++) {
        // each of these is 9 records long
        transformedStream.write(fs.readFileSync(path.join(__dirname + '/fixtures/AAAAAAAAAAAAAA.2015-10-19-17.e5b6526a')));
    }
    transformedStream.end();
}

var expectedUnfiltered = [949, 1879, 2820, 3728, 4592, 5449, 6323, 7220, 8136];
for (var rate = 1; rate < 10; rate++) {
    tape('unfiltered, sample rate ' + (rate * 0.1).toFixed(1), testFunc.bind(null, rate, false, expectedUnfiltered[rate - 1]));
}

var expectedFiltered = [299, 607, 904, 1218, 1509, 1788, 2085, 2384, 2716];
for (var rate = 1; rate < 10; rate++) {
    tape('filtered, sample rate ' + (rate * 0.1).toFixed(1), testFunc.bind(null, rate, 'a\.json', expectedFiltered[rate - 1]));
}

tape('[s3 getObject] restore', function (assert) {
    AWS.restore('S3');
    assert.end();
});
{'fraction_non_alphanumeric': 0.18821705426356589, 'fraction_numerical': 0.07224806201550388, 'mean_word_length': 3.621776504297994, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 3, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '326672', 'n_tokens_mistral': 1267, 'n_tokens_neox': 1096, 'n_words': 283}