<commit_before>'use strict';

const {checkedRun, testServerHandler} = require('./util');

const assert = require('assert');
const fs = require('fs');
const http = require('http');
const https = require('https');
const http2 = require('../node_modules/http2');

function runTestSuite(name, protocol, server) {
    const port = 8000;
    const baseUrl = `${protocol}://localhost:${port}`;
    before('Start web server', (done) => {
        server.on('request', testServerHandler);
        server.listen(port, done);
    });
    after('Stop web server', (done) => {
        server.close(done);
    });
    describe('Misc', () => {
        it('Properly handle repeated keys in query strings', (done) => {
            checkedRun({
                done,
                urls: [
                    `${baseUrl}/get?a=1&b=2&a=1`
                ],
                check: (events, har) => {
                    assert.strictEqual(har.log.entries.length, 1, 'entries');
                    assert.strictEqual(har.log.entries[0].request.queryString.length, 3, 'query string');
                }
            });
        });
        it('Parse application/x-www-form-urlencoded POST', (done) => {
            checkedRun({
                done,
                urls: [
                    `${baseUrl}/generate_post?type=application/x-www-form-urlencode`
                ],
                check: (events, har) => {
                    assert.strictEqual(har.log.entries.length, 2, 'entries');
                    const {postData} = har.log.entries[1].request;
                    assert.strictEqual(postData.mimeType, 'application/x-www-form-urlencoded', 'mimeType');
                    assert.deepEqual(postData.params, [{name: 'name', value: 'value'}], 'params');
                }
            });
        });
        it('Parse multipart/form-data POST', (done) => {
            checkedRun({
                done,
                urls: [
                    `${baseUrl}/generate_post?type=multipart/form-data`
                ],
                check: (events, har) => {
                    assert.strictEqual(har.log.entries.length, 2, 'entries');
                    const {postData} = har.log.entries[1].request;
                    assert(postData.mimeType.startsWith('multipart/form-data', 'mimeType'));
                    assert.deepEqual(postData.params, [], 'params');
                }
            });
        });
    });
    describe('Sizes', () => {
        it('Properly measure fixed-size responses', (done) => {
            const size = 1000;
            checkedRun({
                done,
                urls: [
                    `${baseUrl}/data?size=${size}`
                ],
                check: (events, har) => {
                    assert.strictEqual(har.log.entries.length, 1, 'entries');
                    const {bodySize, headersSize, content, _transferSize} = har.log.entries[0].response;
                    assert.strictEqual(content.size, size, 'size');
                    if (name === 'http2') {
                        assert.strictEqual(bodySize, -1, 'body size');
                        assert.strictEqual(content.compression, undefined, 'compression');
                        // larger due to headers
                        assert(_transferSize > size, 'transfer size');
                    } else {
                        assert.strictEqual(bodySize, size, 'body size');
                        assert.strictEqual(content.compression, content.size - bodySize, 'compression');
                        assert.strictEqual(_transferSize, bodySize + headersSize, 'transfer size');
                    }
                }
            });
        });
        it('Properly measure chunked responses', (done) => {
            const size = 1000;
            const chunks = 10;
            const total = size * chunks;<commit_msg>Add test case for request body fetching<commit_after>'use strict';

const {checkedRun, testServerHandler} = require('./util');

const assert = require('assert');
const fs = require('fs');
const http = require('http');
const https = require('https');
const http2 = require('../node_modules/http2');

function runTestSuite(name, protocol, server) {
    const port = 8000;
    const baseUrl = `${protocol}://localhost:${port}`;
    before('Start web server', (done) => {
        server.on('request', testServerHandler);
        server.listen(port, done);
    });
    after('Stop web server', (done) => {
        server.close(done);
    });
    describe('Misc', () => {
        it('Properly handle repeated keys in query strings', (done) => {
            checkedRun({
                done,
                urls: [
                    `${baseUrl}/get?a=1&b=2&a=1`
                ],
                check: (events, har) => {
                    assert.strictEqual(har.log.entries.length, 1, 'entries');
                    assert.strictEqual(har.log.entries[0].request.queryString.length, 3, 'query string');
                }
            });
        });
        it('Parse application/x-www-form-urlencoded POST', (done) => {
            checkedRun({
                done,
                urls: [
                    `${baseUrl}/generate_post?type=application/x-www-form-urlencode`
                ],
                check: (events, har) => {
                    assert.strictEqual(har.log.entries.length, 2, 'entries');
                    const {postData} = har.log.entries[1].request;
                    assert.strictEqual(postData.mimeType, 'application/x-www-form-urlencoded', 'mimeType');
                    assert.deepEqual(postData.params, [{name: 'name', value: 'value'}], 'params');
                }
            });
        });
        it('Parse multipart/form-data POST', (done) => {
            checkedRun({
                done,
                urls: [
                    `${baseUrl}/generate_post?type=multipart/form-data`
                ],
                check: (events, har) => {
                    assert.strictEqual(har.log.entries.length, 2, 'entries');
                    const {postData} = har.log.entries[1].request;
                    assert(postData.mimeType.startsWith('multipart/form-data', 'mimeType'));
                    assert.deepEqual(postData.params, [], 'params');
                }
            });
        });
        it('Return the response body', (done) => {
            const size = 1000000;
            checkedRun({
                done,
                urls: [
                    `${baseUrl}/data?size=${size}`
                ],
                options: {
                    content: true
                },
                check: (events, har) => {
                    assert.strictEqual(har.log.entries.length, 1, 'entries');
                    assert.strictEqual(har.log.entries[0].response.content.text, Buffer.alloc(size, 'x').toString(), 'content');
                }
            });
        }).timeout(0); // disable for this one
    });
    describe('Sizes', () => {
        it('Properly measure fixed-size responses', (done) => {
            const size = 1000;
            checkedRun({
                done,
                urls: [
                    `${baseUrl}/data?size=${size}`
                ],
                check: (events, har) => {
                    assert.strictEqual(har.log.entries.length, 1, 'entries');
                    const {bodySize, headersSize, content, _transferSize} = har.log.entries[0].response;
                    assert.strictEqual(content.size, size, 'size');
                    if (name === 'http2') {
                        assert.strictEqual(bodySize, -1, 'body size');
                        assert.strictEqual(content.compression, undefined, 'compression');
                        // larger due to headers
                        assert(_transferSize > size, 'transfer size');
                    } else {
                        assert.strictEqual(bodySize, size, 'body size');
                        assert.strictEqual(content.compression, content.size - bodySize, 'compression');
                        assert.strictEqual(_transferSize, bodySize + headersSize, 'transfer size');
                    }
                }
            });
        });
        it('Properly measure chunked responses', (done) => {
            const size = 1000;
            const chunks = 10;
            const total = size * chunks;
{'fraction_non_alphanumeric': 0.14118490205446727, 'fraction_numerical': 0.008361204013377926, 'mean_word_length': 1.5001492982979994, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1940260', 'n_tokens_mistral': 2243, 'n_tokens_neox': 2098, 'n_words': 413}