<commit_before>'use strict'; // eslint-disable-line strict

const proc = require('child_process');
const process = require('process');
const assert = require('assert');

const program = 's3cmd';
const upload = 'package.json';
const download = 'tmpfile';
const bucket = 'universe';
const nonexist = 'VOID';

const isIronman = process.env.IP ? ['-c', `${__dirname}/s3cfg`, ] : null;

function diff(done) {
    process.stdout.write(`diff ${upload} ${download}\n`);
    proc.spawn('diff', [ upload, download, ]).on('exit', code => {
        proc.spawn('rm', [ download, ]);
        assert.deepEqual(code, 0);
        done();
    });
}

function exec(args, done, exitCode) {
    let exit = exitCode;
    if (exit === undefined) {
        exit = 0;
    }
    let av = args;
    if (isIronman) {
        av = args.concat(isIronman);
    }
    process.stdout.write(`${program} ${av}\n`);
    proc.spawn(program, av, { stdio: 'inherit' })
        .on('exit', code => {
            assert.deepEqual(code, exit);
            done();
        });
}

describe('s3cmd putBucket', () => {
    it('should put a valid bucket', (done) => {
        exec(['mb', `s3://${bucket}`, ], done);
    });

    it('put the same bucket, should fail', (done) => {
        exec(['mb', `s3://${bucket}`, ], done, 13);
    });

    it('put an invalid bucket, should fail', (done) => {
        exec(['mb', `s3://${nonexist}`, ], done, 11);
    });
});

describe('s3cmd getBucket', () => {
    it('should list existing bucket', (done) => {
        exec(['ls', `s3://${bucket}`, ], done);
    });

    it('list non existing bucket, should fail', (done) => {
        exec(['ls', `s3://${nonexist}`, ], done, 12);
    });
});

describe('s3cmd putObject', () => {
    it('put file in existing bucket', (done) => {
        exec(['put', upload, `s3://${bucket}`, ], done);
    });

    it('should put file with the same name in existing bucket', (done) => {
        exec(['put', upload, `s3://${bucket}`, ], done);
    });

    it('put file in non existing bucket, should fail', (done) => {
        exec(['put', upload, `s3://${nonexist}`, ], done, 12);
    });
});

describe('s3cmd getObject', () => {
    it('should get existing file in existing bucket', (done) => {
        exec(['get', `s3://${bucket}/${upload}`, download ], done);
    });

    it('downloaded file should equal uploaded file', (done) => {
        diff(done);
    });

    it('get non existing file in existing bucket, should fail', (done) => {
        exec(['get', `s3://${bucket}/${nonexist}`, download, ], done, 12);
    });

    it('get file in non existing bucket, should fail', (done) => {
        exec(['get', `s3://${nonexist}/${nonexist}`, download, ], done, 12);
    });
});

describe('s3cmd delObject', () => {
    it('should delete existing object', (done) => {
        exec(['rm', `s3://${bucket}/${upload}`, ], done);
    });

    it('delete non existing object, should fail', (done) => {
        exec(['rm', `s3://${bucket}/${nonexist}`, ], done, 12);
    });

    it('try to get the deleted object, should fail', (done) => {
        exec(['get', `s3://${bucket}/${upload}`, download, ], done, 12);
    });
});

describe('connector edge cases', () => {
    it('should put previous file in existing bucket', (done) => {
        exec(['put', upload, `s3://${bucket}`, ], done);
    });

    it('should get existing file in existing bucket', (done) => {
        exec(['get', `s3://${bucket}/${upload}`, download ], done);
    });
});

describe('s3cmd delBucket', () => {
    it('delete non-empty bucket, should fail', (done) => {
        exec(['rb', `s3://${bucket}`, ], done, 13);
    });

    it('should delete remaining object', (done) => {
        exec(['rm', `s3://${bucket}/${upload}`, ], done);
    });

    it('should delete empty bucket', (done) => {
        exec(['rb', `s3://${bucket}`, ], done);
    });

    it('try to get the deleted bucket, should fail', (done) => {
        exec(['ls', `s3://${bucket}`, ], done, 12);
    });
});<commit_msg>TEST: s3cmd functional tests of multipart upload<commit_after>'use strict'; // eslint-disable-line strict

const proc = require('child_process');
const process = require('process');
const assert = require('assert');

const program = 's3cmd';
const upload = 'test1MB';
const download = 'tmpfile';
const bucket = 'universe';
const nonexist = 'VOID';

const isIronman = process.env.IP ? ['-c', `${__dirname}/s3cfg`, ] : null;

function diff(putFile, receivedFile, done) {
    process.stdout.write(`diff ${putFile} ${receivedFile}\n`);
    proc.spawn('diff', [ putFile, receivedFile, ]).on('exit', code => {
        assert.deepEqual(code, 0);
        done();
    });
}

function createFile(name, bytes, callback) {
    process.stdout.write(
        `dd if=/dev/urandom of=${name} bs=${bytes} count=1\n`);
    proc.spawn('dd', [ 'if=/dev/urandom', `of=${name}`,
        `bs=${bytes}`, 'count=1' ], { stdio: 'inherit' }).on('exit', code => {
            assert.deepEqual(code, 0);
            callback();
        });
}

function deleteFile(file, callback) {
    process.stdout.write(`rm ${file}\n`);
    proc.spawn('rm', [ `${file}`, ]).on('exit', () => {
        callback();
    });
}

function exec(args, done, exitCode) {
    let exit = exitCode;
    if (exit === undefined) {
        exit = 0;
    }
    let av = args;
    if (isIronman) {
        av = args.concat(isIronman);
    }
    process.stdout.write(`${program} ${av}\n`);
    proc.spawn(program, av, { stdio: 'inherit' })
        .on('exit', code => {
            assert.deepEqual(code, exit);
            done();
        });
}

describe('s3cmd putBucket', () => {
    it('should put a valid bucket', (done) => {
        exec(['mb', `s3://${bucket}`, ], done);
    });

    it('put the same bucket, should fail', (done) => {
        exec(['mb', `s3://${bucket}`, ], done, 13);
    });

    it('put an invalid bucket, should fail', (done) => {
        exec(['mb', `s3://${nonexist}`, ], done, 11);
    });
});

describe('s3cmd getBucket', () => {
    it('should list existing bucket', (done) => {
        exec(['ls', `s3://${bucket}`, ], done);
    });

    it('list non existing bucket, should fail', (done) => {
        exec(['ls', `s3://${nonexist}`, ], done, 12);
    });
});

describe('s3cmd putObject', () => {
    it('should put file in existing bucket', (done) => {
        createFile(upload, 1048576, ()=> {
            exec(['put', upload, `s3://${bucket}`, ], done);
        });
    });

    it('should put file with the same name in existing bucket', (done) => {
        exec(['put', upload, `s3://${bucket}`, ], done);
    });

    it('put file in non existing bucket, should fail', (done) => {
        exec(['put', upload, `s3://${nonexist}`, ], done, 12);
    });
});

describe('s3cmd getObject', () => {
    it('should get existing file in existing bucket', (done) => {
        exec(['get', `s3://${bucket}/${upload}`, download ], done);
    });

    it('downloaded file should equal uploaded file', (done) => {
        diff(upload, download, () => {
            deleteFile(download, () => {
                done();
            });
        });
    });

    it('get non existing file in existing bucket, should fail', (done) => {
        exec(['get', `s3://${bucket}/${nonexist}`, download, ], done, 12);
    });

    it('get file in non existing bucket, should fail', (done) => {
        exec(['get', `s3://${nonexist}/${nonexist}`, download, ], done, 12);
    });
});

describe('s3cmd delObject', () => {
    it('should delete existing object', (done) => {
        exec(['rm', `s3://${bucket}/${upload}`, ], done);
    });

    it('delete non existing object, should fail', (done) => {
        exec(['rm', `s3://${bucket}/${nonexist}`, ], done, 12);
    });

    it('try to get the deleted object, should fail', (done) => {
        exec(['get', `s3://${bucket}/${upload}`, download, ], done, 12);
    });
});

describe('connector edge cases', () => {
    it('should put previous file in existing bucket', (done) => {
        exec(['put', upload, `s3://${bucket}`, ], () => {
            deleteFile(upload, () => {
                done();
            });
        });
    });

    it('should get existing file in existing bucket', (done) => {
        exec(['get', `s3://${bucket}/${upload}`, download ], () => {
            deleteFile(download, () => {
                done();
            });
        });
    });
});

describe('s3cmd multipart upload', () => {
    it('should put an object via a multipart upload', (done) => {
        createFile('test16MB', 16777216, ()=> {
            exec(['put', 'test16MB', `s3://${bucket}`, ], done);
        });
    });

    it('should get an object that was put via multipart upload', (done) => {
        exec(['get', `s3://${bucket}/test16MB`, 'download16MB' ], done);
    });

    it('downloaded file should equal uploaded file', (done) => {
        diff('test16MB', 'download16MB', ()=> {
            deleteFile('test16MB', () => {
                deleteFile('download16MB', () => {
                    done();
                });
            });
        });
    });

    it('should delete multipart uploaded object', (done) => {
        exec(['rm', `s3://${bucket}/test16MB`, ], done);
    });

    it('should not be able to get deleted object', (done) => {
        exec(['get', `s3://${bucket}/test16MB`, download, ], done, 12);
    });
});

describe('s3cmd delBucket', () => {
    it('delete non-empty bucket, should fail', (done) => {
        exec(['rb', `s3://${bucket}`, ], done, 13);
    });

    it('should delete remaining object', (done) => {
        exec(['rm', `s3://${bucket}/${upload}`, ], done);
    });

    it('should delete empty bucket', (done) => {
        exec(['rb', `s3://${bucket}`, ], done);
    });

    it('try to get the deleted bucket, should fail', (done) => {
        exec(['ls', `s3://${bucket}`, ], done, 12);
    });
});
{'fraction_non_alphanumeric': 0.23401236444714704, 'fraction_numerical': 0.014898145332927942, 'mean_word_length': 2.649408284023669, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 82, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 25, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5919226', 'n_tokens_mistral': 3245, 'n_tokens_neox': 3120, 'n_words': 769}