<commit_before>#include "gtest/gtest.h"
#include "S3Downloader.cpp"

TEST(OffsetMgr, simple) {
    OffsetMgr *o = new OffsetMgr(4096, 1000);
    Range r = o->NextOffset();
    EXPECT_EQ(r.offset, 0);
    EXPECT_EQ(r.len, 1000);

    r = o->NextOffset();
    EXPECT_EQ(r.offset, 1000);
    EXPECT_EQ(r.len, 1000);

    r = o->NextOffset();
    EXPECT_EQ(r.offset, 2000);
    EXPECT_EQ(r.len, 1000);

    r = o->NextOffset();
    EXPECT_EQ(r.offset, 3000);
    EXPECT_EQ(r.len, 1000);

    r = o->NextOffset();
    EXPECT_EQ(r.offset, 4000);
    EXPECT_EQ(r.len, 96);
    delete o;
}

TEST(OffsetMgr, reset) {
    OffsetMgr *o = new OffsetMgr(1024, 100);

    o->NextOffset();
    o->NextOffset();
    o->Reset(333);
    Range r = o->NextOffset();

    EXPECT_EQ(r.offset, 333);
    EXPECT_EQ(r.len, 100);
    delete o;
}

#define HOSTSTR "localhost"
#define BUCKETSTR "s3"
TEST(ListBucket, fake) {
    ListBucketResult *r = ListBucket_FakeHTTP(HOSTSTR, BUCKETSTR);
    char urlbuf[256];
    EXPECT_NE(r, (void *)NULL);
    if (!r) return;
    vector<BucketContent *>::iterator i;
    for (i = r->contents.begin(); i != r->contents.end(); i++) {
        BucketContent *p = *i;
        sprintf(urlbuf, "http://%s/%s/%s", HOSTSTR, BUCKETSTR, p->Key());
        printf("%s, %d\n", urlbuf, p->Size());
        // printdata(urlbuf, p->Size(), &cred);
    }
    delete r;
}

TEST(Downloader, divisible) {
    InitLog();
    uint64_t len = 4 * 1024;
    char *buf = (char *)malloc(4 * 1024);
    Downloader *d = new Downloader(8);
    MD5Calc m;

    ASSERT_NE((void *)NULL, buf);

    ASSERT_TRUE(d->init(
        "http://localhost/metro.pivotal.io/debian-8.2.0-amd64-netinst.iso",
        258998272, 4 * 1024 * 1024, NULL));

    while (1) {
        ASSERT_TRUE(d->get(buf, len));
        if (len == 0) {
            break;
        }
        m.Update(buf, len);
        len = 4 * 1024;
    }

    EXPECT_STREQ("762eb3dfc22f85faf659001ebf270b4f", m.Get());
    delete d;
    free(buf);
}

TEST(Downloader, equal) {
    InitLog();
    uint64_t len = 4 * 1024 * 1024;
    char *buf = (char *)malloc(4 * 1024 * 1024);
    Downloader *d = new Downloader(8);
    MD5Calc m;

    ASSERT_NE((void *)NULL, buf);

    ASSERT_TRUE(d->init(
        "http://localhost/metro.pivotal.io/debian-8.2.0-amd64-netinst.iso",
        258998272, 4 * 1024 * 1024, NULL));

    while (1) {
        ASSERT_TRUE(d->get(buf, len));
        if (len == 0) {
            break;
        }
        m.Update(buf, len);
        len = 4 * 1024 * 1024;
    }

    EXPECT_STREQ("762eb3dfc22f85faf659001ebf270b4f", m.Get());
    delete d;
    free(buf);
}<commit_msg>make DownloadTest an interface<commit_after>#include "gtest/gtest.h"
#include "S3Downloader.cpp"

TEST(OffsetMgr, simple) {
    OffsetMgr *o = new OffsetMgr(4096, 1000);
    Range r = o->NextOffset();
    EXPECT_EQ(r.offset, 0);
    EXPECT_EQ(r.len, 1000);

    r = o->NextOffset();
    EXPECT_EQ(r.offset, 1000);
    EXPECT_EQ(r.len, 1000);

    r = o->NextOffset();
    EXPECT_EQ(r.offset, 2000);
    EXPECT_EQ(r.len, 1000);

    r = o->NextOffset();
    EXPECT_EQ(r.offset, 3000);
    EXPECT_EQ(r.len, 1000);

    r = o->NextOffset();
    EXPECT_EQ(r.offset, 4000);
    EXPECT_EQ(r.len, 96);
    delete o;
}

TEST(OffsetMgr, reset) {
    OffsetMgr *o = new OffsetMgr(1024, 100);

    o->NextOffset();
    o->NextOffset();
    o->Reset(333);
    Range r = o->NextOffset();

    EXPECT_EQ(r.offset, 333);
    EXPECT_EQ(r.len, 100);
    delete o;
}

#define HOSTSTR "localhost"
#define BUCKETSTR "s3"
TEST(ListBucket, fake) {
    ListBucketResult *r = ListBucket_FakeHTTP(HOSTSTR, BUCKETSTR);
    char urlbuf[256];
    EXPECT_NE(r, (void *)NULL);
    if (!r) return;
    vector<BucketContent *>::iterator i;
    for (i = r->contents.begin(); i != r->contents.end(); i++) {
        BucketContent *p = *i;
        sprintf(urlbuf, "http://%s/%s/%s", HOSTSTR, BUCKETSTR, p->Key());
        printf("%s, %d\n", urlbuf, p->Size());
        // printdata(urlbuf, p->Size(), &cred);
    }
    delete r;
}

void DownloaderTest(const char *url, uint64_t file_size, const char *md5_str,
                    uint8_t thread_num, uint64_t chunk_size,
                    uint64_t buffer_size) {
    InitLog();
    uint64_t buf_len = buffer_size;
    char *buf = (char *)malloc(buffer_size);
    Downloader *d = new Downloader(thread_num);
    MD5Calc m;

    ASSERT_NE((void *)NULL, buf);

    ASSERT_TRUE(d->init(url, file_size, chunk_size, NULL));

    while (1) {
        ASSERT_TRUE(d->get(buf, buf_len));
        if (buf_len == 0) {
            break;
        }
        m.Update(buf, buf_len);
        buf_len = buffer_size;
    }

    EXPECT_STREQ(md5_str, m.Get());
    delete d;
    free(buf);
}

TEST(Downloader, divisible) {
    DownloaderTest(
        "http://localhost/metro.pivotal.io/debian-8.2.0-amd64-netinst.iso",
        258998272, "762eb3dfc22f85faf659001ebf270b4f", 8, 4 * 1024 * 1024,
        4 * 1024);
}

TEST(Downloader, equal) {
    DownloaderTest(
        "http://localhost/metro.pivotal.io/debian-8.2.0-amd64-netinst.iso",
        258998272, "762eb3dfc22f85faf659001ebf270b4f", 8, 4 * 1024 * 1024,
        4 * 1024 * 1024);
}
{'fraction_non_alphanumeric': 0.16867002714230322, 'fraction_numerical': 0.07289647150058162, 'mean_word_length': 2.635658914728682, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2068599', 'n_tokens_mistral': 2249, 'n_tokens_neox': 1975, 'n_words': 394}