<commit_before>/*
 * Copyright 2015 Intel(r) Corporation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http ://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

#include "test_precomp.hpp"

using namespace vmf;

//Bad compressor that increases the size of input data
class BloatingCompressor : public vmf::Compressor
{
public:
    static const int factor = 5;

    virtual void compress(const vmf::vmf_string& input, vmf::vmf_rawbuffer& output)
    {
        output.clear();
        for(auto it = input.begin(); it != input.end(); ++it)
        {
            for(int i = 0; i < factor; i++)
                output.push_back(*it);
        }
    }

    virtual void decompress(const vmf::vmf_rawbuffer& input, vmf::vmf_string &output)
    {
        output.clear();
        for(auto it = input.begin(); it != input.end();)
        {
            char c = *it++;
            for(int i = 1; i < factor; i++)
            {
                char cc = *it++;
                if(cc != c)
                {
                    throw std::runtime_error("Incorrect input to bloating compressor");
                }
            }
            output.push_back(c);
        }
    }

    std::shared_ptr<Compressor> createNewInstance() const
    {
        return std::shared_ptr<Compressor>(new BloatingCompressor);
    }

    virtual vmf::vmf_string getId()
    {
        return "com.intel.vmf.compressor.test.bloating";
    }
};


class TestCompressor : public ::testing::TestWithParam<std::string>
{
protected:
    void SetUp()
    {
        //register bloating compressor
        std::shared_ptr<Compressor> bloating = std::make_shared<BloatingCompressor>();
        vmf::Compressor::registerNew(bloating);
    }

    std::string generateData(int nChars)
    {
        std::string result;
        srand(0);
        for(int i = 0; i < nChars; i++)
        {
            result.push_back( (char)(rand()%256) );
        }
        return result;
    }

    std::shared_ptr<Compressor> compressor;
};


TEST_P(TestCompressor, LossesOnCompression)
{
    std::string name = GetParam();

    compressor = vmf::Compressor::create(name);
    if(name == "unregistered")
    {
        ASSERT_FALSE((bool)compressor);
    }
    else
    {
        ASSERT_TRUE((bool)compressor);

        int nChars = 0;
        do
        {
            std::string data = generateData(nChars);
            vmf_rawbuffer compressed;
            compressor->compress(data, compressed);
            std::string result;
            compressor->decompress(compressed, result);
            ASSERT_EQ(data, result);
            nChars = nChars ? 2*nChars : 1;
        }
        while(nChars < (1 << 24));
    }
}


TEST_P(TestCompressor, DummyDecompression)
{
    std::string name = GetParam();

    compressor = vmf::Compressor::create(name);

    //we can only expect undefined behaviour
    //on other compressors when decompressing random data
    if(name == "com.intel.vmf.compressor.dummy")
    {
        ASSERT_TRUE((bool)compressor);

        int nChars = 0;
        do
        {
            std::string garbage = generateData(nChars);
            vmf_rawbuffer compressed(garbage.data(), garbage.size());
            std::string result;
            ASSERT_NO_THROW(compressor->decompress(compressed, result));
            ASSERT_EQ(garbage, result);
            nChars = nChars ? 2*nChars : 8;
        }
        while(nChars < (1 << 24));
    }
}


TEST_P(TestCompressor, DecompressionOfEmpty)
{
    std::string name = GetParam();

    compressor = vmf::Compressor::create(name);
    if(name == "unregistered")
    {
        ASSERT_FALSE((bool)compressor);
    }
    else
    {
        ASSERT_TRUE((bool)compressor);

        vmf_rawbuffer compressed;
        std::string result;
        ASSERT_NO_THROW(compressor->decompress(compressed, result));
        ASSERT_TRUE(result.empty());
    }
}


INSTANTIATE_TEST_CASE_P(UnitTest, TestCompressor,
                        ::testing::Values("com.intel.vmf.compressor.dummy",
                                          "com.intel.vmf.compressor.zlib",
                                          "unregistered",
                                          "com.intel.vmf.compressor.test.bloating"));
<commit_msg>try to fix tests<commit_after>/*
 * Copyright 2015 Intel(r) Corporation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http ://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

#include "test_precomp.hpp"

using namespace vmf;

//Bad compressor that increases the size of input data
class BloatingCompressor : public vmf::Compressor
{
public:
    static const int factor = 5;

    virtual void compress(const vmf::vmf_string& input, vmf::vmf_rawbuffer& output)
    {
        output.clear();
        for(auto it = input.begin(); it != input.end(); ++it)
        {
            for(int i = 0; i < factor; i++)
                output.push_back(*it);
        }
    }

    virtual void decompress(const vmf::vmf_rawbuffer& input, vmf::vmf_string &output)
    {
        output.clear();
        for(auto it = input.begin(); it != input.end();)
        {
            char c = *it++;
            for(int i = 1; i < factor; i++)
            {
                char cc = *it++;
                if(cc != c)
                {
                    throw std::runtime_error("Incorrect input to bloating compressor");
                }
            }
            output.push_back(c);
        }
    }

    std::shared_ptr<Compressor> createNewInstance() const
    {
        return std::shared_ptr<Compressor>(new BloatingCompressor);
    }

    virtual vmf::vmf_string getId()
    {
        return "com.intel.vmf.compressor.test.bloating";
    }
};


class TestCompressor : public ::testing::TestWithParam<std::string>
{
protected:
    void SetUp()
    {
        //register bloating compressor
        std::shared_ptr<Compressor> bloating = std::make_shared<BloatingCompressor>();
        vmf::Compressor::registerNew(bloating);
    }

    std::string generateData(int nChars)
    {
        std::string result;
        srand(0);
        for(int i = 0; i < nChars; i++)
        {
            result.push_back( (char)(rand()%256) );
        }
        return result;
    }

    std::shared_ptr<Compressor> compressor;
};


TEST_P(TestCompressor, LossesOnCompression)
{
    std::string name = GetParam();

    compressor = vmf::Compressor::create(name);
    if(name == "unregistered")
    {
        ASSERT_EQ(compressor, nullptr);
    }
    else
    {
        ASSERT_NE(compressor, nullptr);

        int nChars = 0;
        do
        {
            std::string data = generateData(nChars);
            vmf_rawbuffer compressed;
            compressor->compress(data, compressed);
            std::string result;
            compressor->decompress(compressed, result);
            ASSERT_EQ(data, result);
            nChars = nChars ? 2*nChars : 1;
        }
        while(nChars < (1 << 24));
    }
}


TEST_P(TestCompressor, DummyDecompression)
{
    std::string name = GetParam();

    compressor = vmf::Compressor::create(name);

    //we can only expect undefined behaviour
    //on other compressors when decompressing random data
    if(name == "com.intel.vmf.compressor.dummy")
    {
        ASSERT_NE(compressor, nullptr);

        int nChars = 0;
        do
        {
            std::string garbage = generateData(nChars);
            vmf_rawbuffer compressed(garbage.data(), garbage.size());
            std::string result;
            ASSERT_NO_THROW(compressor->decompress(compressed, result));
            ASSERT_EQ(garbage, result);
            nChars = nChars ? 2*nChars : 8;
        }
        while(nChars < (1 << 24));
    }
}


TEST_P(TestCompressor, DecompressionOfEmpty)
{
    std::string name = GetParam();

    compressor = vmf::Compressor::create(name);
    if(name == "unregistered")
    {
        ASSERT_EQ(compressor, nullptr);
    }
    else
    {
        ASSERT_NE(compressor, nullptr);

        vmf_rawbuffer compressed;
        std::string result;
        ASSERT_NO_THROW(compressor->decompress(compressed, result));
        ASSERT_TRUE(result.empty());
    }
}


INSTANTIATE_TEST_CASE_P(UnitTest, TestCompressor,
                        ::testing::Values("com.intel.vmf.compressor.dummy",
                                          "com.intel.vmf.compressor.zlib",
                                          "unregistered",
                                          "com.intel.vmf.compressor.test.bloating"));

{'fraction_non_alphanumeric': 0.11568921232876712, 'fraction_numerical': 0.0059931506849315065, 'mean_word_length': 2.1411764705882352, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3299676', 'n_tokens_mistral': 2830, 'n_tokens_neox': 2661, 'n_words': 634}