<commit_before># Copyright (c) 2013, <NAME> (<EMAIL>)
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.  THIS SOFTWARE IS
# PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
# NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import pyeclib
from pyeclib.ec_iface import ECDriver
import unittest
import random
import string
import sys
import os


class TestNullDriver(unittest.TestCase):

    def setUp(self):
        self.null_driver = ECDriver("pyeclib.core.ECNullDriver", k=8, m=2)

    def tearDown(self):
        pass

    def test_null_driver(self):
        self.null_driver.encode('')
        self.null_driver.decode([])


class TestStripeDriver(unittest.TestCase):

    def setUp(self):
        self.stripe_driver = ECDriver(
            "pyeclib.core.ECStripingDriver",
            k=8,
            m=0)

    def tearDown(self):
        pass


class TestPyECLibDriver(unittest.TestCase):

    def __init__(self, *args):
        self.file_sizes = ["100-K"]
        self.num_iterations = 100

        unittest.TestCase.__init__(self, *args)

    def setUp(self):

        if not os.path.isdir("test_files"):
            os.mkdir("./test_files")

        for size_str in self.file_sizes:

            size_desc = size_str.split("-")

            size = int(size_desc[0])

            if (size_desc[1] == 'M'):
                size *= 1000000
            elif (size_desc[1] == 'K'):
                size *= 1000

            buf = ''.join(random.choice(string.letters)
                          for i in range(size))

            filename = "test_file.%s" % size_str
            with open("test_files/%s" % filename, "wb") as fp:
                fp.write(buf)

    def tearDown(self):
        for size_str in self.file_sizes:
            filename = "test_files/test_file.%s" % size_str
            os.unlink(filename)
        os.rmdir("./test_files")

    def test_small_encode(self):
        pyeclib_drivers = []
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=2, ec_type="rs_vand"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=11, m=2, ec_type="rs_vand"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=10, m=2, ec_type="rs_vand"))

        encode_strs = ["a", "hello", "hellohyhi", "yo"]

        for pyeclib_driver in pyeclib_drivers:
            for encode_str in encode_strs:
                encoded_fragments = pyeclib_driver.encode(encode_str)
                decoded_str = pyeclib_driver.decode(encoded_fragments)

                self.assertTrue(decoded_str == encode_str)

    def test_verify_fragment_algsig_chksum_fail(self):
        pyeclib_drivers = []
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=2, ec_type="rs_vand",
                chksum_type="algsig"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=3, ec_type="rs_vand",
                chksum_type="algsig"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=6, ec_type="flat_xor_4",
                chksum_type="algsig"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=10, m=5, ec_type="flat_xor_4",
                chksum_type="algsig"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=10, m=5, ec_type="flat_xor_3",
                chksum_type="algsig"))

        filesize = 1024 * 1024 * 3

        file_str = ''.join(random.choice(string.letters)
                           for i in range(filesize))
        fragment_to_corrupt = random.randint(0, 12)

        for pyeclib_driver in pyeclib_drivers:
            fragments = pyeclib_driver.encode(file_str)

            fragment_metadata_list = []

            i = 0
            for fragment in fragments:
                if i == fragment_to_corrupt:
                    corrupted_fragment = fragment[
                        :100] + chr(ord(fragment[100]) + 1) + fragment[101:]
                    fragment_metadata_list.append(
                        pyeclib_driver.get_metadata(corrupted_fragment))
                else:
                    fragment_metadata_list.append(
                        pyeclib_driver.get_metadata(fragment))
                i += 1

            self.assertTrue(
                pyeclib_driver.verify_stripe_metadata(fragment_metadata_list)
                != -1)

    def test_verify_fragment_inline_succeed(self):
        pyeclib_drivers = []
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=2, ec_type="rs_vand",
                chksum_type="algsig"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=3, ec_type="rs_vand",
                chksum_type="algsig"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=6, ec_type="flat_xor_4",
                chksum_type="algsig"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=10, m=5, ec_type="flat_xor_4",
                chksum_type="algsig"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=10, m=5, ec_type="flat_xor_3",
                chksum_type="algsig"))

        filesize = 1024 * 1024 * 3

        file_str = ''.join(random.choice(string.letters)
                           for i in range(filesize))

        for pyeclib_driver in pyeclib_drivers:
            fragments = pyeclib_driver.encode(file_str)

            fragment_metadata_list = []

            for fragment in fragments:
                fragment_metadata_list.append(
                    pyeclib_driver.get_metadata(fragment))

            self.assertTrue(
                pyeclib_driver.verify_stripe_metadata(fragment_metadata_list)
                == -1)

    def test_verify_fragment_inline_chksum_fail(self):
        pyeclib_drivers = []
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=2, ec_type="rs_vand",
                chksum_type="inline"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=3, ec_type="rs_vand",
                chksum_type="inline"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=4, ec_type="rs_vand",
                chksum_type="inline"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=2, ec_type="rs_cauchy_orig",
                chksum_type="inline"))

        filesize = 1024 * 1024 * 3

        file_str = ''.join(random.choice(string.letters)
                           for i in range(filesize))
        fragment_to_corrupt = random.randint(0, 12)

        for pyeclib_driver in pyeclib_drivers:
            fragments = pyeclib_driver.encode(file_str)

            fragment_metadata_list = []

            i = 0
            for fragment in fragments:
                if i == fragment_to_corrupt:
                    corrupted_fragment = fragment[
                        :100] + chr(ord(fragment[100]) + 1) + fragment[101:]
                    fragment_metadata_list.append(
                        pyeclib_driver.get_metadata(corrupted_fragment))
                else:
                    fragment_metadata_list.append(
                        pyeclib_driver.get_metadata(fragment))
                i += 1

            self.assertTrue(pyeclib_driver.verify_stripe_metadata(
                fragment_metadata_list) == fragment_to_corrupt)

    def test_verify_fragment_inline_chksum_succeed(self):
        pyeclib_drivers = []
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=2, ec_type="rs_vand",
                chksum_type="inline"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=3, ec_type="rs_vand",
                chksum_type="inline"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=4, ec_type="rs_vand",
                chksum_type="inline"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=2, ec_type="rs_cauchy_orig",
                chksum_type="inline"))

        filesize = 1024 * 1024 * 3

        file_str = ''.join(random.choice(string.letters)
                           for i in range(filesize))

        for pyeclib_driver in pyeclib_drivers:
            fragments = pyeclib_driver.encode(file_str)

            fragment_metadata_list = []

            for fragment in fragments:
                fragment_metadata_list.append(<commit_msg>Use str.ascii_letters for Python3 compatibility (take2)<commit_after># Copyright (c) 2013, <NAME> (<EMAIL>)
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.  THIS SOFTWARE IS
# PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
# NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import pyeclib
from pyeclib.ec_iface import ECDriver
import unittest
import random
import string
import sys
import os

from string import ascii_letters

class TestNullDriver(unittest.TestCase):

    def setUp(self):
        self.null_driver = ECDriver("pyeclib.core.ECNullDriver", k=8, m=2)

    def tearDown(self):
        pass

    def test_null_driver(self):
        self.null_driver.encode('')
        self.null_driver.decode([])


class TestStripeDriver(unittest.TestCase):

    def setUp(self):
        self.stripe_driver = ECDriver(
            "pyeclib.core.ECStripingDriver",
            k=8,
            m=0)

    def tearDown(self):
        pass


class TestPyECLibDriver(unittest.TestCase):

    def __init__(self, *args):
        self.file_sizes = ["100-K"]
        self.num_iterations = 100

        unittest.TestCase.__init__(self, *args)

    def setUp(self):

        if not os.path.isdir("test_files"):
            os.mkdir("./test_files")

        for size_str in self.file_sizes:

            size_desc = size_str.split("-")

            size = int(size_desc[0])

            if (size_desc[1] == 'M'):
                size *= 1000000
            elif (size_desc[1] == 'K'):
                size *= 1000

            buf = ''.join(random.choice(string.ascii_letters)
                          for i in range(size))

            filename = "test_file.%s" % size_str
            with open("test_files/%s" % filename, "wb") as fp:
                fp.write(buf)

    def tearDown(self):
        for size_str in self.file_sizes:
            filename = "test_files/test_file.%s" % size_str
            os.unlink(filename)
        os.rmdir("./test_files")

    def test_small_encode(self):
        pyeclib_drivers = []
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=2, ec_type="rs_vand"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=11, m=2, ec_type="rs_vand"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=10, m=2, ec_type="rs_vand"))

        encode_strs = ["a", "hello", "hellohyhi", "yo"]

        for pyeclib_driver in pyeclib_drivers:
            for encode_str in encode_strs:
                encoded_fragments = pyeclib_driver.encode(encode_str)
                decoded_str = pyeclib_driver.decode(encoded_fragments)

                self.assertTrue(decoded_str == encode_str)

    def test_verify_fragment_algsig_chksum_fail(self):
        pyeclib_drivers = []
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=2, ec_type="rs_vand",
                chksum_type="algsig"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=3, ec_type="rs_vand",
                chksum_type="algsig"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=6, ec_type="flat_xor_4",
                chksum_type="algsig"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=10, m=5, ec_type="flat_xor_4",
                chksum_type="algsig"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=10, m=5, ec_type="flat_xor_3",
                chksum_type="algsig"))

        filesize = 1024 * 1024 * 3

        file_str = ''.join(random.choice(string.ascii_letters)
                           for i in range(filesize))
        fragment_to_corrupt = random.randint(0, 12)

        for pyeclib_driver in pyeclib_drivers:
            fragments = pyeclib_driver.encode(file_str)

            fragment_metadata_list = []

            i = 0
            for fragment in fragments:
                if i == fragment_to_corrupt:
                    corrupted_fragment = fragment[
                        :100] + chr(ord(fragment[100]) + 1) + fragment[101:]
                    fragment_metadata_list.append(
                        pyeclib_driver.get_metadata(corrupted_fragment))
                else:
                    fragment_metadata_list.append(
                        pyeclib_driver.get_metadata(fragment))
                i += 1

            self.assertTrue(
                pyeclib_driver.verify_stripe_metadata(fragment_metadata_list)
                != -1)

    def test_verify_fragment_inline_succeed(self):
        pyeclib_drivers = []
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=2, ec_type="rs_vand",
                chksum_type="algsig"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=3, ec_type="rs_vand",
                chksum_type="algsig"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=6, ec_type="flat_xor_4",
                chksum_type="algsig"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=10, m=5, ec_type="flat_xor_4",
                chksum_type="algsig"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=10, m=5, ec_type="flat_xor_3",
                chksum_type="algsig"))

        filesize = 1024 * 1024 * 3

        file_str = ''.join(random.choice(string.ascii_letters)
                           for i in range(filesize))

        for pyeclib_driver in pyeclib_drivers:
            fragments = pyeclib_driver.encode(file_str)

            fragment_metadata_list = []

            for fragment in fragments:
                fragment_metadata_list.append(
                    pyeclib_driver.get_metadata(fragment))

            self.assertTrue(
                pyeclib_driver.verify_stripe_metadata(fragment_metadata_list)
                == -1)

    def test_verify_fragment_inline_chksum_fail(self):
        pyeclib_drivers = []
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=2, ec_type="rs_vand",
                chksum_type="inline"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=3, ec_type="rs_vand",
                chksum_type="inline"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=4, ec_type="rs_vand",
                chksum_type="inline"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=2, ec_type="rs_cauchy_orig",
                chksum_type="inline"))

        filesize = 1024 * 1024 * 3

        file_str = ''.join(random.choice(string.ascii_letters)
                           for i in range(filesize))
        fragment_to_corrupt = random.randint(0, 12)

        for pyeclib_driver in pyeclib_drivers:
            fragments = pyeclib_driver.encode(file_str)

            fragment_metadata_list = []

            i = 0
            for fragment in fragments:
                if i == fragment_to_corrupt:
                    corrupted_fragment = fragment[
                        :100] + chr(ord(fragment[100]) + 1) + fragment[101:]
                    fragment_metadata_list.append(
                        pyeclib_driver.get_metadata(corrupted_fragment))
                else:
                    fragment_metadata_list.append(
                        pyeclib_driver.get_metadata(fragment))
                i += 1

            self.assertTrue(pyeclib_driver.verify_stripe_metadata(
                fragment_metadata_list) == fragment_to_corrupt)

    def test_verify_fragment_inline_chksum_succeed(self):
        pyeclib_drivers = []
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=2, ec_type="rs_vand",
                chksum_type="inline"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=3, ec_type="rs_vand",
                chksum_type="inline"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=4, ec_type="rs_vand",
                chksum_type="inline"))
        pyeclib_drivers.append(
            ECDriver(
                "pyeclib.core.ECPyECLibDriver",
                k=12, m=2, ec_type="rs_cauchy_orig",
                chksum_type="inline"))

        filesize = 1024 * 1024 * 3

        file_str = ''.join(random.choice(string.ascii_letters)
                           for i in range(filesize))

        for pyeclib_driver in pyeclib_drivers:
            fragments = pyeclib_driver.encode(file_str)

            fragment_metadata_list = []

            for fragment in fragments:
                fragment_metadata_list.append(
{'fraction_non_alphanumeric': 0.09084883775978791, 'fraction_numerical': 0.015717464375325476, 'mean_word_length': 1.841156691324815, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6620266', 'n_tokens_mistral': 7085, 'n_tokens_neox': 6230, 'n_words': 1234}