<commit_before>from __future__ import absolute_import
import logging
from unittest import TestCase
import os
import traceback
from AppCompatProcessor import main
import tempfile
from shutil import copyfile

# Setup the logger
logger = logging.getLogger()
DB = None


class TestAppLoadMP(TestCase):

    def BuildTestPath(self, folder):
        master_test_folder = os.path.join(os.path.abspath(os.path.join(os.path.join(os.path.dirname(__file__), os.pardir), os.pardir)), "appcompatprocessor-DataSets")
        load_test_path = os.path.join(master_test_folder, folder)

        return load_test_path


    def test_ingestTestSets1(self):
        test_sets = [("StructuredRepo/HX_Grabstuffr/ICE_Exec-Defaults/AmCache-Default", 1, 1, 223),
                     ("StructuredRepo/HX_Grabstuffr/ICE_Exec-Defaults/AppCompat-Default", 3, 3, 632),
                     ("StructuredRepo/HX_Grabstuffr/ICE_Exec-Defaults/ShimShady64b-Default", 3, 3, 479),
                     ("StructuredRepo/HX_Grabstuffr/ICE_Exec-Defaults", 3, 6, 1113)]

        for test in test_sets:
            load_test_path = self.BuildTestPath(test[0])
            # Get temp db name for the test
            tempdb = tempfile.NamedTemporaryFile(suffix='.db', prefix='testCase_ingestTestSets', dir=tempfile.gettempdir())
            tempdb.close()

            try:
                (db_filenameFullPath, db_version, num_hosts, num_instances, num_entries) = main([tempdb.name, "load", load_test_path])
            except Exception as e:
                print traceback.format_exc()
                self.fail(traceback.format_exc())

            # Remove temp db
            os.remove(tempdb.name)

            if num_hosts <> test[1] or num_instances <> test[2] or num_entries <> test[3]:
                print("Test %s failed!" % test[0])
                self.fail("Test %s failed!" % test[0])

            self.assertEquals(num_hosts, test[1], "test failed!")
            self.assertEquals(num_instances, test[2], "test failed!")
            self.assertEquals(num_entries, test[3], "test failed!")


    def test_ingestTestSets2(self):
        test_sets = [("StructuredRepo/HX_Grabstuffr/ICE_32and64bShimShadyDefaults", 2, 2, 308),
                     ("StructuredRepo/HX_Grabstuffr/HXRegistryAPIAudit_AMCacheAquisition", 3, 4, 626),
                     ("StructuredRepo/HX_Grabstuffr/ICE_ACPBundle-Tunned", 3, 7, 1105),
                     ("StructuredRepo/HX_Grabstuffr/ICE_Stacking-Defaults", 2, 2, 147)]

        for test in test_sets:
            load_test_path = self.BuildTestPath(test[0])
            # Get temp db name for the test
            tempdb = tempfile.NamedTemporaryFile(suffix='.db', prefix='testCase_ingestTestSets', dir=tempfile.gettempdir())
            tempdb.close()

            try:
                (db_filenameFullPath, db_version, num_hosts, num_instances, num_entries) = main([tempdb.name, "load", load_test_path])
            except Exception as e:
                print traceback.format_exc()
                self.fail(traceback.format_exc())

            # Remove temp db
            os.remove(tempdb.name)

            if num_hosts <> test[1] or num_instances <> test[2] or num_entries <> test[3]:
                print("Test %s failed!" % test[0])
                self.fail("Test %s failed!" % test[0])

            self.assertEquals(num_hosts, test[1], "test failed!")
            self.assertEquals(num_instances, test[2], "test failed!")<commit_msg>Updated unittest after adding Windows 10 18003 Build 17134.1 test data.<commit_after>from __future__ import absolute_import
import logging
from unittest import TestCase
import os
import traceback
from AppCompatProcessor import main
import tempfile
from shutil import copyfile

# Setup the logger
logger = logging.getLogger()
DB = None


class TestAppLoadMP(TestCase):

    def BuildTestPath(self, folder):
        master_test_folder = os.path.join(os.path.abspath(os.path.join(os.path.join(os.path.dirname(__file__), os.pardir), os.pardir)), "appcompatprocessor-DataSets")
        load_test_path = os.path.join(master_test_folder, folder)

        return load_test_path


    def test_ingestTestSets1(self):
        test_sets = [("StructuredRepo/HX_Grabstuffr/ICE_Exec-Defaults/AmCache-Default", 2, 2, 366),
                     ("StructuredRepo/HX_Grabstuffr/ICE_Exec-Defaults/AppCompat-Default", 3, 3, 632),
                     ("StructuredRepo/HX_Grabstuffr/ICE_Exec-Defaults/ShimShady64b-Default", 3, 3, 479),
                     ("StructuredRepo/HX_Grabstuffr/ICE_Exec-Defaults", 3, 6, 1113)]

        for test in test_sets:
            load_test_path = self.BuildTestPath(test[0])
            # Get temp db name for the test
            tempdb = tempfile.NamedTemporaryFile(suffix='.db', prefix='testCase_ingestTestSets', dir=tempfile.gettempdir())
            tempdb.close()

            try:
                (db_filenameFullPath, db_version, num_hosts, num_instances, num_entries) = main([tempdb.name, "load", load_test_path])
            except Exception as e:
                print traceback.format_exc()
                self.fail(traceback.format_exc())

            # Remove temp db
            os.remove(tempdb.name)

            if num_hosts <> test[1] or num_instances <> test[2] or num_entries <> test[3]:
                print("Test %s failed!" % test[0])
                self.fail("Test %s failed!" % test[0])

            self.assertEquals(num_hosts, test[1], "test failed!")
            self.assertEquals(num_instances, test[2], "test failed!")
            self.assertEquals(num_entries, test[3], "test failed!")


    def test_ingestTestSets2(self):
        test_sets = [("StructuredRepo/HX_Grabstuffr/ICE_32and64bShimShadyDefaults", 2, 2, 308),
                     ("StructuredRepo/HX_Grabstuffr/HXRegistryAPIAudit_AMCacheAquisition", 3, 4, 626),
                     ("StructuredRepo/HX_Grabstuffr/ICE_ACPBundle-Tunned", 3, 7, 1105),
                     ("StructuredRepo/HX_Grabstuffr/ICE_Stacking-Defaults", 3, 3, 368)]

        for test in test_sets:
            load_test_path = self.BuildTestPath(test[0])
            # Get temp db name for the test
            tempdb = tempfile.NamedTemporaryFile(suffix='.db', prefix='testCase_ingestTestSets', dir=tempfile.gettempdir())
            tempdb.close()

            try:
                (db_filenameFullPath, db_version, num_hosts, num_instances, num_entries) = main([tempdb.name, "load", load_test_path])
            except Exception as e:
                print traceback.format_exc()
                self.fail(traceback.format_exc())

            # Remove temp db
            os.remove(tempdb.name)

            if num_hosts <> test[1] or num_instances <> test[2] or num_entries <> test[3]:
                print("Test %s failed!" % test[0])
                self.fail("Test %s failed!" % test[0])

            self.assertEquals(num_hosts, test[1], "test failed!")
            self.assertEquals(num_instances, test[2], "test failed!")
{'fraction_non_alphanumeric': 0.11133428981348636, 'fraction_numerical': 0.02109038737446198, 'mean_word_length': 2.5988642230252967, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4709081', 'n_tokens_mistral': 2312, 'n_tokens_neox': 2108, 'n_words': 437}