<commit_before>"""
    Unit tests for the ascii (n-column) reader
"""
import warnings
warnings.simplefilter("ignore")

import unittest
from DataLoader.loader import  Loader
 
import os.path

class abs_reader(unittest.TestCase):
    
    def setUp(self):
        self.loader = Loader()
        
    def test_checkdata(self):
        """
            Test .ABS file loaded as ascii
        """
        f = self.loader.load("ascii_test_1.txt")
        # The length of the data is 10
        self.assertEqual(len(f.x), 10)
        
    def test_truncated_1(self):
        """
            Test an ascii file with header and a 
            comment line in the middle of the data section.
            The business rule says that we should stop
            reading at the first comment once the data
            section has started (and treat the comment
            as though it were the start of a footer).
        """
        # Test .ABS file loaded as ascii
        f = self.loader.load("ascii_test_2.txt")
        # The length of the data is 10
        self.assertEqual(len(f.x), 5)
        
    def test_truncated_2(self):
        """
            Test a 6-col ascii file with header and a 
            line with only 2 columns in the middle of the data section.
            The business rule says that we should stop
            reading at the first inconsitent line.
        """
        # Test .ABS file loaded as ascii
        f = self.loader.load("ascii_test_3.txt")
        # The length of the data is 10
        self.assertEqual(len(f.x), 5)
        
        
        
if __name__ == '__main__':
    unittest.main()
   <commit_msg>added one more test data<commit_after>"""
    Unit tests for the ascii (n-column) reader
"""
import warnings
warnings.simplefilter("ignore")

import unittest
from DataLoader.loader import  Loader
 
import os.path

class abs_reader(unittest.TestCase):
    
    def setUp(self):
        self.loader = Loader()
        
    def test_checkdata(self):
        """
            Test .ABS file loaded as ascii
        """
        f = self.loader.load("ascii_test_1.txt")
        # The length of the data is 10
        self.assertEqual(len(f.x), 10)
        
    def test_truncated_1(self):
        """
            Test an ascii file with header and a 
            comment line in the middle of the data section.
            The business rule says that we should stop
            reading at the first comment once the data
            section has started (and treat the comment
            as though it were the start of a footer).
        """
        # Test .ABS file loaded as ascii
        f = self.loader.load("ascii_test_2.txt")
        # The length of the data is 10
        self.assertEqual(len(f.x), 5)
        
    def test_truncated_2(self):
        """
            Test a 6-col ascii file with header and a 
            line with only 2 columns in the middle of the data section.
            The business rule says that we should stop
            reading at the first inconsitent line.
        """
        # Test .ABS file loaded as ascii
        f = self.loader.load("ascii_test_3.txt")
        # The length of the data is 10
        self.assertEqual(len(f.x), 5)
        
    def test_truncated_3(self):
        """
            Test a 6-col ascii file with complex header and 
            many lines with 2 or 2 columns in the middle of the data section.
            The business rule says that we should stop
            reading at the last line of header.
        """
        # Test .ABS file loaded as ascii
        f = self.loader.load("ascii_test_4.abs")
        # The length of the data is 10
        self.assertEqual(len(f.x), 5)
        
        
if __name__ == '__main__':
    unittest.main()
   
{'fraction_non_alphanumeric': 0.07252007252007252, 'fraction_numerical': 0.010878010878010878, 'mean_word_length': 1.6307901907356948, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4986093', 'n_tokens_mistral': 1164, 'n_tokens_neox': 1040, 'n_words': 401}