<commit_before>import os
import subprocess
import tempfile
import shutil


class BaseParserTestCase(object):
    """This BaseParserTestCase object is used to collect a bunch of
    standardized tests that should be run for every BaseParser.
    """

    # 'txt', for example. this is mandatory and potentially the only thing that
    #  has to be specified to subclass this unittest
    extension = ''

    # User can specify a particular filename root (without
    # extension!), but these have good defaults that are specified by
    # the @property methods below
    raw_text_filename_root = ''
    standardized_text_filename_root = ''
    unicode_text_filename_root = ''

    def __init__(self, *args, **kwargs):
        super(BaseParserTestCase, self).__init__(*args, **kwargs)
        if self.extension == '':
            raise NotImplementedError(
                'need to specify `extension` class attribute on test case'
            )

    def get_filename(self, filename_root, default_filename_root):
        if filename_root:
            filename = os.path.join(
                os.path.dirname(os.path.abspath(__file__)),
                self.extension,
                filename_root + '.' + self.extension,
            )
            if not os.path.exists(filename):
                raise Exception((
                    'expected filename "%(filename)s" to exist for testing '
                    'purposes but it doesnt'
                ) % locals())
            return filename
        return self.get_filename(default_filename_root, default_filename_root)
        
    @property
    def raw_text_filename(self):
        return self.get_filename(self.raw_text_filename_root,
                                 "raw_text")

    @property
    def standardized_text_filename(self):
        return self.get_filename(self.standardized_text_filename_root,
                                 "standardized_text")

    @property
    def unicode_text_filename(self):
        return self.get_filename(self.unicode_text_filename_root,
                                 "unicode_text")

    def test_raw_text(self):
        """This tests an arbitrarily specified raw text file that is
        contributed by the user
        """
        self.compare_cli_output(self.raw_text_filename)
        self.compare_python_output(self.raw_text_filename)

    # def test_standardized_text(self):
    #     """This tests a standardized text output that should be identical
    #     across all filetypes (e.g., #49)
    #     """
    #     self.compare_cli_output(self.standardized_text_filename)
    #     self.compare_python_output(self.standardized_text_filename)

    # def test_unicode_text(self):
    #     """This tests a unicode text output
    #     """
    #     self.compare_cli_output(self.unicode_text_filename)
    #     self.compare_python_output(self.unicode_text_filename)

    def get_expected_filename(self, filename):
        basename, extension = os.path.splitext(filename)
        return basename + '.txt'
<commit_msg>broke out tests for cli and python separately<commit_after>import os
import subprocess
import tempfile
import shutil


class BaseParserTestCase(object):
    """This BaseParserTestCase object is used to collect a bunch of
    standardized tests that should be run for every BaseParser.
    """

    # 'txt', for example. this is mandatory and potentially the only thing that
    #  has to be specified to subclass this unittest
    extension = ''

    # User can specify a particular filename root (without
    # extension!), but these have good defaults that are specified by
    # the @property methods below
    raw_text_filename_root = ''
    standardized_text_filename_root = ''
    unicode_text_filename_root = ''

    def __init__(self, *args, **kwargs):
        super(BaseParserTestCase, self).__init__(*args, **kwargs)
        if self.extension == '':
            raise NotImplementedError(
                'need to specify `extension` class attribute on test case'
            )

    def get_filename(self, filename_root, default_filename_root):
        if filename_root:
            filename = os.path.join(
                os.path.dirname(os.path.abspath(__file__)),
                self.extension,
                filename_root + '.' + self.extension,
            )
            if not os.path.exists(filename):
                raise Exception((
                    'expected filename "%(filename)s" to exist for testing '
                    'purposes but it doesnt'
                ) % locals())
            return filename
        return self.get_filename(default_filename_root, default_filename_root)
        
    @property
    def raw_text_filename(self):
        return self.get_filename(self.raw_text_filename_root,
                                 "raw_text")

    @property
    def standardized_text_filename(self):
        return self.get_filename(self.standardized_text_filename_root,
                                 "standardized_text")

    @property
    def unicode_text_filename(self):
        return self.get_filename(self.unicode_text_filename_root,
                                 "unicode_text")

    def test_raw_text_cli(self):
        """Make sure raw text matches from the command line"""
        self.compare_cli_output(self.raw_text_filename)

    def test_raw_text_python(self):
        """Make sure raw text matches from python"""
        self.compare_python_output(self.raw_text_filename)

    # def test_standardized_text_cli(self):
    #     """Make sure standardized text matches from the command line"""
    #     self.compare_cli_output(self.standardized_text_filename)

    # def test_standardized_text_python(self):
    #     """Make sure standardized text matches from python"""
    #     self.compare_python_output(self.standardized_text_filename)

    # def test_unicode_text_cli(self):
    #     """Make sure unicode text matches from the command line"""
    #     self.compare_cli_output(self.unicode_text_filename)

    # def test_unicode_text_python(self):
    #     """Make sure unicode text matches from python"""
    #     self.compare_python_output(self.unicode_text_filename)

    def get_expected_filename(self, filename):
        basename, extension = os.path.splitext(filename)
        return basename + '.txt'

{'fraction_non_alphanumeric': 0.07904761904761905, 'fraction_numerical': 0.00031746031746031746, 'mean_word_length': 2.488925802879291, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7078529', 'n_tokens_mistral': 1702, 'n_tokens_neox': 1631, 'n_words': 432}