<commit_before>import os

import Test
import TestRunner
import Util

class GoogleTest(object):
    def __init__(self, test_sub_dir, test_suffix):
        self.test_sub_dir = str(test_sub_dir)
        self.test_suffix = str(test_suffix)

    def getGTestTests(self, path, litConfig, localConfig):
        """getGTestTests(path) - [name]

        Return the tests available in gtest executable.

        Args:
          path: String path to a gtest executable
          litConfig: LitConfig instance
          localConfig: TestingConfig instance"""

        try:
            lines = Util.capture([path, '--gtest_list_tests'],
                                 env=localConfig.environment).split('\n')
        except:
            litConfig.error("unable to discover google-tests in %r" % path)
            raise StopIteration

        nested_tests = []
        for ln in lines:
            if not ln.strip():
                continue

            prefix = ''
            index = 0
            while ln[index*2:index*2+2] == '  ':
                index += 1
            while len(nested_tests) > index:
                nested_tests.pop()
            
            ln = ln[index*2:]
            if ln.endswith('.'):
                nested_tests.append(ln)
            else:
                yield ''.join(nested_tests) + ln

    def getTestsInDirectory(self, testSuite, path_in_suite,
                            litConfig, localConfig):
        source_path = testSuite.getSourcePath(path_in_suite)
        for filename in os.listdir(source_path):
            # Check for the one subdirectory (build directory) tests will be in.
            if filename != self.test_sub_dir:
                continue

            filepath = os.path.join(source_path, filename)
            for subfilename in os.listdir(filepath):
                if subfilename.endswith(self.test_suffix):
                    execpath = os.path.join(filepath, subfilename)

                    # Discover the tests in this executable.
                    for name in self.getGTestTests(execpath, litConfig,
                                                   localConfig):
                        testPath = path_in_suite + (filename, subfilename, name)
                        yield Test.Test(testSuite, testPath, localConfig)

    def execute(self, test, litConfig):
        testPath,testName = os.path.split(test.getSourcePath())
        while not os.path.exists(testPath):
            # Handle GTest parametrized and typed tests, whose name includes
            # some '/'s.
            testPath, namePrefix = os.path.split(testPath)
            testName = os.path.join(namePrefix, testName)

        cmd = [testPath, '--gtest_filter=' + testName]
        out, err, exitCode = TestRunner.executeCommand(
            cmd, env=test.config.environment)
            
        if not exitCode:
            return Test.PASS,''

        return Test.FAIL, out + err

###

class FileBasedTest(object):
    def getTestsInDirectory(self, testSuite, path_in_suite,
                            litConfig, localConfig):
        source_path = testSuite.getSourcePath(path_in_suite)
        for filename in os.listdir(source_path):
            filepath = os.path.join(source_path, filename)
            if not os.path.isdir(filepath):
                base,ext = os.path.splitext(filename)
                if ext in localConfig.suffixes:
                    yield Test.Test(testSuite, path_in_suite + (filename,),
                                    localConfig)

class ShTest(FileBasedTest):
    def __init__(self, execute_external = False):
        self.execute_external = execute_external

    def execute(self, test, litConfig):
        return TestRunner.executeShTest(test, litConfig,
                                        self.execute_external)

class TclTest(FileBasedTest):
    def execute(self, test, litConfig):
        return TestRunner.executeTclTest(test, litConfig)

###

import re
import tempfile

class OneCommandPerFileTest:
    # FIXME: Refactor into generic test for running some command on a directory
    # of inputs.

    def __init__(self, command, dir, recursive=False,
                 pattern=".*", useTempInput=False):
        if isinstance(command, str):
            self.command = [command]
        else:
            self.command = list(command)
        self.dir = str(dir)
        self.recursive = bool(recursive)
        self.pattern = re.compile(pattern)
        self.useTempInput = useTempInput

    def getTestsInDirectory(self, testSuite, path_in_suite,
                            litConfig, localConfig):
        for dirname,subdirs,filenames in os.walk(self.dir):
            if not self.recursive:
                subdirs[:] = []

            subdirs[:] = [d for d in subdirs
                          if (d != '.svn' and
                              d not in localConfig.excludes)]

            for filename in filenames:
                if (not self.pattern.match(filename) or
                    filename in localConfig.excludes):
                    continue

                path = os.path.join(dirname,filename)
                suffix = path[len(self.dir):]
                if suffix.startswith(os.sep):
                    suffix = suffix[1:]
                test = Test.Test(testSuite,
                                 path_in_suite + tuple(suffix.split(os.sep)),
                                 localConfig)
                # FIXME: Hack?
                test.source_path = path
                yield test

    def createTempInput(self, tmp, test):
        abstract

    def execute(self, test, litConfig):
        if test.config.unsupported:
            return (Test.UNSUPPORTED, 'Test is unsupported')

        cmd = list(self.command)

        # If using temp input, create a temporary file and hand it to the
        # subclass.
        if self.useTempInput:
            tmp = tempfile.NamedTemporaryFile(suffix='.cpp')
            self.createTempInput(tmp, test)
            tmp.flush()<commit_msg>lit: Ignore dot files when scanning for tests (e.g., editor temprary files, etc.)<commit_after>import os

import Test
import TestRunner
import Util

class GoogleTest(object):
    def __init__(self, test_sub_dir, test_suffix):
        self.test_sub_dir = str(test_sub_dir)
        self.test_suffix = str(test_suffix)

    def getGTestTests(self, path, litConfig, localConfig):
        """getGTestTests(path) - [name]

        Return the tests available in gtest executable.

        Args:
          path: String path to a gtest executable
          litConfig: LitConfig instance
          localConfig: TestingConfig instance"""

        try:
            lines = Util.capture([path, '--gtest_list_tests'],
                                 env=localConfig.environment).split('\n')
        except:
            litConfig.error("unable to discover google-tests in %r" % path)
            raise StopIteration

        nested_tests = []
        for ln in lines:
            if not ln.strip():
                continue

            prefix = ''
            index = 0
            while ln[index*2:index*2+2] == '  ':
                index += 1
            while len(nested_tests) > index:
                nested_tests.pop()
            
            ln = ln[index*2:]
            if ln.endswith('.'):
                nested_tests.append(ln)
            else:
                yield ''.join(nested_tests) + ln

    def getTestsInDirectory(self, testSuite, path_in_suite,
                            litConfig, localConfig):
        source_path = testSuite.getSourcePath(path_in_suite)
        for filename in os.listdir(source_path):
            # Check for the one subdirectory (build directory) tests will be in.
            if filename != self.test_sub_dir:
                continue

            filepath = os.path.join(source_path, filename)
            for subfilename in os.listdir(filepath):
                if subfilename.endswith(self.test_suffix):
                    execpath = os.path.join(filepath, subfilename)

                    # Discover the tests in this executable.
                    for name in self.getGTestTests(execpath, litConfig,
                                                   localConfig):
                        testPath = path_in_suite + (filename, subfilename, name)
                        yield Test.Test(testSuite, testPath, localConfig)

    def execute(self, test, litConfig):
        testPath,testName = os.path.split(test.getSourcePath())
        while not os.path.exists(testPath):
            # Handle GTest parametrized and typed tests, whose name includes
            # some '/'s.
            testPath, namePrefix = os.path.split(testPath)
            testName = os.path.join(namePrefix, testName)

        cmd = [testPath, '--gtest_filter=' + testName]
        out, err, exitCode = TestRunner.executeCommand(
            cmd, env=test.config.environment)
            
        if not exitCode:
            return Test.PASS,''

        return Test.FAIL, out + err

###

class FileBasedTest(object):
    def getTestsInDirectory(self, testSuite, path_in_suite,
                            litConfig, localConfig):
        source_path = testSuite.getSourcePath(path_in_suite)
        for filename in os.listdir(source_path):
            # Ignore dot files.
            if filename.startswith('.'):
                continue

            filepath = os.path.join(source_path, filename)
            if not os.path.isdir(filepath):
                base,ext = os.path.splitext(filename)
                if ext in localConfig.suffixes:
                    yield Test.Test(testSuite, path_in_suite + (filename,),
                                    localConfig)

class ShTest(FileBasedTest):
    def __init__(self, execute_external = False):
        self.execute_external = execute_external

    def execute(self, test, litConfig):
        return TestRunner.executeShTest(test, litConfig,
                                        self.execute_external)

class TclTest(FileBasedTest):
    def execute(self, test, litConfig):
        return TestRunner.executeTclTest(test, litConfig)

###

import re
import tempfile

class OneCommandPerFileTest:
    # FIXME: Refactor into generic test for running some command on a directory
    # of inputs.

    def __init__(self, command, dir, recursive=False,
                 pattern=".*", useTempInput=False):
        if isinstance(command, str):
            self.command = [command]
        else:
            self.command = list(command)
        self.dir = str(dir)
        self.recursive = bool(recursive)
        self.pattern = re.compile(pattern)
        self.useTempInput = useTempInput

    def getTestsInDirectory(self, testSuite, path_in_suite,
                            litConfig, localConfig):
        for dirname,subdirs,filenames in os.walk(self.dir):
            if not self.recursive:
                subdirs[:] = []

            subdirs[:] = [d for d in subdirs
                          if (d != '.svn' and
                              d not in localConfig.excludes)]

            for filename in filenames:
                if (filename.startswith('.') or
                    not self.pattern.match(filename) or
                    filename in localConfig.excludes):
                    continue

                path = os.path.join(dirname,filename)
                suffix = path[len(self.dir):]
                if suffix.startswith(os.sep):
                    suffix = suffix[1:]
                test = Test.Test(testSuite,
                                 path_in_suite + tuple(suffix.split(os.sep)),
                                 localConfig)
                # FIXME: Hack?
                test.source_path = path
                yield test

    def createTempInput(self, tmp, test):
        abstract

    def execute(self, test, litConfig):
        if test.config.unsupported:
            return (Test.UNSUPPORTED, 'Test is unsupported')

        cmd = list(self.command)

        # If using temp input, create a temporary file and hand it to the
        # subclass.
        if self.useTempInput:
            tmp = tempfile.NamedTemporaryFile(suffix='.cpp')
            self.createTempInput(tmp, test)
            tmp.flush()
{'fraction_non_alphanumeric': 0.08639751044140528, 'fraction_numerical': 0.0011465072475636722, 'mean_word_length': 1.6916464624201013, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6060702', 'n_tokens_mistral': 3313, 'n_tokens_neox': 3139, 'n_words': 803}