<commit_before>import re
import textwrap
import unittest

# FIXME: make this work w/ python3
from StringIO import StringIO

from host_fake import FakeHost
from main import main, VERSION


def default_test_files():
    in_files = {}
    in_files['build.ninja'] = textwrap.dedent("""
        rule cat
            command = cat $in > $out

        build ab : cat a b
        build cd : cat c d
        build abcd : cat ab cd
        """)
    in_files['a'] = 'hello '
    in_files['b'] = 'world\n'
    in_files['c'] = 'how are '
    in_files['d'] = 'you?\n'
    out_files = in_files.copy()
    out_files['ab'] = 'hello world\n'
    out_files['cd'] = 'how are you?\n'
    out_files['abcd'] = 'hello world\nhow are you?\n'

    return in_files, out_files


class UnitTestMixin(object):
    def _files_to_ignore(self):
        # return ['.ninja_deps', '.ninja_log']
        return ['.pyn.db']

    def _host(self):
        return FakeHost()

    def _call(self, host, args):
        host.stdout = StringIO()
        host.stderr = StringIO()
        returncode = main(host, args)
        out = host.stdout.getvalue()
        err = host.stderr.getvalue()
        return returncode, out, err


class CheckMixin(object):
    def _write_files(self, host, files):
        for path, contents in list(files.items()):
            dirname = host.dirname(path)
            if dirname:
                host.maybe_mkdir(dirname)
            host.write(path, contents)

    def _read_files(self, host, tmpdir):
        out_files = {}
        for f in host.files_under(tmpdir):
            out_files[f] = host.read(tmpdir, f)
        return out_files

    def assert_files(self, expected_files, actual_files):
        for k, v in expected_files.items():
            self.assertEqual(expected_files[k], v)
        interesting_files = set(actual_files.keys()).difference(
            self._files_to_ignore())
        self.assertEqual(interesting_files, set(expected_files.keys()))

    def check(self, in_files, expected_out_files=None,
              expected_return_code=0, expected_out=None, expected_err=None,
              args=None):
        host = self._host()
        args = args or []

        try:
            orig_wd = host.getcwd()
            tmpdir = host.mkdtemp()
            host.chdir(tmpdir)
            self._write_files(host, in_files)

            returncode, actual_out, actual_err = self._call(host, args)

            actual_out_files = self._read_files(host, tmpdir)
        finally:
            host.rmtree(tmpdir)
            host.chdir(orig_wd)

        self.assertEqual(returncode, expected_return_code)
        if expected_out is not None:
            # FIXME: add a test that actual checks stderr for matches.
            self.assertEqual(expected_out, actual_out)
        if expected_err is not None:
            self.assertEqual(expected_err, actual_err)
        if expected_out_files:
            self.assert_files(expected_out_files, actual_out_files)


class TestArgs(unittest.TestCase, UnitTestMixin):
    def check(self, argv, returncode, out_regex, err_regex):
        host = self._host()
        host.write('build.ninja', textwrap.dedent("""
            rule cc_binary
                description = cc_binary $out
                command = cc -o $out $in

            build hello : cc_binary hello.c
            """))
        actual_returncode, out, err = self._call(host, argv)
        if returncode is not None:
            self.assertEqual(actual_returncode, returncode)
        self.assertTrue(re.match(out_regex, out, re.MULTILINE),
                        '%s does not match %s' % (out_regex, out))
        self.assertTrue(re.match(err_regex, err, re.MULTILINE),
                        '%s does not match %s' % (err_regex, err))

    def test_bad_arg(self):
        self.check(['--bad-arg'], 2, '', '')

    def test_bad_dir(self):
        self.check(['-C', 'missing_dir'], 2, '', '"missing_dir" not found\n')

    def test_bad_file(self):
        self.check(['-f', 'missing_build.ninja'], 2, '',
                   '"missing_build.ninja" not found\n')

    def test_bad_tool(self):
        self.check(['-t', 'foo'], 2, '', 'unsupported tool "foo"\n')

    def test_list(self):
        self.check(['-t', 'list'], 0, '.+', '')

    def test_chdir(self):
        self.check(['-n', '-C', '.'], 0, '', '.*')

    def test_check(self):
        self.check(['-t', 'check'], 0, '', '.*')

    def test_clean(self):
        self.check(['-n', '-t', 'clean'], 0, '', '.*')

    def test_question(self):
        self.check(['-t', 'question'], 1, '', '.*')

    def test_debug(self):
        self.check(['-d', 'foo'], 2, '', '-d is not supported yet\n')

    def test_dry_run(self):
        self.check(['-n'], 0, '.+', '')

    def test_usage(self):
        self.check(['--help'], 0, 'usage:.+', '')
        self.check(['-h'], 0, 'usage:.+', '')

    def test_version(self):
        self.check(['--version'], 0, VERSION + '\n', '')


class TestBuild(unittest.TestCase, UnitTestMixin, CheckMixin):
    def test_basic(self):
        in_files, out_files = default_test_files()
        host = self._host()
        try:
            orig_wd = host.getcwd()
            tmpdir = host.mkdtemp()
            host.chdir(tmpdir)
            self._write_files(host, in_files)<commit_msg>clean up TestArgs to reuse CheckMixin<commit_after>import re
import textwrap
import unittest

# FIXME: make this work w/ python3
from StringIO import StringIO

from host_fake import FakeHost
from main import main, VERSION


def default_test_files():
    in_files = {}
    in_files['build.ninja'] = textwrap.dedent("""
        rule cat
            command = cat $in > $out

        build ab : cat a b
        build cd : cat c d
        build abcd : cat ab cd
        """)
    in_files['a'] = 'hello '
    in_files['b'] = 'world\n'
    in_files['c'] = 'how are '
    in_files['d'] = 'you?\n'
    out_files = in_files.copy()
    out_files['ab'] = 'hello world\n'
    out_files['cd'] = 'how are you?\n'
    out_files['abcd'] = 'hello world\nhow are you?\n'

    return in_files, out_files


class UnitTestMixin(object):
    def _files_to_ignore(self):
        # return ['.ninja_deps', '.ninja_log']
        return ['.pyn.db']

    def _host(self):
        return FakeHost()

    def _call(self, host, args):
        host.stdout = StringIO()
        host.stderr = StringIO()
        returncode = main(host, args)
        out = host.stdout.getvalue()
        err = host.stderr.getvalue()
        return returncode, out, err


class CheckMixin(object):
    def _write_files(self, host, files):
        for path, contents in list(files.items()):
            dirname = host.dirname(path)
            if dirname:
                host.maybe_mkdir(dirname)
            host.write(path, contents)

    def _read_files(self, host, tmpdir):
        out_files = {}
        for f in host.files_under(tmpdir):
            out_files[f] = host.read(tmpdir, f)
        return out_files

    def assert_files(self, expected_files, actual_files):
        for k, v in expected_files.items():
            self.assertEqual(expected_files[k], v)
        interesting_files = set(actual_files.keys()).difference(
            self._files_to_ignore())
        self.assertEqual(interesting_files, set(expected_files.keys()))

    def check(self, in_files, expected_out_files=None,
              expected_return_code=None, expected_out=None, expected_err=None,
              args=None):
        host = self._host()
        args = args or []

        try:
            orig_wd = host.getcwd()
            tmpdir = host.mkdtemp()
            host.chdir(tmpdir)
            self._write_files(host, in_files)

            returncode, actual_out, actual_err = self._call(host, args)

            actual_out_files = self._read_files(host, tmpdir)
        finally:
            host.rmtree(tmpdir)
            host.chdir(orig_wd)

        if expected_return_code is not None:
            self.assertEqual(returncode, expected_return_code)
        if expected_out is not None:
            self.assertEqual(expected_out, actual_out)
        if expected_err is not None:
            self.assertEqual(expected_err, actual_err)
        if expected_out_files:
            self.assert_files(expected_out_files, actual_out_files)

        return returncode, actual_out, actual_err


class TestArgs(unittest.TestCase, UnitTestMixin, CheckMixin):
    def check_args(self, argv, returncode, out_regex, err_regex):
        in_files = {}
        in_files['build.ninja'] = textwrap.dedent("""
            rule cc_binary
                description = cc_binary $out
                command = cc -o $out $in

            build hello : cc_binary hello.c
            """)
        _, out, err = self.check(in_files, expected_return_code=returncode,
                                 args=argv)
        self.assertTrue(re.match(out_regex, out, re.MULTILINE),
                        '%s does not match %s' % (out_regex, out))
        self.assertTrue(re.match(err_regex, err, re.MULTILINE),
                        '%s does not match %s' % (err_regex, err))

    def test_bad_arg(self):
        self.check_args(['--bad-arg'], 2, '', '')

    def test_bad_dir(self):
        self.check_args(['-C', 'missing_dir'], 2, '', '"missing_dir" not found\n')

    def test_bad_file(self):
        self.check_args(['-f', 'missing_build.ninja'], 2, '',
                         '"missing_build.ninja" not found\n')

    def test_bad_tool(self):
        self.check_args(['-t', 'foo'], 2, '', 'unsupported tool "foo"\n')

    def test_list(self):
        self.check_args(['-t', 'list'], 0, '.+', '')

    def test_chdir(self):
        self.check_args(['-n', '-C', '.'], 0, '', '.*')

    def test_check(self):
        self.check_args(['-t', 'check'], 0, '', '.*')

    def test_clean(self):
        self.check_args(['-n', '-t', 'clean'], 0, '', '.*')

    def test_question(self):
        self.check_args(['-t', 'question'], 1, '', '.*')

    def test_debug(self):
        self.check_args(['-d', 'foo'], 2, '', '-d is not supported yet\n')

    def test_dry_run(self):
        self.check_args(['-n'], 0, '.+', '')

    def test_usage(self):
        self.check_args(['--help'], 0, 'usage:.+', '')
        self.check_args(['-h'], 0, 'usage:.+', '')

    def test_version(self):
        self.check_args(['--version'], 0, VERSION + '\n', '')


class TestBuild(unittest.TestCase, UnitTestMixin, CheckMixin):
    def test_basic(self):
        in_files, out_files = default_test_files()
        host = self._host()
        try:
            orig_wd = host.getcwd()
            tmpdir = host.mkdtemp()
            host.chdir(tmpdir)
            self._write_files(host, in_files)
{'fraction_non_alphanumeric': 0.14171974522292993, 'fraction_numerical': 0.002903709254402398, 'mean_word_length': 2.265137614678899, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3543235', 'n_tokens_mistral': 3481, 'n_tokens_neox': 3360, 'n_words': 754}