<commit_before># coding:utf-8

from __future__ import absolute_import, division, print_function, unicode_literals
from .. import source, lexer, diagnostic, ast, coverage
from ..coverage import parser
import unittest, sys, re, ast as pyast

if sys.version_info >= (3,):
    def unicode(x): return x

def tearDownModule():
    coverage.report(parser)

class ParserTestCase(unittest.TestCase):

    maxDiff = None

    versions = [(2, 6), (2, 7), (3, 0), (3, 1), (3, 2), (3, 3), (3, 4), (3, 5)]

    def parser_for(self, code, version, interactive=False):
        code = code.replace("·", "\n")

        self.source_buffer = source.Buffer(code, str(version))
        self.engine = diagnostic.Engine()
        self.engine.render_diagnostic = lambda diag: None
        self.lexer = lexer.Lexer(self.source_buffer, version, self.engine,
                                 interactive=interactive)

        old_next = self.lexer.next
        def lexer_next(**args):
            token = old_next(**args)
            # print(repr(token))
            return token
        self.lexer.next = lexer_next

        self.parser = parser.Parser(self.lexer, version, self.engine)
        return self.parser

    def flatten_ast(self, node):
        if node is None:
            return None

        # Validate locs and fields
        for attr in node.__dict__:
            if attr.endswith("loc") or attr.endswith("_locs"):
                self.assertTrue(attr in node._locs,
                                "%s not in %s._locs" % (attr, repr(node)))
            else:
                self.assertTrue(attr in node._fields,
                                "%s not in %s._fields" % (attr, repr(node)))
        for loc in node._locs:
            self.assertTrue(loc in node.__dict__,
                            "loc %s not in %s" % (loc, repr(node)))
        for field in node._fields:
            self.assertTrue(field in node.__dict__,
                            "field %s not in %s" % (field, repr(node)))

        flat_node = { "ty": unicode(type(node).__name__) }
        for field in node._fields:
            value = getattr(node, field)
            if isinstance(value, ast.AST):
                value = self.flatten_ast(value)
            if isinstance(value, list) and len(value) > 0 and \
                    any([isinstance(x, ast.AST) for x in value]):
                value = list(map(self.flatten_ast, value))
            flat_node[unicode(field)] = value
        return flat_node

    def flatten_python_ast(self, node):
        if node is None:
            return None

        flat_node = { "ty": unicode(type(node).__name__) }
        for field in node._fields:
            if field == "ctx":
                flat_node["ctx"] = None
                continue

            value = getattr(node, field)
            if isinstance(value, pyast.AST):
                value = self.flatten_python_ast(value)
            if isinstance(value, list) and len(value) > 0 and \
                    any([isinstance(x, pyast.AST) for x in value]):
                value = list(map(self.flatten_python_ast, value))
            flat_node[unicode(field)] = value
        return flat_node

    _loc_re  = re.compile(r"\s*([~^]*)<?\s+([a-z_0-9.]+)")
    _path_re = re.compile(r"(([a-z_]+)|([0-9]+))(\.)?")

    def match_loc(self, ast, matcher, root=lambda x: (0, x)):
        offset, ast = root(ast)

        matcher_pos = 0
        while matcher_pos < len(matcher):
            matcher_match = self._loc_re.match(matcher, matcher_pos)
            if matcher_match is None:
                raise Exception("invalid location matcher %s" % matcher[matcher_pos:])

            range = source.Range(self.source_buffer,
                matcher_match.start(1) - matcher_pos + offset,
                matcher_match.end(1) - matcher_pos + offset)
            path = matcher_match.group(2)

            path_pos = 0
            obj = ast
            while path_pos < len(path):
                path_match = self._path_re.match(path, path_pos)
                if path_match is None:
                    raise Exception("invalid location matcher path %s" % path)

                path_field = path_match.group(2)
                path_index = path_match.group(3)
                path_last  = not path_match.group(4)

                if path_field is not None:
                    obj = getattr(obj, path_field)
                elif path_index is not None:
                    obj = obj[int(path_index)]

                if path_last:
                    self.assertEqual(obj, range)

                path_pos = path_match.end(0)

            matcher_pos = matcher_match.end(0)

    def assertParsesGen(self, expected_flat_ast, code,
                        loc_matcher="", ast_slicer=lambda x: (0, x),
                        only_if=lambda ver: True, validate_if=lambda: True):
        for version in self.versions:
            if not only_if(version):
                continue

            ast = self.parser_for(code + "\n", version).file_input()
            flat_ast = self.flatten_ast(ast)
            self.assertEqual({"ty": "Module", "body": expected_flat_ast},
                             flat_ast)
            self.match_loc(ast, loc_matcher, ast_slicer)

            if version == sys.version_info[0:2] and validate_if():
                python_ast = pyast.parse(code.replace("·", "\n") + "\n")
                flat_python_ast = self.flatten_python_ast(python_ast)
                self.assertEqual({"ty": "Module", "body": expected_flat_ast},
                                 flat_python_ast)

    def assertParsesSuite(self, expected_flat_ast, code, loc_matcher="", **kwargs):
        self.assertParsesGen(expected_flat_ast, code,<commit_msg>Only validate ASTs against the specific Python version we target.<commit_after># coding:utf-8

from __future__ import absolute_import, division, print_function, unicode_literals
from .. import source, lexer, diagnostic, ast, coverage
from ..coverage import parser
import unittest, sys, re, ast as pyast

if sys.version_info >= (3,):
    def unicode(x): return x

def tearDownModule():
    coverage.report(parser)

class ParserTestCase(unittest.TestCase):

    maxDiff = None

    versions = [(2, 6), (2, 7), (3, 0), (3, 1), (3, 2), (3, 3), (3, 4), (3, 5)]

    def parser_for(self, code, version, interactive=False):
        code = code.replace("·", "\n")

        self.source_buffer = source.Buffer(code, str(version))
        self.engine = diagnostic.Engine()
        self.engine.render_diagnostic = lambda diag: None
        self.lexer = lexer.Lexer(self.source_buffer, version, self.engine,
                                 interactive=interactive)

        old_next = self.lexer.next
        def lexer_next(**args):
            token = old_next(**args)
            # print(repr(token))
            return token
        self.lexer.next = lexer_next

        self.parser = parser.Parser(self.lexer, version, self.engine)
        return self.parser

    def flatten_ast(self, node):
        if node is None:
            return None

        # Validate locs and fields
        for attr in node.__dict__:
            if attr.endswith("loc") or attr.endswith("_locs"):
                self.assertTrue(attr in node._locs,
                                "%s not in %s._locs" % (attr, repr(node)))
            else:
                self.assertTrue(attr in node._fields,
                                "%s not in %s._fields" % (attr, repr(node)))
        for loc in node._locs:
            self.assertTrue(loc in node.__dict__,
                            "loc %s not in %s" % (loc, repr(node)))
        for field in node._fields:
            self.assertTrue(field in node.__dict__,
                            "field %s not in %s" % (field, repr(node)))

        flat_node = { "ty": unicode(type(node).__name__) }
        for field in node._fields:
            value = getattr(node, field)
            if isinstance(value, ast.AST):
                value = self.flatten_ast(value)
            if isinstance(value, list) and len(value) > 0 and \
                    any([isinstance(x, ast.AST) for x in value]):
                value = list(map(self.flatten_ast, value))
            flat_node[unicode(field)] = value
        return flat_node

    def flatten_python_ast(self, node):
        if node is None:
            return None

        flat_node = { "ty": unicode(type(node).__name__) }
        for field in node._fields:
            if field == "ctx":
                flat_node["ctx"] = None
                continue

            value = getattr(node, field)
            if isinstance(value, pyast.AST):
                value = self.flatten_python_ast(value)
            if isinstance(value, list) and len(value) > 0 and \
                    any([isinstance(x, pyast.AST) for x in value]):
                value = list(map(self.flatten_python_ast, value))
            flat_node[unicode(field)] = value
        return flat_node

    _loc_re  = re.compile(r"\s*([~^]*)<?\s+([a-z_0-9.]+)")
    _path_re = re.compile(r"(([a-z_]+)|([0-9]+))(\.)?")

    def match_loc(self, ast, matcher, root=lambda x: (0, x)):
        offset, ast = root(ast)

        matcher_pos = 0
        while matcher_pos < len(matcher):
            matcher_match = self._loc_re.match(matcher, matcher_pos)
            if matcher_match is None:
                raise Exception("invalid location matcher %s" % matcher[matcher_pos:])

            range = source.Range(self.source_buffer,
                matcher_match.start(1) - matcher_pos + offset,
                matcher_match.end(1) - matcher_pos + offset)
            path = matcher_match.group(2)

            path_pos = 0
            obj = ast
            while path_pos < len(path):
                path_match = self._path_re.match(path, path_pos)
                if path_match is None:
                    raise Exception("invalid location matcher path %s" % path)

                path_field = path_match.group(2)
                path_index = path_match.group(3)
                path_last  = not path_match.group(4)

                if path_field is not None:
                    obj = getattr(obj, path_field)
                elif path_index is not None:
                    obj = obj[int(path_index)]

                if path_last:
                    self.assertEqual(obj, range)

                path_pos = path_match.end(0)

            matcher_pos = matcher_match.end(0)

    def assertParsesGen(self, expected_flat_ast, code,
                        loc_matcher="", ast_slicer=lambda x: (0, x),
                        only_if=lambda ver: True, validate_if=lambda: True):
        for version in self.versions:
            if not only_if(version):
                continue

            ast = self.parser_for(code + "\n", version).file_input()
            flat_ast = self.flatten_ast(ast)
            self.assertEqual({"ty": "Module", "body": expected_flat_ast},
                             flat_ast)
            self.match_loc(ast, loc_matcher, ast_slicer)

            compatible_pyast_version = \
                (sys.version_info[0:2] == (2, 7) or
                 sys.version_info[0:2] == (3, 4))
            if compatible_pyast_version and version == sys.version_info[0:2] and validate_if():
                python_ast = pyast.parse(code.replace("·", "\n") + "\n")
                flat_python_ast = self.flatten_python_ast(python_ast)
                self.assertEqual({"ty": "Module", "body": expected_flat_ast},
                                 flat_python_ast)

    def assertParsesSuite(self, expected_flat_ast, code, loc_matcher="", **kwargs):
        self.assertParsesGen(expected_flat_ast, code,
{'fraction_non_alphanumeric': 0.1134453781512605, 'fraction_numerical': 0.007202881152460984, 'mean_word_length': 1.8328880252611124, 'pattern_counts': {'":': 14, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '224541', 'n_tokens_mistral': 3556, 'n_tokens_neox': 3377, 'n_words': 826}