<commit_before># Copyright 2015 datawire. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import OrderedDict
from .ast import *
from .parser import Parser, ParseError as GParseError
from .dispatch import dispatch

class Root(AST):

    def __init__(self):
        self.parent = None
        self.index = 0
        self.files = []

    def add(self, file):
        self.files.append(file)

    @property
    def children(self):
        for f in self.files:
            yield f

class InitParent:

    def __init__(self):
        self.stack = []
        self.count = 0

    def name(self, ast):
        if isinstance(ast, (Definition, Param, TypeParam)):
            return ast.name.text
        else:
            return str(ast.index)

    def visit_AST(self, ast):
        ast.resolved = None
        ast.count = 0
        if self.stack:
            ast.parent = self.stack[-1]
            ast.index = ast.parent.count
            ast.parent.count += 1
        self.stack.append(ast)
        ast.id = ".".join([self.name(a) for a in self.stack[1:]])

    def leave_AST(self, ast):
        self.stack.pop()

def pkg_name(pkg):
    if pkg.package is None:
        return code(pkg.name)
    else:
        return "%s.%s" % (pkg_name(pkg.package), code(pkg.name))

class InitEnv:

    def visit_Package(self, p):
        name = pkg_name(p)
        if name in p.root.env:
            p.env = p.root.env[name].env
        else:
            p.env = {}
            p.root.env[name] = p

    def visit_Class(self, c):
        c.env = {}

    def visit_Function(self, f):
        f.env = {}

    def visit_Macro(self, m):
        m.env = {}

    def visit_AST(self, ast):
        if ast.parent:
            ast.env = ast.parent.env
        else:
            ast.env = {}

class Def:

    def __init__(self):
        self.duplicates = []

    def define(self, env, node, name=None, leaf=True, dup=True):
        if name is None:
            name = node.name.text
        if name in env:
            if dup:
                self.duplicates.append((node, name, env[name]))
        else:
            env[name] = node
        if leaf:
            node.resolved = texpr(node)

    def visit_Package(self, p):
        self.define(p.parent.env, p, dup=False)

    def visit_Class(self, c):
        self.define(c.parent.env, c)
        for p in c.parameters:
            self.define(c.env, p)

    def visit_Function(self, f):
        self.define(f.parent.env, f)

    def visit_Method(self, m):
        # we don't put constructors in the namespace
        if m.type:
            self.define(m.parent.env, m)
        self.define(m.env, m.parent, "self")

    def visit_Macro(self, m):
        self.define(m.parent.env, m)

    def visit_MethodMacro(self, mm):
        self.define(mm.parent.env, mm)
        self.define(mm.env, mm.parent, "self")

    def visit_Declaration(self, d):
        self.define(d.env, d, leaf=False)


class TypeExpr(object):

    def __init__(self, type, bindings):
        self.type = type
        self.bindings = bindings

    def get(self, attr, errors):
        name = attr.text
        if name in self.type.env:
            tgt = self.type.env[name].resolved
            return texpr(tgt.type, self.bindings, tgt.bindings)
        else:
            errors.append("%s:%s has no such attribute: %s" % (lineinfo(attr), self.type.name, name))
            return None

    @dispatch(list)
    def invoke(self, errors):
        return self.invoke(self.type, errors)

    @dispatch(Function)
    def invoke(self, fun, errors):
        return texpr(fun.type.resolved.type, self.bindings)

    @dispatch(Class)
    def invoke(self, cls, errors):
        return texpr(cls, self.bindings)

    @dispatch(Macro)
    def invoke(self, mac, errors):
        return texpr(mac.type.resolved.type, self.bindings)

    @property
    def id(self):
        return self.pprint(self.type)

    @dispatch(Package)
    def pprint(self, pkg):
        return pkg.id

    @dispatch(Class)
    def pprint(self, cls):
        if cls.parameters:
            params = [repr(self.bindings.get(p, p)) for p in cls.parameters]<commit_msg>removed duplicate code<commit_after># Copyright 2015 datawire. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import OrderedDict
from .ast import *
from .parser import Parser, ParseError as GParseError
from .dispatch import dispatch

class Root(AST):

    def __init__(self):
        self.parent = None
        self.index = 0
        self.files = []

    def add(self, file):
        self.files.append(file)

    @property
    def children(self):
        for f in self.files:
            yield f

class InitParent:

    def __init__(self):
        self.stack = []
        self.count = 0

    def name(self, ast):
        if isinstance(ast, (Definition, Param, TypeParam)):
            return ast.name.text
        else:
            return str(ast.index)

    def visit_AST(self, ast):
        ast.resolved = None
        ast.count = 0
        if self.stack:
            ast.parent = self.stack[-1]
            ast.index = ast.parent.count
            ast.parent.count += 1
        self.stack.append(ast)
        ast.id = ".".join([self.name(a) for a in self.stack[1:]])

    def leave_AST(self, ast):
        self.stack.pop()

def pkg_name(pkg):
    if pkg.package is None:
        return code(pkg.name)
    else:
        return "%s.%s" % (pkg_name(pkg.package), code(pkg.name))

class InitEnv:

    def visit_Package(self, p):
        name = pkg_name(p)
        if name in p.root.env:
            p.env = p.root.env[name].env
        else:
            p.env = {}
            p.root.env[name] = p

    def visit_Class(self, c):
        c.env = {}

    def visit_Function(self, f):
        f.env = {}

    def visit_Macro(self, m):
        m.env = {}

    def visit_AST(self, ast):
        if ast.parent:
            ast.env = ast.parent.env
        else:
            ast.env = {}

class Def:

    def __init__(self):
        self.duplicates = []

    def define(self, env, node, name=None, leaf=True, dup=True):
        if name is None:
            name = node.name.text
        if name in env:
            if dup:
                self.duplicates.append((node, name, env[name]))
        else:
            env[name] = node
        if leaf:
            node.resolved = texpr(node)

    def visit_Package(self, p):
        self.define(p.parent.env, p, dup=False)

    def visit_Class(self, c):
        self.define(c.parent.env, c)
        for p in c.parameters:
            self.define(c.env, p)

    def visit_Function(self, f):
        self.define(f.parent.env, f)

    def visit_Method(self, m):
        # we don't put constructors in the namespace
        if m.type:
            self.define(m.parent.env, m)
        self.define(m.env, m.parent, "self")

    def visit_Macro(self, m):
        self.define(m.parent.env, m)

    def visit_MethodMacro(self, mm):
        self.define(mm.parent.env, mm)
        self.define(mm.env, mm.parent, "self")

    def visit_Declaration(self, d):
        self.define(d.env, d, leaf=False)


class TypeExpr(object):

    def __init__(self, type, bindings):
        self.type = type
        self.bindings = bindings

    def get(self, attr, errors):
        name = attr.text
        if name in self.type.env:
            tgt = self.type.env[name].resolved
            return texpr(tgt.type, self.bindings, tgt.bindings)
        else:
            errors.append("%s:%s has no such attribute: %s" % (lineinfo(attr), self.type.name, name))
            return None

    @dispatch(list)
    def invoke(self, errors):
        return self.invoke(self.type, errors)

    @dispatch(Callable)
    def invoke(self, dfn, errors):
        return texpr(dfn.type.resolved.type, self.bindings)

    @dispatch(Class)
    def invoke(self, cls, errors):
        return texpr(cls, self.bindings)

    @property
    def id(self):
        return self.pprint(self.type)

    @dispatch(Package)
    def pprint(self, pkg):
        return pkg.id

    @dispatch(Class)
    def pprint(self, cls):
        if cls.parameters:
            params = [repr(self.bindings.get(p, p)) for p in cls.parameters]
{'fraction_non_alphanumeric': 0.11540134870567761, 'fraction_numerical': 0.0030454644333260824, 'mean_word_length': 2.085570469798658, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5539445', 'n_tokens_mistral': 2926, 'n_tokens_neox': 2835, 'n_words': 807}