<commit_before>from pydoctor import model, ast_pp

from compiler import visitor, transformer, ast
import os, sys, posixpath, symbol, token

class mystr(str):
    pass

class MyTransformer(transformer.Transformer):
    def get_docstring(self, node, n=None):
        if n is None:
            n = node[0]
            node = node[1:]
        if n == symbol.suite:
            if len(node) == 1:
                return self.get_docstring(node[0])
            for sub in node:
                if sub[0] == symbol.stmt:
                    return self.get_docstring(sub)
            return None
        if n == symbol.file_input:
            for sub in node:
                if sub[0] == symbol.stmt:
                    return self.get_docstring(sub)
            return None
        if n == symbol.atom:
            if node[0][0] == token.STRING:
                s = ''
                for t in node:
                    s = s + eval(t[1])
                r = mystr(s)
                r.orig = ''.join(t[1] for t in node)
                r.linenumber = node[0][2]
                return r
            return None
        if n == symbol.stmt or n == symbol.simple_stmt \
           or n == symbol.small_stmt:
            return self.get_docstring(node[0])
        if n in transformer._doc_nodes and len(node) == 1:
            return self.get_docstring(node[0])
        return None

def parseFile(path):
    f = open(path, "U")
    # XXX The parser API tolerates files without a trailing newline,
    # but not strings without a trailing newline.  Always add an extra
    # newline to the file contents, since we're going through the string
    # version of the API.
    src = f.read() + "\n"
    f.close()
    return parse(src)

def parse(buf):
    return MyTransformer().parsesuite(buf)


class ModuleVistor(object):
    def __init__(self, builder, modname):
        self.builder = builder
        self.system = builder.system
        self.modname = modname
        self.morenodes = []

    def default(self, node):
        for child in node.getChildNodes():
            self.visit(child)

    def visitModule(self, node):
        # i don't think this needs to be so confusing any more...
        m = None
        if self.builder.current and self.modname in self.builder.current.contents:
            m = self.builder.current.contents[self.modname]
        elif self.modname in self.system.allobjects:
            m = self.system.allobjects[self.modname]
        if m:
            assert m.docstring is None
            m.docstring = node.doc
            self.builder.push(m)
            self.default(node)
            self.builder.pop(m)
        else:
            if not self.builder.current:
                roots = [x for x in self.system.rootobjects if x.name == self.modname]
                if roots:
                    mod, = roots
                    self.builder.push(mod)
                    self.default(node)
                    self.builder.pop(mod)
                    return
            m = self.builder.pushModule(self.modname, node.doc)
            if m is None:
                return
            self.default(node)
            self.builder.popModule()

    def visitClass(self, node):
        rawbases = []
        bases = []
        baseobjects = []
        current = self.builder.current
        for n in node.bases:
            str_base = ast_pp.pp(n)
            rawbases.append(str_base)
            base = current.dottedNameToFullName(str_base)
            bob = self.system.objForFullName(base)
            if not bob and self.system.options.resolvealiases:
                base = self.system.resolveAlias(base)
                bob = self.system.objForFullName(base)
            if bob:
                assert (bob.parentMod is self.builder.currentMod or
                        bob.parentMod.processed)
            bases.append(base)
            baseobjects.append(bob)

        cls = self.builder.pushClass(node.name, node.doc)
        if cls is None:
            return
        if node.lineno is not None:
            cls.linenumber = node.lineno
        if cls.parentMod.sourceHref:
            cls.sourceHref = cls.parentMod.sourceHref + '#L' + str(cls.linenumber)
        cls.rawbases = rawbases
        cls.bases = bases
        cls.baseobjects = baseobjects
        for b in cls.baseobjects:
            if b is not None:
                b.subclasses.append(cls)
        self.default(node)
        self.builder.popClass()

    def visitFrom(self, node):
        modname = self.builder.expandModname(node.modname)
        name2fullname = self.builder.current._name2fullname
        for fromname, asname in node.names:
            if fromname == '*':
                self.builder.warning("import *", modname)
                if modname not in self.system.allobjects:
                    return
                mod = self.system.allobjects[modname]
                if isinstance(mod, model.Package):
                    self.builder.warning("import * from a package", modname)
                    return
                assert mod.processed
                if mod.all is not None:
                    names = mod.all
                else:
                    names = [k for k in mod.contents.keys() if not k.startswith('_')]
                for n in names:
                    name2fullname[n] = modname + '.' + n
                return
            if asname is None:
                asname = fromname
            if isinstance(self.builder.current, model.Module) and \
                   self.builder.current.all is not None and \
                   asname in self.builder.current.all and \
                   modname in self.system.allobjects:
                mod = self.system.allobjects[modname]
                if 0 and isinstance(mod, model.Module) and fromname in mod.contents:
                    print 'moving', mod.contents[fromname], 'into', self.builder.current
                    # this code attempts to preserve "rather a lot" of
                    # invariants assumed by various bits of pydoctor and that
                    # are of course not written down anywhere :/
                    ob = mod.contents[fromname]
                    targetmod = self.builder.current
                    del self.system.allobjects[ob.fullName()]
                    ob.parent = ob.parentMod = targetmod
                    ob.prefix = targetmod.fullName() + '.'
                    ob.name = asname
                    self.system.allobjects[ob.fullName()] = ob
                    del mod.contents[fromname]
                    mod.orderedcontents.remove(ob)
                    mod._name2fullname[fromname] = ob.fullName()
                    targetmod.contents[asname] = ob
                    targetmod.orderedcontents.append(ob)
            else:
                name2fullname[asname] = modname + '.' + fromname

    def visitImport(self, node):
        name2fullname = self.builder.current._name2fullname
        for fromname, asname in node.names:
            fullname = self.builder.expandModname(fromname)
            if asname is None:
                asname = fromname.split('.', 1)[0]
                # aaaaargh! python sucks.
                parts = fullname.split('.')
                for i, part in enumerate(fullname.split('.')[::-1]):
                    if part == asname:
                        fullname = '.'.join(parts[:len(parts)-i])
                        name2fullname[asname] = fullname
                        break
                else:
                    fullname = '.'.join(parts)
                    name2fullname[asname] = '.'.join(parts)
            else:
                name2fullname[asname] = fullname

    def visitAssign(self, node):
        if isinstance(self.builder.current, model.Class):
            if len(node.nodes) != 1:
                return
            if not isinstance(node.nodes[0], ast.AssName):
                return
            target = node.nodes[0].name
            if not isinstance(node.expr, ast.CallFunc):
                return
            func = node.expr.node
            if not isinstance(func, ast.Name):
                return
            func = func.name
            args = node.expr.args
            if len(args) != 1:
                return
            arg, = args
            if not isinstance(arg, ast.Name):
                return
            arg = arg.name
            if target == arg and func in ['staticmethod', 'classmethod']:
                target = self.builder.current.contents.get(target)
                if target and isinstance(target, model.Function):
                    if target.kind != 'Method':
                        self.system.msg('ast', 'XXX')
                    else:
                        target.kind = func.title().replace('m', ' M')
        elif isinstance(self.builder.current, model.Module):
            mod = self.builder.current
            if len(node.nodes) != 1 or \
                   not isinstance(node.nodes[0], ast.AssName) or \
                   node.nodes[0].name != '__all__':
                return
            if mod.all is not None:
                self.builder.system.msg(
                    'all', "multiple assignments to %s.__all__ ??"%(mod.fullName(),))
            if not isinstance(node.expr, ast.List):
                self.builder.system.msg(
                    'all', "couldn't parse %s.__all__"%(mod.fullName(),))
                return
            items = node.expr.nodes
            names = []
            for item in items:
                if not isinstance(item, ast.Const) or not isinstance(item.value, str):
                    mod.system.msg('all', "couldn't parse %s.__all__"%(mod.fullName(),))
                    return
                names.append(item.value)
            mod.all = names

    def visitFunction(self, node):
        func = self.builder.pushFunction(node.name, node.doc)
        if func is None:
            return
        func.decorators = node.decorators
        if isinstance(func.parent, model.Class) and node.decorators:
            isclassmethod = False
            isstaticmethod = False
            for d in node.decorators.nodes:
                if isinstance(d, ast.Name):
                    if d.name == 'classmethod':
                        isclassmethod = True
                    elif d.name == 'staticmethod':
                        isstaticmethod = True
            if isstaticmethod:
                if isclassmethod:
                    self.system.msg('ast', '%r is both class- and static-method?'%(func.fullName(),), thresh=-1)
                else:
                    func.kind = 'Static Method'
            elif isclassmethod:
                func.kind = 'Class Method'
        if node.lineno is not None:
            func.linenumber = node.lineno
        if func.parentMod.sourceHref:
            func.sourceHref = func.parentMod.sourceHref + '#L' + str(func.linenumber)
        # ast.Function has a pretty lame representation of
        # arguments. Let's convert it to a nice concise format
        # somewhat like what inspect.getargspec returns
        argnames = node.argnames[:]
        kwname = starargname = None
        if node.kwargs:
            kwname = argnames.pop(-1)
        if node.varargs:
            starargname = argnames.pop(-1)
        defaults = []
        for default in node.defaults:
            try:
                defaults.append(ast_pp.pp(default))
            except (KeyboardInterrupt, SystemExit):
                raise
            except Exception, e:
                self.builder.warning("unparseable default", "%s: %s %r"%(e.__class__.__name__,
                                                                       e, default))
                defaults.append('???')
        # argh, convert unpacked-arguments from tuples to lists,
        # because that's what getargspec uses and the unit test
        # compares it
        argnames2 = []
        for argname in argnames:
            if isinstance(argname, tuple):
                argname = list(argname)
            argnames2.append(argname)
        func.argspec = (argnames2, starargname, kwname, tuple(defaults))
        #self.postpone(func, node.code)
        self.builder.popFunction()

class ASTBuilder(object):
    Class = model.Class
    Module = model.Module
    Package = model.Package
    Function = model.Function
    ModuleVistor = ModuleVistor

    def __init__(self, system):
        self.system = system
        self.current = None
        self.currentMod = None
        self._stack = []
        self.ast_cache = {}

    def _push(self, cls, name, docstring):
        if self.current:
            if isinstance(self.current, model.Module) and \
                   self.current.name == '__init__':
                prefix = self.current.parent.fullName() + '.'
            else:
                prefix = self.current.fullName() + '.'
            parent = self.current
        else:
            prefix = ''
            parent = None
        obj = cls(self.system, prefix, name, docstring, parent)
        if parent:
            parent.orderedcontents.append(obj)
            parent.contents[name] = obj
            parent._name2fullname[name] = obj.fullName()
        else:
            self.system.rootobjects.append(obj)
        self.push(obj)
        self.system.orderedallobjects.append(obj)
        fullName = obj.fullName()
        #print 'push', cls.__name__, fullName
        if fullName in self.system.allobjects:
            obj = self.handleDuplicate(obj)
        else:
            self.system.allobjects[obj.fullName()] = obj
        return obj

    def handleDuplicate(self, obj):
        '''This is called when we see two objects with the same
        .fullName(), for example:

        class C:
            if something:
                def meth(self):
                    implementation 1
            else:
                def meth(self):
                    implementation 2

        The default is that the second definition "wins".
        '''
        i = 0
        fn = obj.fullName()
        while (fn + ' ' + str(i)) in self.system.allobjects:
            i += 1
        prev = self.system.allobjects[obj.fullName()]
        prev.name = obj.name + ' ' + str(i)
        self.system.allobjects[prev.fullName()] = prev
        self.warning("duplicate", self.system.allobjects[obj.fullName()])
        self.system.allobjects[obj.fullName()] = obj
        return obj


    def _pop(self, cls):
        assert isinstance(self.current, cls)
##         if self.current.parent:
##             print 'pop', self.current.fullName(), '->', self.current.parent.fullName()
##         else:
##             print 'pop', self.current.fullName(), '->', self.current.parent
        self.pop(self.current)

    def push(self, obj):
        self._stack.append(self.current)
        self.current = obj
        if isinstance(obj, model.Module):
            assert self.currentMod is None
            obj.parentMod = self.currentMod = obj
        elif self.currentMod is not None:
            if obj.parentMod is not None:
                assert obj.parentMod is self.currentMod
            else:
                obj.parentMod = self.currentMod
        else:
            assert obj.parentMod is None

    def pop(self, obj):
        assert self.current is obj, "%r is not %r"%(self.current, obj)
        self.current = self._stack.pop()
        if isinstance(obj, model.Module):
            self.currentMod = None

    def pushClass(self, name, docstring):
        return self._push(self.Class, name, docstring)
    def popClass(self):
        self._pop(self.Class)

    def pushModule(self, name, docstring):
        return self._push(self.Module, name, docstring)
    def popModule(self):
        self._pop(self.Module)

    def pushFunction(self, name, docstring):
        return self._push(self.Function, name, docstring)
    def popFunction(self):
        self._pop(self.Function)

    def pushPackage(self, name, docstring):
        return self._push(self.Package, name, docstring)
    def popPackage(self):
        self._pop(self.Package)

    def warning(self, type, detail):
        self.system._warning(self.current, type, detail)

    def _finalStateComputations(self):
        pass

    def processModuleAST(self, ast, moduleName):
        visitor.walk(ast, self.ModuleVistor(self, moduleName))

    def expandModname(self, modname, givewarning=True):
        c = self.current
        if '.' in modname:
            prefix, suffix = modname.split('.', 1)
            suffix = '.' + suffix
        else:
            prefix, suffix = modname, ''
        while c is not None and not isinstance(c, model.Package):
            c = c.parent
        while c is not None:
            if prefix in c.contents:
                break
            c = c.parent
        if c is not None:
            if givewarning:
                self.warning("local import", modname)
            return c.contents[prefix].fullName() + suffix
        else:
            return prefix + suffix

    def parseFile(self, filePath):
        if filePath in self.ast_cache:
            return self.ast_cache[filePath]
        try:
            ast = parseFile(filePath)
        except (SyntaxError, ValueError):
            self.warning("cannot parse", filePath)
            ast = None
        self.ast_cache[filePath] = ast
        return ast


    # if we assume:
    #
    # - that svn://divmod.org/trunk is checked out into ~/src/Divmod
    #
    # - that http://divmod.org/trac/browser/trunk is the trac URL to the
    #   above directory
    #
    # - that ~/src/Divmod/Nevow/nevow is passed to pydoctor as an
    #   "--add-package" argument
    #
    # we want to work out the sourceHref for nevow.flat.ten.  the answer
    # is http://divmod.org/trac/browser/trunk/Nevow/nevow/flat/ten.py.
    #
    # we can work this out by finding that Divmod is the top of the svn
    # checkout, and posixpath.join-ing the parts of the filePath that
    # follows that.
    #
    #  http://divmod.org/trac/browser/trunk
    #                          ~/src/Divmod/Nevow/nevow/flat/ten.py

    def setSourceHref(self, mod):
        if self.system.sourcebase is None:
            mod.sourceHref = None
            return

        trailing = []
        dir, fname = os.path.split(mod.filepath)
        while os.path.exists(os.path.join(dir, '.svn')):
            dir, dirname = os.path.split(dir)
            trailing.append(dirname)

        # now trailing[-1] would be 'Divmod' in the above example
        del trailing[-1]
        trailing.reverse()
        trailing.append(fname)

        mod.sourceHref = posixpath.join(mod.system.sourcebase, *trailing)


    def addModule(self, modpath):
        assert self.system.state in ['blank', 'preparse']
        fname = os.path.basename(modpath)
        modname = os.path.splitext(fname)[0]
        mod = self.pushModule(modname, None)
        self.system.progress("addModule", len(self.system.orderedallobjects),
                             None, "modules and packages discovered")
        mod.filepath = modpath
        mod.processed = False
        self.setSourceHref(mod)
        self.popModule()
        self.system.state = 'preparse'

    def preprocessDirectory(self, dirpath):
        assert self.system.state in ['blank', 'preparse']
        if not os.path.exists(dirpath):
            raise Exception, "package path %r does not exist!"%(dirpath,)
        if not os.path.exists(os.path.join(dirpath, '__init__.py')):
            raise Exception, "you must pass a package directory to preprocessDirectory"
        if os.path.basename(dirpath):
            package = self.pushPackage(os.path.basename(dirpath), None)
            package.filepath = dirpath
            self.setSourceHref(package)
        else:
            package = None
        for fname in os.listdir(dirpath):
            fullname = os.path.join(dirpath, fname)
            if os.path.isdir(fullname) and os.path.exists(os.path.join(fullname, '__init__.py')):
                self.preprocessDirectory(fullname)
            elif fname.endswith('.py') and not fname.startswith('.'):
                self.addModule(fullname)
        if package:
            self.popPackage()
        self.system.state = 'preparse'

    def analyseImports(self):
        assert self.system.state in ['preparse']
        modlist = list(self.system.objectsOfType(model.Module))
        for i, mod in enumerate(modlist):
            if mod.parent:
                self.push(mod.parent)
            isf = ImportFinder(self, mod)
            ast = self.parseFile(mod.filepath)
            if not ast:
                continue
            self.system.progress('analyseImports', i+1, len(modlist),
                                 "modules parsed")
            visitor.walk(ast, isf)
            if mod.parent:<commit_msg>porting patches from bzr... ------------------------------------------------------------ revno: 391 committer: <NAME> <<EMAIL>> branch nick: pydoctor.dev timestamp: Mon 2007-05-14 02:41:00 +0100 message:   go back (yet again) to trying to find __all__ nice and early.   also reenable the __all__-initiated object moving code<commit_after>from pydoctor import model, ast_pp

from compiler import visitor, transformer, ast
import os, sys, posixpath, symbol, token

class mystr(str):
    pass

class MyTransformer(transformer.Transformer):
    def get_docstring(self, node, n=None):
        if n is None:
            n = node[0]
            node = node[1:]
        if n == symbol.suite:
            if len(node) == 1:
                return self.get_docstring(node[0])
            for sub in node:
                if sub[0] == symbol.stmt:
                    return self.get_docstring(sub)
            return None
        if n == symbol.file_input:
            for sub in node:
                if sub[0] == symbol.stmt:
                    return self.get_docstring(sub)
            return None
        if n == symbol.atom:
            if node[0][0] == token.STRING:
                s = ''
                for t in node:
                    s = s + eval(t[1])
                r = mystr(s)
                r.orig = ''.join(t[1] for t in node)
                r.linenumber = node[0][2]
                return r
            return None
        if n == symbol.stmt or n == symbol.simple_stmt \
           or n == symbol.small_stmt:
            return self.get_docstring(node[0])
        if n in transformer._doc_nodes and len(node) == 1:
            return self.get_docstring(node[0])
        return None

def parseFile(path):
    f = open(path, "U")
    # XXX The parser API tolerates files without a trailing newline,
    # but not strings without a trailing newline.  Always add an extra
    # newline to the file contents, since we're going through the string
    # version of the API.
    src = f.read() + "\n"
    f.close()
    return parse(src)

def parse(buf):
    return MyTransformer().parsesuite(buf)


class ModuleVistor(object):
    def __init__(self, builder, modname):
        self.builder = builder
        self.system = builder.system
        self.modname = modname
        self.morenodes = []

    def default(self, node):
        for child in node.getChildNodes():
            self.visit(child)

    def visitModule(self, node):
        # i don't think this needs to be so confusing any more...
        m = None
        if self.builder.current and self.modname in self.builder.current.contents:
            m = self.builder.current.contents[self.modname]
        elif self.modname in self.system.allobjects:
            m = self.system.allobjects[self.modname]
        if m:
            assert m.docstring is None
            m.docstring = node.doc
            self.builder.push(m)
            self.default(node)
            self.builder.pop(m)
        else:
            if not self.builder.current:
                roots = [x for x in self.system.rootobjects if x.name == self.modname]
                if roots:
                    mod, = roots
                    self.builder.push(mod)
                    self.default(node)
                    self.builder.pop(mod)
                    return
            m = self.builder.pushModule(self.modname, node.doc)
            if m is None:
                return
            self.default(node)
            self.builder.popModule()

    def visitClass(self, node):
        rawbases = []
        bases = []
        baseobjects = []
        current = self.builder.current
        for n in node.bases:
            str_base = ast_pp.pp(n)
            rawbases.append(str_base)
            base = current.dottedNameToFullName(str_base)
            bob = self.system.objForFullName(base)
            if not bob and self.system.options.resolvealiases:
                base = self.system.resolveAlias(base)
                bob = self.system.objForFullName(base)
            if bob:
                assert (bob.parentMod is self.builder.currentMod or
                        bob.parentMod.processed)
            bases.append(base)
            baseobjects.append(bob)

        cls = self.builder.pushClass(node.name, node.doc)
        if cls is None:
            return
        if node.lineno is not None:
            cls.linenumber = node.lineno
        if cls.parentMod.sourceHref:
            cls.sourceHref = cls.parentMod.sourceHref + '#L' + str(cls.linenumber)
        cls.rawbases = rawbases
        cls.bases = bases
        cls.baseobjects = baseobjects
        for b in cls.baseobjects:
            if b is not None:
                b.subclasses.append(cls)
        self.default(node)
        self.builder.popClass()

    def visitFrom(self, node):
        modname = self.builder.expandModname(node.modname)
        name2fullname = self.builder.current._name2fullname
        for fromname, asname in node.names:
            if fromname == '*':
                self.builder.warning("import *", modname)
                if modname not in self.system.allobjects:
                    return
                mod = self.system.allobjects[modname]
                if isinstance(mod, model.Package):
                    self.builder.warning("import * from a package", modname)
                    return
                assert mod.processed
                if mod.all is not None:
                    names = mod.all
                else:
                    names = [k for k in mod.contents.keys() if not k.startswith('_')]
                for n in names:
                    name2fullname[n] = modname + '.' + n
                return
            if asname is None:
                asname = fromname
            if isinstance(self.builder.current, model.Module) and \
                   self.builder.current.all is not None and \
                   asname in self.builder.current.all and \
                   modname in self.system.allobjects:
                mod = self.system.allobjects[modname]
                if isinstance(mod, model.Module) and fromname in mod.contents:
                    print 'moving', mod.contents[fromname], 'into', self.builder.current
                    # this code attempts to preserve "rather a lot" of
                    # invariants assumed by various bits of pydoctor and that
                    # are of course not written down anywhere :/
                    ob = mod.contents[fromname]
                    targetmod = self.builder.current
                    del self.system.allobjects[ob.fullName()]
                    ob.parent = ob.parentMod = targetmod
                    ob.prefix = targetmod.fullName() + '.'
                    ob.name = asname
                    self.system.allobjects[ob.fullName()] = ob
                    del mod.contents[fromname]
                    mod.orderedcontents.remove(ob)
                    mod._name2fullname[fromname] = ob.fullName()
                    targetmod.contents[asname] = ob
                    targetmod.orderedcontents.append(ob)
            else:
                name2fullname[asname] = modname + '.' + fromname

    def visitImport(self, node):
        name2fullname = self.builder.current._name2fullname
        for fromname, asname in node.names:
            fullname = self.builder.expandModname(fromname)
            if asname is None:
                asname = fromname.split('.', 1)[0]
                # aaaaargh! python sucks.
                parts = fullname.split('.')
                for i, part in enumerate(fullname.split('.')[::-1]):
                    if part == asname:
                        fullname = '.'.join(parts[:len(parts)-i])
                        name2fullname[asname] = fullname
                        break
                else:
                    fullname = '.'.join(parts)
                    name2fullname[asname] = '.'.join(parts)
            else:
                name2fullname[asname] = fullname

    def visitAssign(self, node):
        if isinstance(self.builder.current, model.Class):
            if len(node.nodes) != 1:
                return
            if not isinstance(node.nodes[0], ast.AssName):
                return
            target = node.nodes[0].name
            if not isinstance(node.expr, ast.CallFunc):
                return
            func = node.expr.node
            if not isinstance(func, ast.Name):
                return
            func = func.name
            args = node.expr.args
            if len(args) != 1:
                return
            arg, = args
            if not isinstance(arg, ast.Name):
                return
            arg = arg.name
            if target == arg and func in ['staticmethod', 'classmethod']:
                target = self.builder.current.contents.get(target)
                if target and isinstance(target, model.Function):
                    if target.kind != 'Method':
                        self.system.msg('ast', 'XXX')
                    else:
                        target.kind = func.title().replace('m', ' M')

    def visitFunction(self, node):
        func = self.builder.pushFunction(node.name, node.doc)
        if func is None:
            return
        func.decorators = node.decorators
        if isinstance(func.parent, model.Class) and node.decorators:
            isclassmethod = False
            isstaticmethod = False
            for d in node.decorators.nodes:
                if isinstance(d, ast.Name):
                    if d.name == 'classmethod':
                        isclassmethod = True
                    elif d.name == 'staticmethod':
                        isstaticmethod = True
            if isstaticmethod:
                if isclassmethod:
                    self.system.msg('ast', '%r is both class- and static-method?'%(func.fullName(),), thresh=-1)
                else:
                    func.kind = 'Static Method'
            elif isclassmethod:
                func.kind = 'Class Method'
        if node.lineno is not None:
            func.linenumber = node.lineno
        if func.parentMod.sourceHref:
            func.sourceHref = func.parentMod.sourceHref + '#L' + str(func.linenumber)
        # ast.Function has a pretty lame representation of
        # arguments. Let's convert it to a nice concise format
        # somewhat like what inspect.getargspec returns
        argnames = node.argnames[:]
        kwname = starargname = None
        if node.kwargs:
            kwname = argnames.pop(-1)
        if node.varargs:
            starargname = argnames.pop(-1)
        defaults = []
        for default in node.defaults:
            try:
                defaults.append(ast_pp.pp(default))
            except (KeyboardInterrupt, SystemExit):
                raise
            except Exception, e:
                self.builder.warning("unparseable default", "%s: %s %r"%(e.__class__.__name__,
                                                                       e, default))
                defaults.append('???')
        # argh, convert unpacked-arguments from tuples to lists,
        # because that's what getargspec uses and the unit test
        # compares it
        argnames2 = []
        for argname in argnames:
            if isinstance(argname, tuple):
                argname = list(argname)
            argnames2.append(argname)
        func.argspec = (argnames2, starargname, kwname, tuple(defaults))
        #self.postpone(func, node.code)
        self.builder.popFunction()

class ASTBuilder(object):
    Class = model.Class
    Module = model.Module
    Package = model.Package
    Function = model.Function
    ModuleVistor = ModuleVistor

    def __init__(self, system):
        self.system = system
        self.current = None
        self.currentMod = None
        self._stack = []
        self.ast_cache = {}

    def _push(self, cls, name, docstring):
        if self.current:
            if isinstance(self.current, model.Module) and \
                   self.current.name == '__init__':
                prefix = self.current.parent.fullName() + '.'
            else:
                prefix = self.current.fullName() + '.'
            parent = self.current
        else:
            prefix = ''
            parent = None
        obj = cls(self.system, prefix, name, docstring, parent)
        if parent:
            parent.orderedcontents.append(obj)
            parent.contents[name] = obj
            parent._name2fullname[name] = obj.fullName()
        else:
            self.system.rootobjects.append(obj)
        self.push(obj)
        self.system.orderedallobjects.append(obj)
        fullName = obj.fullName()
        #print 'push', cls.__name__, fullName
        if fullName in self.system.allobjects:
            obj = self.handleDuplicate(obj)
        else:
            self.system.allobjects[obj.fullName()] = obj
        return obj

    def handleDuplicate(self, obj):
        '''This is called when we see two objects with the same
        .fullName(), for example:

        class C:
            if something:
                def meth(self):
                    implementation 1
            else:
                def meth(self):
                    implementation 2

        The default is that the second definition "wins".
        '''
        i = 0
        fn = obj.fullName()
        while (fn + ' ' + str(i)) in self.system.allobjects:
            i += 1
        prev = self.system.allobjects[obj.fullName()]
        prev.name = obj.name + ' ' + str(i)
        self.system.allobjects[prev.fullName()] = prev
        self.warning("duplicate", self.system.allobjects[obj.fullName()])
        self.system.allobjects[obj.fullName()] = obj
        return obj


    def _pop(self, cls):
        assert isinstance(self.current, cls)
##         if self.current.parent:
##             print 'pop', self.current.fullName(), '->', self.current.parent.fullName()
##         else:
##             print 'pop', self.current.fullName(), '->', self.current.parent
        self.pop(self.current)

    def push(self, obj):
        self._stack.append(self.current)
        self.current = obj
        if isinstance(obj, model.Module):
            assert self.currentMod is None
            obj.parentMod = self.currentMod = obj
        elif self.currentMod is not None:
            if obj.parentMod is not None:
                assert obj.parentMod is self.currentMod
            else:
                obj.parentMod = self.currentMod
        else:
            assert obj.parentMod is None

    def pop(self, obj):
        assert self.current is obj, "%r is not %r"%(self.current, obj)
        self.current = self._stack.pop()
        if isinstance(obj, model.Module):
            self.currentMod = None

    def pushClass(self, name, docstring):
        return self._push(self.Class, name, docstring)
    def popClass(self):
        self._pop(self.Class)

    def pushModule(self, name, docstring):
        return self._push(self.Module, name, docstring)
    def popModule(self):
        self._pop(self.Module)

    def pushFunction(self, name, docstring):
        return self._push(self.Function, name, docstring)
    def popFunction(self):
        self._pop(self.Function)

    def pushPackage(self, name, docstring):
        return self._push(self.Package, name, docstring)
    def popPackage(self):
        self._pop(self.Package)

    def warning(self, type, detail):
        self.system._warning(self.current, type, detail)

    def _finalStateComputations(self):
        pass

    def processModuleAST(self, ast, moduleName):
        visitor.walk(ast, self.ModuleVistor(self, moduleName))

    def expandModname(self, modname, givewarning=True):
        c = self.current
        if '.' in modname:
            prefix, suffix = modname.split('.', 1)
            suffix = '.' + suffix
        else:
            prefix, suffix = modname, ''
        while c is not None and not isinstance(c, model.Package):
            c = c.parent
        while c is not None:
            if prefix in c.contents:
                break
            c = c.parent
        if c is not None:
            if givewarning:
                self.warning("local import", modname)
            return c.contents[prefix].fullName() + suffix
        else:
            return prefix + suffix

    def parseFile(self, filePath):
        if filePath in self.ast_cache:
            return self.ast_cache[filePath]
        try:
            ast = parseFile(filePath)
        except (SyntaxError, ValueError):
            self.warning("cannot parse", filePath)
            ast = None
        self.ast_cache[filePath] = ast
        return ast


    # if we assume:
    #
    # - that svn://divmod.org/trunk is checked out into ~/src/Divmod
    #
    # - that http://divmod.org/trac/browser/trunk is the trac URL to the
    #   above directory
    #
    # - that ~/src/Divmod/Nevow/nevow is passed to pydoctor as an
    #   "--add-package" argument
    #
    # we want to work out the sourceHref for nevow.flat.ten.  the answer
    # is http://divmod.org/trac/browser/trunk/Nevow/nevow/flat/ten.py.
    #
    # we can work this out by finding that Divmod is the top of the svn
    # checkout, and posixpath.join-ing the parts of the filePath that
    # follows that.
    #
    #  http://divmod.org/trac/browser/trunk
    #                          ~/src/Divmod/Nevow/nevow/flat/ten.py

    def setSourceHref(self, mod):
        if self.system.sourcebase is None:
            mod.sourceHref = None
            return

        trailing = []
        dir, fname = os.path.split(mod.filepath)
        while os.path.exists(os.path.join(dir, '.svn')):
            dir, dirname = os.path.split(dir)
            trailing.append(dirname)

        # now trailing[-1] would be 'Divmod' in the above example
        del trailing[-1]
        trailing.reverse()
        trailing.append(fname)

        mod.sourceHref = posixpath.join(mod.system.sourcebase, *trailing)


    def addModule(self, modpath):
        assert self.system.state in ['blank', 'preparse']
        fname = os.path.basename(modpath)
        modname = os.path.splitext(fname)[0]
        mod = self.pushModule(modname, None)
        self.system.progress("addModule", len(self.system.orderedallobjects),
                             None, "modules and packages discovered")
        mod.filepath = modpath
        mod.processed = False
        self.setSourceHref(mod)
        self.popModule()
        self.system.state = 'preparse'

    def preprocessDirectory(self, dirpath):
        assert self.system.state in ['blank', 'preparse']
        if not os.path.exists(dirpath):
            raise Exception, "package path %r does not exist!"%(dirpath,)
        if not os.path.exists(os.path.join(dirpath, '__init__.py')):
            raise Exception, "you must pass a package directory to preprocessDirectory"
        if os.path.basename(dirpath):
            package = self.pushPackage(os.path.basename(dirpath), None)
            package.filepath = dirpath
            self.setSourceHref(package)
        else:
            package = None
        for fname in os.listdir(dirpath):
            fullname = os.path.join(dirpath, fname)
            if os.path.isdir(fullname) and os.path.exists(os.path.join(fullname, '__init__.py')):
                self.preprocessDirectory(fullname)
            elif fname.endswith('.py') and not fname.startswith('.'):
                self.addModule(fullname)
        if package:
            self.popPackage()
        self.system.state = 'preparse'

    def analyseImports(self):
        assert self.system.state in ['preparse']
        modlist = list(self.system.objectsOfType(model.Module))
        for i, mod in enumerate(modlist):
            if mod.parent:
                self.push(mod.parent)
            isf = ImportFinder(self, mod)
            ast = self.parseFile(mod.filepath)
            if not ast:
                continue
            findAll(ast, mod)
            self.system.progress('analyseImports', i+1, len(modlist),
                                 "modules parsed")
            visitor.walk(ast, isf)
            if mod.parent:
{'fraction_non_alphanumeric': 0.10043209725850158, 'fraction_numerical': 0.0029538852134853404, 'mean_word_length': 1.6012192024384049, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5653620', 'n_tokens_mistral': 11420, 'n_tokens_neox': 11023, 'n_words': 2838}