<commit_before># Copyright BlueDynamics Alliance - http://bluedynamics.com
# GNU General Public License Version 2

import os
import _ast
import ast
import types
import copy
import exceptions
from odict import odict
from zope.component import provideHandler
from node.ext.directory.interfaces import IFileAddedEvent
from node.ext.python.interfaces import (
    CODESECTION_STARTTOKEN,
    CODESECTION_ENDTOKEN,
    Call,
    IModule,
    IFunction,
    IDocstring,
    IImport,
    IAttribute,
    IDecorator,
)
from node.ext.python.nodes import (
    Module,
    Docstring,
    ProtectedSection,
    Import,
    Attribute,
    Decorator,
    Function,
    Class,
    Block,
)


CODESECTION_STARTTOKEN = '##code-section '
CODESECTION_ENDTOKEN = '##/code-section '


POSITION_INSERT = 0
POSITION_AFTER = 1
POSITION_BEFORE = -1


class BaseParser(object):
    
    def __init__(self, model):
        self.model = model
    
    def __call__(self):
        raise NotImplemented(u'BaseParser does not implement ``__call__``')

    def _createastchild(self, astnode):
        if hasattr(astnode, 'lineno'):
            if astnode.lineno - 1 in self.model.readlines:
                return
        if isinstance(astnode, _ast.Import) \
          or isinstance(astnode, _ast.ImportFrom):
            import_ = Import(None, [], astnode, self.model.buffer)
            import_.readlines = self.model.readlines
            self.model[str(import_.uuid)] = import_
        elif isinstance(astnode, _ast.FunctionDef):
            function = Function(None, astnode, self.model.buffer)
            function.readlines = self.model.readlines
            self.model[str(function.uuid)] = function
            for childastnode in astnode.body:
                function.parser._createastchild(childastnode)
            function.initdecorators()
        elif isinstance(astnode, _ast.ClassDef):
            class_ = Class(None, astnode, self.model.buffer)
            class_.readlines = self.model.readlines
            self.model[str(class_.uuid)] = class_
            for childastnode in astnode.body:
                class_.parser._createastchild(childastnode)
        elif isinstance(astnode, _ast.Expr) \
          and isinstance(astnode.value, _ast.Str):
            docstring = Docstring(None, astnode, self.model.buffer)
            docstring.readlines = self.model.readlines
            self.model[str(docstring.uuid)] = docstring
        elif isinstance(astnode, _ast.Assign):
            if not IFunction.providedBy(self.model):
                attribute = Attribute([], None, astnode, self.model.buffer)
                attribute.readlines = self.model.readlines
                self.model[str(attribute.uuid)] = attribute
    
    def _marklines(self, *args):
        for arg in args:
            if not arg in self.model.readlines:
                self.model.readlines.append(arg)
    
    def _findbodyend(self, node):
        if not hasattr(node, '_fields'):
            return
        for fieldname in node._fields:
            fields = getattr(node, fieldname)
            if type(fields) is not types.ListType:
                fields = [fields]
            for field in fields:
                if hasattr(field, 'lineno'):
                    if field.lineno > self.model.bufend:
                        self.model.bufend = field.lineno
                self._findbodyend(field)
    
    def _checkbodyendsmultilined(self):
        pointer = self.model.bufend
        buflen = len(self.model.buffer)
        source = ''
        while True:
            if buflen - 1 <= pointer:
                break
            line = self.model.buffer[pointer].strip()
            source = '%s\n%s' % (source, line)
            try:
                compile(source, '<string>', 'exec')
                break
            except SyntaxError, e:
                pointer += 1
        self.model.bufend = pointer
    
    def _checkbodyendsprotected(self):
        pointer = self.model.bufend
        if pointer < len(self.model.buffer) - 1:
            next = self.model.buffer[pointer].strip()
            if next.startswith(CODESECTION_ENDTOKEN):
                self.model.bufend += 1
    
    def _findnodeposition(self, startlineno, endlineno, indent):
        values = [v for v in self.model.values() \
                  if not IDecorator.providedBy(v)]
        if not values:
            return self.model, POSITION_INSERT
        last = None
        for child in values:
            # inrange case
            if child.startlineno <= startlineno \
               and child.endlineno >= endlineno:
                return child.parser._findnodeposition(startlineno,
                                                      endlineno,
                                                      indent)
            # before case
            if  endlineno < child.startlineno:
                return child, POSITION_BEFORE
            last = child
        # after case - indent check
        if last.indent == indent:
            return last, POSITION_AFTER
        return self.model, POSITION_AFTER
    
    def _findindent(self, lines):
        indent = None
        for line in lines:
            if not line.strip():
                continue
            curindent = 0
            for char in line:
                if char != u' ':
                    break
                curindent += 1
            if indent is None or curindent < indent:
                indent = curindent
        if indent is None:
            return None
        return indent / 4 # XXX improve
    
    def _cutline(self, line):
        return line[self.model.indent * 4:] # XXX improve
    
    def _resolvearg(self, arg):
        if isinstance(arg, _ast.Str):
            return arg.s
        elif isinstance(arg, _ast.Num):
            return arg.n
        elif isinstance(arg, _ast.Name):
            return arg.id
        elif isinstance(arg, _ast.Call):
#            import pdb;pdb.set_trace()
            args = list()
            for a in arg.args:
                args.append(self._resolvearg(a))
            kwargs = odict()
            for keyword in arg.keywords:
                kwargs[keyword.arg] = self._resolvearg(keyword.value)
            return Call(name=arg.func.id, args=args, kwargs=kwargs)
        elif isinstance(arg, _ast.Tuple) or isinstance(arg, _ast.List):
            ret = list()
            for a in arg.elts:
                ret.append(self._resolvearg(a))
            if isinstance(arg, _ast.Tuple):<commit_msg>correctly handle call args in the form of x.y.f()<commit_after># Copyright BlueDynamics Alliance - http://bluedynamics.com
# GNU General Public License Version 2

import os
import _ast
import ast
import types
import copy
import exceptions
from odict import odict
from zope.component import provideHandler
from node.ext.directory.interfaces import IFileAddedEvent
from node.ext.python.interfaces import (
    CODESECTION_STARTTOKEN,
    CODESECTION_ENDTOKEN,
    Call,
    IModule,
    IFunction,
    IDocstring,
    IImport,
    IAttribute,
    IDecorator,
)
from node.ext.python.nodes import (
    Module,
    Docstring,
    ProtectedSection,
    Import,
    Attribute,
    Decorator,
    Function,
    Class,
    Block,
)


CODESECTION_STARTTOKEN = '##code-section '
CODESECTION_ENDTOKEN = '##/code-section '


POSITION_INSERT = 0
POSITION_AFTER = 1
POSITION_BEFORE = -1


class BaseParser(object):
    
    def __init__(self, model):
        self.model = model
    
    def __call__(self):
        raise NotImplemented(u'BaseParser does not implement ``__call__``')

    def _createastchild(self, astnode):
        if hasattr(astnode, 'lineno'):
            if astnode.lineno - 1 in self.model.readlines:
                return
        if isinstance(astnode, _ast.Import) \
          or isinstance(astnode, _ast.ImportFrom):
            import_ = Import(None, [], astnode, self.model.buffer)
            import_.readlines = self.model.readlines
            self.model[str(import_.uuid)] = import_
        elif isinstance(astnode, _ast.FunctionDef):
            function = Function(None, astnode, self.model.buffer)
            function.readlines = self.model.readlines
            self.model[str(function.uuid)] = function
            for childastnode in astnode.body:
                function.parser._createastchild(childastnode)
            function.initdecorators()
        elif isinstance(astnode, _ast.ClassDef):
            class_ = Class(None, astnode, self.model.buffer)
            class_.readlines = self.model.readlines
            self.model[str(class_.uuid)] = class_
            for childastnode in astnode.body:
                class_.parser._createastchild(childastnode)
        elif isinstance(astnode, _ast.Expr) \
          and isinstance(astnode.value, _ast.Str):
            docstring = Docstring(None, astnode, self.model.buffer)
            docstring.readlines = self.model.readlines
            self.model[str(docstring.uuid)] = docstring
        elif isinstance(astnode, _ast.Assign):
            if not IFunction.providedBy(self.model):
                attribute = Attribute([], None, astnode, self.model.buffer)
                attribute.readlines = self.model.readlines
                self.model[str(attribute.uuid)] = attribute
    
    def _marklines(self, *args):
        for arg in args:
            if not arg in self.model.readlines:
                self.model.readlines.append(arg)
    
    def _findbodyend(self, node):
        if not hasattr(node, '_fields'):
            return
        for fieldname in node._fields:
            fields = getattr(node, fieldname)
            if type(fields) is not types.ListType:
                fields = [fields]
            for field in fields:
                if hasattr(field, 'lineno'):
                    if field.lineno > self.model.bufend:
                        self.model.bufend = field.lineno
                self._findbodyend(field)
    
    def _checkbodyendsmultilined(self):
        pointer = self.model.bufend
        buflen = len(self.model.buffer)
        source = ''
        while True:
            if buflen - 1 <= pointer:
                break
            line = self.model.buffer[pointer].strip()
            source = '%s\n%s' % (source, line)
            try:
                compile(source, '<string>', 'exec')
                break
            except SyntaxError, e:
                pointer += 1
        self.model.bufend = pointer
    
    def _checkbodyendsprotected(self):
        pointer = self.model.bufend
        if pointer < len(self.model.buffer) - 1:
            next = self.model.buffer[pointer].strip()
            if next.startswith(CODESECTION_ENDTOKEN):
                self.model.bufend += 1
    
    def _findnodeposition(self, startlineno, endlineno, indent):
        values = [v for v in self.model.values() \
                  if not IDecorator.providedBy(v)]
        if not values:
            return self.model, POSITION_INSERT
        last = None
        for child in values:
            # inrange case
            if child.startlineno <= startlineno \
               and child.endlineno >= endlineno:
                return child.parser._findnodeposition(startlineno,
                                                      endlineno,
                                                      indent)
            # before case
            if  endlineno < child.startlineno:
                return child, POSITION_BEFORE
            last = child
        # after case - indent check
        if last.indent == indent:
            return last, POSITION_AFTER
        return self.model, POSITION_AFTER
    
    def _findindent(self, lines):
        indent = None
        for line in lines:
            if not line.strip():
                continue
            curindent = 0
            for char in line:
                if char != u' ':
                    break
                curindent += 1
            if indent is None or curindent < indent:
                indent = curindent
        if indent is None:
            return None
        return indent / 4 # XXX improve
    
    def _cutline(self, line):
        return line[self.model.indent * 4:] # XXX improve
    
    def _resolvearg(self, arg):
        if isinstance(arg, _ast.Str):
            return arg.s
        elif isinstance(arg, _ast.Num):
            return arg.n
        elif isinstance(arg, _ast.Name):
            return arg.id
        elif isinstance(arg, _ast.Call):
            args = list()
            for a in arg.args:
                args.append(self._resolvearg(a))
            kwargs = odict()
            for keyword in arg.keywords:
                kwargs[keyword.arg] = self._resolvearg(keyword.value)
            try:
                return Call(name=arg.func.id, args=args, kwargs=kwargs)
            except AttributeError:
                return Call(name=arg.func.attr, args=args, kwargs=kwargs)
        elif isinstance(arg, _ast.Tuple) or isinstance(arg, _ast.List):
            ret = list()
            for a in arg.elts:
                ret.append(self._resolvearg(a))
            if isinstance(arg, _ast.Tuple):
{'fraction_non_alphanumeric': 0.08679072605247101, 'fraction_numerical': 0.0019829164124466137, 'mean_word_length': 1.7358648028374712, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5299040', 'n_tokens_mistral': 3624, 'n_tokens_neox': 3548, 'n_words': 888}