<commit_before>#!/usr/bin/env python2
"""
ut
python cyth/cyth_script.py ~/code/fpath
cyth_script.py ~/code/ibeis/ibeis/model/hots
cyth_script.py "~/code/vtool/vtool"

"""
from __future__ import absolute_import, division, print_function
from six.moves import zip, map, range
import utool
import sys
from os.path import splitext, isfile
import ast
#import codegen  # NOQA
import astor
import re

#class CythTransformer(ast.NodeTransformer):
#    #


#BASE_CLASS = codegen.SourceGenerator
#BASE_CLASS = ast.NodeVisitor
#BASE_CLASS = astor.ExplicitNodeVisitor
BASE_CLASS = astor.codegen.SourceGenerator


# See astor/codegen.py for details
# https://github.com/berkerpeksag/astor/blob/master/astor/codegen.py


def is_docstring(node):
    return isinstance(node, ast.Expr) and isinstance(node.value, ast.Str)


class CythVisitor(BASE_CLASS):
    indent_level = 0
    emit = sys.stdout.write

    def __init__(self, indent_with=' ' * 4, add_line_information=False):
        super(CythVisitor, self).__init__(indent_with, add_line_information)

    def get_result(self):
        return ''.join(self.result)

    def process_args(self, args, vararg, kwarg, defaults=None):
        processed_argslist = map(self.visit, args)
        if vararg:
            processed_argslist.append('*%s' % vararg)
        if kwarg:
            processed_argslist.append('**%s' % kwarg)
        if defaults is not None:
            first_non_default = len(args) - len(defaults)
            for (ix, de) in enumerate(defaults):
                processed_argslist[first_non_default + ix] += '=%s' % self.visit(defaults[ix])
        return processed_argslist

    def signature(self, node, typedict={}):
        types_minus_sigtypes = typedict.copy()
        want_comma = []

        def write_comma():
            if want_comma:
                self.write(', ')
            else:
                want_comma.append(True)

        def loop_args(args, defaults):
            padding = [None] * (len(args) - len(defaults))
            for arg, default in zip(args, padding + defaults):
                if arg.id in typedict:
                    arg_ = typedict[arg.id] + ' ' + arg.id
                    types_minus_sigtypes.pop(arg.id)
                else:
                    arg_ = arg
                self.write(write_comma, arg_)
                self.conditional_write('=', default)

        loop_args(node.args, node.defaults)
        self.conditional_write(write_comma, '*', node.vararg)
        self.conditional_write(write_comma, '**', node.kwarg)

        kwonlyargs = getattr(node, 'kwonlyargs', None)
        if kwonlyargs:
            if node.vararg is None:
                self.write(write_comma, '*')
            loop_args(kwonlyargs, node.kw_defaults)
        return types_minus_sigtypes

    def parse_cythdef(self, cyth_def):
        """ Hacky string manipulation parsing """
        #ast.literal_eval(utool.unindent(cyth_def))

        typedict = {}
        cdef_mode = False
        current_indent = 0
        for line_ in cyth_def.splitlines():
            indent_level = utool.get_indentation(line_)
            # Check indentation
            if indent_level > current_indent:
                current_indent = indent_level
            elif indent_level > current_indent:
                current_indent = indent_level
                cdef_mode = False
            line = line_.strip()
            if line.startswith('#'):
                continue
            if len(line) == 0:
                continue
            # parse cdef
            if line.startswith('cdef:'):
                cdef_mode = True
                continue
            if cdef_mode or line.startswith('cdef '):
                assign_str = line.replace('cdef ', '')
                pos = assign_str.rfind(' ')
                type_ = assign_str[:pos]
                varstr = assign_str[(pos + 1):]
                typedict[varstr] = type_
        return typedict

    def typedict_to_cythdef(self, typedict):
        res = ['cdef:']
        for (id_, type_) in typedict.iteritems():
            #print("%s, %s" % (repr(id_), repr(type_)))
            res.append(self.indent_with + type_ + ' ' + id_)
        return res

    def parse_cyth_markup(self, docstr, toplevel=False):
        comment_str = docstr.strip()
        has_markup = comment_str.find('<CYTH') != -1
        # type returned_action = [`defines of string * (string, string) Hashtbl.t | `replace of string] option
        defines = False
        replace = False
        typedict = {}
        cyth_def = ''
        if has_markup:
            def_tag = '<CYTH>'
            end_tag = '</CYTH>'
            repl_tag = '<CYTH:REPLACE>'
            regex_flags = re.DOTALL | re.MULTILINE
            def_regex = re.compile(def_tag + '(.*)' + end_tag, regex_flags)
            repl_regex = re.compile(repl_tag + '(.*)' + end_tag, regex_flags)
            match = def_regex.search(comment_str)
            if match:
                cyth_def = match.group(1)
                defines = True
            match = repl_regex.search(comment_str)
            if match:
                cyth_def = match.group(1)
                replace = True
            #print('cyth_def: %r' % cyth_def)
            if replace or toplevel:
                cyth_def = utool.unindent(cyth_def)
                return ('replace', cyth_def)
            if defines:
                typedict = self.parse_cythdef(cyth_def)
                return ('defines', cyth_def, typedict)
        return None

    def visit_Module(self, node):
        for subnode in node.body:
            if is_docstring(subnode):
                print('Encountered global docstring: %s' % repr(subnode.value.s))
                action = self.parse_cyth_markup(subnode.value.s, toplevel=True)
                if action:
                    if action[0] == 'replace':
                        cyth_def = action[1]
                        self.newline(extra=1)
                        self.write(cyth_def)
            elif isinstance(subnode, ast.FunctionDef):
                self.visit(subnode)
            else:
                print('Skipping a global %r' % subnode.__class__)
        #return BASE_CLASS.visit_Module(self, node)

    def visit_FunctionDef(self, node):
        #super(CythVisitor, self).visit_FunctionDef(node)
        new_body = []
        cyth_action = None
        for stmt in node.body:
            if is_docstring(stmt):<commit_msg>Genericized parse_cyth_markup, making it easier to extend.<commit_after>#!/usr/bin/env python2
"""
ut
python cyth/cyth_script.py ~/code/fpath
cyth_script.py ~/code/ibeis/ibeis/model/hots
cyth_script.py "~/code/vtool/vtool"

"""
from __future__ import absolute_import, division, print_function
from six.moves import zip, map, range
import utool
import sys
from os.path import splitext, isfile
import ast
#import codegen  # NOQA
import astor
import re

#class CythTransformer(ast.NodeTransformer):
#    #


#BASE_CLASS = codegen.SourceGenerator
#BASE_CLASS = ast.NodeVisitor
#BASE_CLASS = astor.ExplicitNodeVisitor
BASE_CLASS = astor.codegen.SourceGenerator


# See astor/codegen.py for details
# https://github.com/berkerpeksag/astor/blob/master/astor/codegen.py


def is_docstring(node):
    return isinstance(node, ast.Expr) and isinstance(node.value, ast.Str)


class CythVisitor(BASE_CLASS):
    indent_level = 0
    emit = sys.stdout.write

    def __init__(self, indent_with=' ' * 4, add_line_information=False):
        super(CythVisitor, self).__init__(indent_with, add_line_information)

    def get_result(self):
        return ''.join(self.result)

    def process_args(self, args, vararg, kwarg, defaults=None):
        processed_argslist = map(self.visit, args)
        if vararg:
            processed_argslist.append('*%s' % vararg)
        if kwarg:
            processed_argslist.append('**%s' % kwarg)
        if defaults is not None:
            first_non_default = len(args) - len(defaults)
            for (ix, de) in enumerate(defaults):
                processed_argslist[first_non_default + ix] += '=%s' % self.visit(defaults[ix])
        return processed_argslist

    def signature(self, node, typedict={}):
        types_minus_sigtypes = typedict.copy()
        want_comma = []

        def write_comma():
            if want_comma:
                self.write(', ')
            else:
                want_comma.append(True)

        def loop_args(args, defaults):
            padding = [None] * (len(args) - len(defaults))
            for arg, default in zip(args, padding + defaults):
                if arg.id in typedict:
                    arg_ = typedict[arg.id] + ' ' + arg.id
                    types_minus_sigtypes.pop(arg.id)
                else:
                    arg_ = arg
                self.write(write_comma, arg_)
                self.conditional_write('=', default)

        loop_args(node.args, node.defaults)
        self.conditional_write(write_comma, '*', node.vararg)
        self.conditional_write(write_comma, '**', node.kwarg)

        kwonlyargs = getattr(node, 'kwonlyargs', None)
        if kwonlyargs:
            if node.vararg is None:
                self.write(write_comma, '*')
            loop_args(kwonlyargs, node.kw_defaults)
        return types_minus_sigtypes

    def parse_cythdef(self, cyth_def):
        """ Hacky string manipulation parsing """
        #ast.literal_eval(utool.unindent(cyth_def))

        typedict = {}
        cdef_mode = False
        current_indent = 0
        for line_ in cyth_def.splitlines():
            indent_level = utool.get_indentation(line_)
            # Check indentation
            if indent_level > current_indent:
                current_indent = indent_level
            elif indent_level > current_indent:
                current_indent = indent_level
                cdef_mode = False
            line = line_.strip()
            if line.startswith('#'):
                continue
            if len(line) == 0:
                continue
            # parse cdef
            if line.startswith('cdef:'):
                cdef_mode = True
                continue
            if cdef_mode or line.startswith('cdef '):
                assign_str = line.replace('cdef ', '')
                pos = assign_str.rfind(' ')
                type_ = assign_str[:pos]
                varstr = assign_str[(pos + 1):]
                typedict[varstr] = type_
        return typedict

    def typedict_to_cythdef(self, typedict):
        res = ['cdef:']
        for (id_, type_) in typedict.iteritems():
            #print("%s, %s" % (repr(id_), repr(type_)))
            res.append(self.indent_with + type_ + ' ' + id_)
        return res

    def parse_cyth_markup(self, docstr, toplevel=False):
        comment_str = docstr.strip()
        has_markup = comment_str.find('<CYTH') != -1
        # type returned_action = [`defines of string * (string, string) Hashtbl.t | `replace of string] option
        tags_to_actions = [
            ('<CYTH>', lambda cyth_def: ('defines', cyth_def, self.parse_cythdef(cyth_def))),
            ('<CYTH:REPLACE>', lambda cyth_def: ('replace', utool.unindent(cyth_def))),
        ]
        end_tag = '</CYTH>'
        regex_flags = re.DOTALL | re.MULTILINE
        regex_to_actions = [(re.compile(tag + '(.*?)' + end_tag, regex_flags), act)
                            for tag, act in tags_to_actions]
        if has_markup:
            if toplevel:
                comment_str = re.sub('<CYTH>', '<CYTH:REPLACE>', comment_str)
            for (regex, action) in regex_to_actions:
                match = regex.search(comment_str)
                if match:
                    cyth_def = match.group(1)
                    return action(cyth_def)
            utool.printex(NotImplementedError('no known cyth tag in docstring'),
                           iswarning=True,
                           key_list=['comment_str'])
        return None

    def visit_Module(self, node):
        for subnode in node.body:
            if is_docstring(subnode):
                print('Encountered global docstring: %s' % repr(subnode.value.s))
                action = self.parse_cyth_markup(subnode.value.s, toplevel=True)
                if action:
                    if action[0] == 'replace':
                        cyth_def = action[1]
                        self.newline(extra=1)
                        self.write(cyth_def)
            elif isinstance(subnode, ast.FunctionDef):
                self.visit(subnode)
            else:
                print('Skipping a global %r' % subnode.__class__)
        #return BASE_CLASS.visit_Module(self, node)

    def visit_FunctionDef(self, node):
        #super(CythVisitor, self).visit_FunctionDef(node)
        new_body = []
        cyth_action = None
        for stmt in node.body:
            if is_docstring(stmt):
{'fraction_non_alphanumeric': 0.10523441766128404, 'fraction_numerical': 0.001794211716982604, 'mean_word_length': 1.9621072088724585, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '614260', 'n_tokens_mistral': 3948, 'n_tokens_neox': 3815, 'n_words': 806}