<commit_before>from __future__ import print_function

import __builtin__

import sys
import os
import ntpath
import datetime as dt
import re
import tokenize
import keyword
from collections import OrderedDict

import random
import string

import isort

from VisualFoxpro9Visitor import VisualFoxpro9Visitor

import vfpfunc

class RedirectedBuiltin(object):
    def __init__(self, name):
        self.name = name
        self.func = getattr(__builtin__, name)

    def __call__(self, *args, **kwargs):
        try:
            return self.func(*args, **kwargs)
        except:
            return make_func_code(self.name, *args, **kwargs)

for func in ('chr', 'int', 'str'):
    globals()[func] = RedirectedBuiltin(func)

if sys.version_info >= (3,):
    unicode=str

def isinstance(obj, istype):
    if not __builtin__.isinstance(istype, tuple):
        istype = (istype,)
    istype = tuple(x.func if __builtin__.isinstance(x, RedirectedBuiltin) else x for x in istype)
    return __builtin__.isinstance(obj, istype)

def random_variable(length=10):
    return '_' + ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(length))

class CodeStr(unicode):
    def __repr__(self):
        return unicode(self)

    def __add__(self, val):
        return CodeStr('{} + {}'.format(self, repr(val)))

    def __sub__(self, val):
        return CodeStr('{} - {}'.format(self, repr(val)))

    def __mul__(self, val):
        return CodeStr('{} * {}'.format(self, repr(val)))

def make_func_code(funcname, *args, **kwargs):
    args = [repr(x) for x in args]
    args += ['{}={}'.format(key, repr(kwargs[key])) for key in kwargs]
    return CodeStr('{}({})'.format(funcname, ', '.join(args)))

def string_type(val):
    return isinstance(val, (str, unicode)) and not isinstance(val, CodeStr)

def create_string(val):
    try:
        return str(val)
    except UnicodeEncodeError: #can this happen?
        return val

def add_args_to_code(codestr, args):
    return CodeStr(codestr.format(*[repr(arg) for arg in args]))

class PythonConvertVisitor(VisualFoxpro9Visitor):
    def __init__(self, filename):
        super(PythonConvertVisitor, self).__init__()
        self.filename = filename
        self.imports = []
        self.scope = {}
        self._saved_scope = None
        self._scope_count = 0
        self.withid = ''
        self.class_list = []
        self.function_list = []
        self.used_scope = False

    def visit(self, ctx):
        if ctx:
            return super(type(self), self).visit(ctx)

    def enable_scope(self, enabled):
        self._scope_count = max(self._scope_count + 1 - 2*int(enabled), 0)
        if enabled and self._scope_count == 0:
            if self._saved_scope:
                self.scope = self._saved_scope
            else:
                self.new_scope()
        else:
            if self.scope:
                self._saved_scope = self.scope
            self.scope = None

    def new_scope(self):
        self._scope_count = 0
        self.scope = {}

    def delete_scope(self):
        self._scope_count = 0
        self.scope = None

    def has_scope(self):
        return self.scope is not None

    def visit_with_disabled_scope(self, ctx):
        self.enable_scope(False)
        retval = self.visit(ctx)
        self.enable_scope(True)
        return retval

    def getCtxText(self, ctx):
        start, stop = ctx.getSourceInterval()
        tokens = ctx.parser._input.tokens[start:stop+1]
        return ''.join(token.text for token in tokens)

    def visitPrg(self, ctx):
        self.scope = None

        if ctx.classDef():
            self.class_list = [self.visit(classDef.classDefStart())[0] for classDef in ctx.classDef()]
        if ctx.funcDef():
            self.function_list = [self.visit(funcdef.funcDefStart())[0] for funcdef in ctx.funcDef()]

        self.imports = ['from __future__ import division, print_function']
        defs = []

        for child in ctx.children:
            if isinstance(child, ctx.parser.FuncDefContext):
                self.used_scope = False
                funcname, parameters, funcbody = self.visit(child)
                defs += [CodeStr('def {}({}):'.format(funcname, ', '.join([str(repr(p)) + '=False' for p in parameters]))), funcbody]
            else:
                defs += self.visit(child)

        imports = isort.SortImports(file_contents='\n'.join(set(self.imports))).output.splitlines()
        return  [CodeStr(imp) for imp in imports] + defs

    def visitLine(self, ctx):
        retval = self.visitChildren(ctx)
        if retval is None:
            lines = self.getCtxText(ctx)
            print(lines)
            retval = [CodeStr('#FIX ME: {}'.format(line)) for line in lines.split('\n') if line]
        return retval if isinstance(retval, list) else [retval]

    andfix = re.compile('^&&')
    frontfix = re.compile('^\**')
    endfix = re.compile('\**$')

    def visitLineComment(self, ctx):
        repl = lambda x: '#' * len(x.group())
        comment = ctx.getText().split('\n')[0].strip()
        if len(comment) == 0:
            return ''
        comment = self.andfix.sub('', comment)
        comment = self.frontfix.sub(repl, comment)
        return [CodeStr(self.endfix.sub(repl, comment))]

    def visitCmdStmt(self, ctx):
        return self.visit(ctx.cmd() or ctx.setup())

    def visitLines(self, ctx):
        retval = sum((self.visit(line) for line in ctx.line()), [])
        def badline(line):
            return line.startswith('#') or not line if hasattr(line, 'startswith') else not line
        if not retval or all(badline(l) for l in retval):
            retval.append(CodeStr('pass'))
        return retval

    def visitClassDef(self, ctx):
        assignments = []
        self.used_scope = False
        for stmt in ctx.classDefProperty():
            assignment = self.visit(stmt)
            if isinstance(stmt, ctx.parser.ClassDefLineCommentContext) and assignment:
                assignments.append(assignment)
            else:
                assignments += assignment
        assign_scope = self.used_scope

        comments = []
        for comment in ctx.lineComment():
            comments.append(self.visit(comment) + [comment.start.line])

        funcs = OrderedDict()
        for funcdef in ctx.funcDef():
            self.used_scope = False
            funcname, parameters, funcbody = self.visit(funcdef)
            if funcname == 'init' and assign_scope and not self.used_scope:
                self.used_scope = True
                funcname, parameters, funcbody = self.visit(funcdef)
            parameters = [CodeStr('self')] + [add_args_to_code('{}=False', (p,)) for p in parameters]
            if '.' in funcname:
                newfuncname = funcname.replace('.', '_')
                assignments.append(CodeStr('def {}({}):'.format(newfuncname, ', '.join(parameters))))
                assignments.append(funcbody)
                assignments.append(CodeStr('self.{} = {}'.format(funcname, newfuncname)))
            else:
                funcs.update({funcname: [parameters, funcbody, funcdef.start.line]})

        if assignments and 'init' not in funcs:
            funcs['init'] = [[CodeStr('self')], [], float('inf')]

        try:
            funcbody = funcs['init'][1]
            if funcbody:
                funcs['init'][1] = [funcbody[0]] + assignments + funcbody[1:]
            else:
                self.used_scope = assign_scope
                funcs['init'][1] = self.modify_func_body(assignments)
        except KeyError:
            pass

        classname, supername = self.visit(ctx.classDefStart())
        retval = [CodeStr('class {}({}):'.format(classname, supername))]
        if funcs:
            for funcname in funcs:
                parameters, funcbody, line_number = funcs[funcname]
                while comments and comments[0][1] < line_number:
                    retval.append([comments.pop(0)[0]])
                func = make_func_code(funcname, *parameters)
                retval.append([add_args_to_code('def {}:', (func,)), funcbody])
        else:
            retval.append([CodeStr('pass')])
        while comments:
            retval.append([comments.pop(0)[0]])

        return retval

    def visitClassDefStart(self, ctx):
        names = [CodeStr(self.visit(identifier).title()) for identifier in ctx.identifier()]
        if len(names) < 2:
            names.append('Custom')
        classname, supername = names
        if hasattr(vfpfunc, classname):
            raise Exception(classname + ' is a reserved classname')
        if hasattr(vfpfunc, supername):
            self.imports.append('from vfp2py import vfpfunc')
            supername = add_args_to_code('{}.{}', (CodeStr('vfpfunc'), supername))
        return classname, supername

    def visitClassDefAssign(self, ctx):
        #FIXME - come up with a less hacky way to make this work
        args1 = self.visit_with_disabled_scope(ctx.assign())
        used_scope = self.used_scope
        args2 = self.visit(ctx.assign())
        args = []
        for arg1, arg2 in zip(args1, args2):
            ident = arg1[:arg1.find(' = ')]
            value = arg2[arg2.find(' = '):]
            if 'vfpfunc.variable' in value or 'vfpfunc.function' in value:
                used_scope = True
            args.append(ident + value)
        self.used_scope = used_scope
        return [CodeStr('self.' + arg) for arg in args]

    def visitClassDefAddObject(self, ctx):
        name = self.visit_with_disabled_scope(ctx.identifier())
        keywords = [self.visit_with_disabled_scope(idAttr) for idAttr in ctx.idAttr()[1:]]
        kwargs = {key: self.visit(expr) for key, expr in zip(keywords, ctx.expr())}
        objtype = create_string(self.visit_with_disabled_scope(ctx.idAttr()[0])).title()
        newobj = self.func_call('createobject', objtype, **kwargs)
        return [add_args_to_code('self.{} = {}', (name, newobj))]

    def visitNodefault(self, ctx):
        return []

    def visitFuncDefStart(self, ctx):
        params = self.visit_with_disabled_scope(ctx.parameters()) or []
        params += self.visit_with_disabled_scope(ctx.parameterDef()) or []
        return self.visit(ctx.idAttr2()), params

    def visitParameterDef(self, ctx):
        return self.visit(ctx.parameters())

    def visitParameter(self, ctx):
        return self.visit(ctx.idAttr()[0])

    def visitParameters(self, ctx):
        return [self.visit(parameter) for parameter in ctx.parameter()]

    def modify_func_body(self, body):
        while len(body) > 0 and (not body[-1] or (isinstance(body[-1], CodeStr) and (body[-1] == 'return'))):
            body.pop()
        if len(body) == 0:
            body.append(CodeStr('pass'))
        if not self.used_scope:
            return body
        while CodeStr('pass') in body:
            body.pop(body.index(CodeStr('pass')))
        def fix_returns(lines):
            newbody = []
            for line in lines:
                if isinstance(line, list):
                    newbody.append(fix_returns(line))
                elif isinstance(line, CodeStr) and line.startswith('return '):
                    return_value = CodeStr(line[7:])
                    if 'vfpfunc.variable[' in return_value or 'vfpfunc.function[' in return_value:
                        newbody.append(add_args_to_code('function_return_value = {}', [return_value]))
                        newbody.append(CodeStr('vfpfunc.variable.popscope()'))
                        newbody.append(CodeStr('return function_return_value'))
                    else:
                        newbody.append(CodeStr('vfpfunc.variable.popscope()'))
                        newbody.append(line)
                else:
                    newbody.append(line)
            return newbody
        body = [CodeStr('vfpfunc.variable.pushscope()')] + fix_returns(body)
        if isinstance(body[-1], CodeStr) and body[-1].startswith('return '):
            return body
        body.append(CodeStr('vfpfunc.variable.popscope()'))
        return body

    def visitFuncDef(self, ctx):
        name, parameters = self.visit(ctx.funcDefStart())
        self.new_scope()
        self.scope.update({key: False for key in parameters})
        body = self.visit(ctx.lines())
        self.delete_scope()
        body = self.modify_func_body(body)
        return name, parameters, body

    def visitPrintStmt(self, ctx):
        kwargs = {}
        if len([child for child in ctx.children if child.getText() == '?']) > 1:
            kwargs['end'] = ''
        return [make_func_code('print', *(self.visit(ctx.args()) or []), **kwargs)]

    def visitIfStart(self, ctx):
        return self.visit(ctx.expr())

    def visitIfStmt(self, ctx):
        evaluation = self.visit(ctx.ifStart())

        ifBlock = self.visit(ctx.ifBody)
        retval = [CodeStr('if {}:'.format(evaluation)), ifBlock]

        if ctx.elseBody:
            elseBlock = self.visit(ctx.elseBody)
            retval += [CodeStr('else:'), elseBlock]

        return retval

    def visitCaseStmt(self, ctx):
        n = 0
        retval = []
        for elem in ctx.caseElement():
            if elem.lineComment():
                retval += self.visit(elem.lineComment())
            else:
                expr, lines = self.visit(elem.singleCase())
                if n == 0:
                    retval += [CodeStr('if {}:'.format(expr)), lines]
                else:
                    retval += [CodeStr('elif {}:'.format(expr)), lines]
                n += 1
        if n == 0:
            retval += [CodeStr('if True:'), [CodeStr('pass')]]
        if ctx.otherwise():
            retval += [CodeStr('else:'), self.visit(ctx.otherwise())]
        return retval

    def visitSingleCase(self, ctx):
        return self.visit(ctx.caseExpr()), self.visit(ctx.lines())

    def visitCaseExpr(self, ctx):
        return self.visit(ctx.expr())

    def visitOtherwise(self, ctx):
        return self.visit(ctx.lines())

    def visitForStart(self, ctx):
        loopvar = self.visit_with_disabled_scope(ctx.idAttr())
        self.scope[loopvar] = False
        if ctx.EACH():
            iterator = self.visit(ctx.expr(0))
            return add_args_to_code('for {} in {}:', (loopvar, iterator))
        loop_start = int(self.visit(ctx.loopStart))
        loop_stop = int(self.visit(ctx.loopStop)) + 1
        if ctx.loopStep:
            loop_step = int(self.visit(ctx.loopStep))
            return CodeStr('for {} in range({}, {}, {}):'.format(loopvar, loop_start, loop_stop, loop_step))
        else:
            return CodeStr('for {} in range({}, {}):'.format(loopvar, loop_start, loop_stop))

    def visitForStmt(self, ctx):
        return [self.visit(ctx.forStart()), self.visit(ctx.lines())]

    def visitWhileStart(self, ctx):
        return CodeStr('while {}:'.format(self.visit(ctx.expr())))

    def visitWhileStmt(self, ctx):
        return [self.visit(ctx.whileStart()), self.visit(ctx.lines())]

    def visitWithStmt(self, ctx):
        self.withid = self.visit(ctx.idAttr())
        lines = self.visit(ctx.lines())
        self.withid = ''
        return lines

    def visitScanStmt(self, ctx):
        lines = self.visit(ctx.lines())
        lines.append(make_func_code('vfpfunc.db.skip', None, 1))
        if ctx.FOR():
            condition = self.visit(ctx.expr())
            lines = [add_args_to_code('if not {}:', [condition]), [CodeStr('continue')]] + lines
        save_variable_name = CodeStr(random_variable(length=10) + '_current_record')
        save_recno = CodeStr('{} = {}'.format(save_variable_name, 'vfpfunc.db.recno()'))
        return_recno = make_func_code('vfpfunc.db.goto', None, CodeStr(save_variable_name))
        return [save_recno, CodeStr('while not vfpfunc.db.eof():'), lines, return_recno]

    def visitTryStmt(self, ctx):
        try_lines = self.visit(ctx.tryLines)
        finally_lines = self.visit(ctx.finallyLines) or []
        if not ctx.CATCH():
            return try_lines + finally_lines

        identifier = self.visit(ctx.identifier())
        if identifier:
            self.scope[identifier] = False

        try_lines = [CodeStr('try:'), try_lines]

        if identifier:
            catch_lines = [CodeStr('except Exception as {}:'.format(identifier))]
            catch_lines.append(make_func_code('#vfpfunc.pyexception_to_foxexception', identifier))
        else:
            catch_lines = [CodeStr('except:')]
        catch_lines.append(self.visit(ctx.catchLines))

        finally_lines = [CodeStr('finally:'), finally_lines] if finally_lines else []

        if ctx.identifier():
            del self.scope[identifier]

        return try_lines + catch_lines + finally_lines

    def visitBreakLoop(self, ctx):
        return CodeStr('break')

    def visitContinueLoop(self, ctx):
        return CodeStr('continue')

    def visitDeclaration(self, ctx):
        if ctx.ARRAY() or ctx.DIMENSION() or ctx.DEFINE():
            name = self.visit(ctx.identifier())
            value = self.visit(ctx.arrayIndex())
            if ctx.LOCAL():
                self.scope[name] = False
                array = make_func_code('vfpfunc.Array', *value)
                return [
                    add_args_to_code('{} = {}', [name, array])
                ]
            func = 'vfpfunc.array'
            kwargs = {'public': True} if ctx.PUBLIC() else {}
            self.used_scope = True
            return make_func_code(func, *([str(name)] + value), **kwargs)
        else:
            names = self.visit_with_disabled_scope(ctx.parameters())
            if ctx.LOCAL():
                for name in names:
                    self.scope[name] = False
                return CodeStr(' = '.join([repr(arg) for arg in (names + [False])]) + ' #LOCAL Declaration')
            self.used_scope = True
            func = 'vfpfunc.variable.add_public' if ctx.PUBLIC() else 'vfpfunc.variable.add_private'
            return make_func_code(func, *[str(name) for name in names])

    def visitAssign(self, ctx):
        value = self.visit(ctx.expr())
        args = []
        for var in ctx.idAttr():
            trailer = self.visit(var.trailer()) or []
            if len(trailer) > 0 and isinstance(trailer[-1], list):
                identifier = self.visit(ctx.idAttr()[0].identifier())
                arg = self.createIdAttr(identifier, trailer[:-1] + [[]])
                args.append('{}[{}]'.format(arg[:-2], ','.join(repr(x) for x in trailer[-1])))
            else:
                args.append(self.visit(var))
        if len(args) == 1:
            try:
                if isinstance(value, CodeStr):
                    for op in ('+', '-', '*', '/'):
                        start = str(args[0]) + ' ' + op + ' '
                        if value.startswith(start):
                            return [CodeStr('{} {}= {}'.format(args[0], op, value[len(start):]))]
            except Exception as e:
                pass
        return [CodeStr(' = '.join(args + [repr(value)]))]

    def visitArgs(self, ctx):
        return [self.visit(c) for c in ctx.expr()]

    def visitSpecialArgs(self, ctx):
        return [self.visit(c) for c in ctx.specialExpr()]

    def visitComparison(self, ctx):
        symbol_dict = {ctx.parser.GREATERTHAN: '>',
                       ctx.parser.GTEQ: '>=',
                       ctx.parser.LESSTHAN: '<',
                       ctx.parser.LTEQ: '<=',
                       ctx.parser.NOTEQUALS: '!=',
                       ctx.parser.HASH: '!=',
                       ctx.parser.EQUALS: '==',
                       ctx.parser.DOUBLEEQUALS: '==',
                       ctx.parser.DOLLAR: 'in',
                       ctx.parser.OR: 'or',
                       ctx.parser.AND: 'and'
                      }
        left = self.visit(ctx.expr(0))
        right = self.visit(ctx.expr(1))
        symbol = symbol_dict[ctx.op.type]
        return CodeStr('{} {} {}'.format(repr(left), symbol, repr(right)))

    def visitBooleanOperation(self, ctx):
        return self.visitComparison(ctx)

    def visitUnaryNegation(self, ctx):
        return add_args_to_code('{}' if ctx.op.type == ctx.parser.PLUS_SIGN else '-{}', (self.visit(ctx.expr()),))

    def visitBooleanNegation(self, ctx):
        return CodeStr('not {}'.format(repr(self.visit(ctx.expr()))))

    def func_call(self, funcname, *args, **kwargs):
        if not kwargs and len(args) == 1 and isinstance(args[0], (list, tuple)):
            args = args[0]
        args = list(args)
        if funcname in self.function_list:
            return make_func_code(funcname, *args)
        if funcname == 'chr' and len(args) == 1:
            return chr(int(args[0]))
        if funcname == 'space' and len(args) == 1:
            return int(args[0]) * ' '
        if funcname == 'asc':
            return make_func_code('ord', CodeStr(str(repr(args[0])) + '[0]'))
        if funcname == 'len':
            return make_func_code('len', *args)
        if funcname == 'alen':
            if len(args) == 1:
                return make_func_code('len', *args)
            else:
                args[1] = int(args[1])
                return add_args_to_code('{}.alen({})', args)
        if funcname == 'ascan':
            if len(args) == 3:
                args = [add_args_to_code('{}[{}:]', [args[0], args[2]]), args[1]]
            elif len(args) == 4:
                args = [add_args_to_code('{}[{}:({} + {})]', [args[0], args[2], args[2], args[3]]), args[1]]
            if len(args) == 2:
                return add_args_to_code('{}.index({})', args)
        if funcname == 'empty':
            return add_args_to_code('(not {} if {} is not None else False)', args + args)
        if funcname == 'occurs':
            return add_args_to_code('{}.count({})', reversed(args))
        if funcname in ('at', 'rat'):
            funcname = {
                'at': 'find',
                'rat': 'rfind',
            }[funcname]
            return add_args_to_code('{}.{}({})', [args[1], CodeStr(funcname), args[0]]) + 1
        if funcname in ('repli', 'replicate'):
            args[1:] = [int(arg) for arg in args[1:]]
            return add_args_to_code('({} * {})', args)
        if funcname in ('date', 'datetime', 'time') and len(args) == 0:
            self.imports.append('import datetime as dt')
            if funcname == 'date':
                return make_func_code('dt.datetime.now().date')
            elif funcname == 'datetime':
                return make_func_code('dt.datetime.now')
            elif funcname == 'time':
                return make_func_code('dt.datetime.now().time().strftime', '%H:%M:%S')
        if funcname in ('year', 'month', 'day', 'hour', 'minute', 'sec', 'dow', 'cdow', 'cmonth'):
            funcname = {
                'sec': 'second',
                'dow': 'weekday()',
                'cdow': "strftime('%A')",
                'cmonth': "strftime('%B')",
            }.get(funcname, funcname)
            retval = add_args_to_code('{}.{}', [args[0], CodeStr(funcname)])
            if funcname in ('weekday()'):
                self.imports.append('from vfp2py import vfpfunc')
                return make_func_code('vfpfunc.dow_fix', retval, *args[1:])
            return retval
        if funcname == 'dtoc':
            if len(args) == 1 or args[1] == 1:
                if len(args) > 1 and args[1] == 1:
                    args[1] = '%Y%m%d'
                else:
                    args.append('%m/%d/%Y')
                return make_func_code('{}.{}'.format(args[0], 'strftime'), args[1])
        if funcname == 'iif' and len(args) == 3:<commit_msg>Add missing datetime import for most date/datetime function inlining.<commit_after>from __future__ import print_function

import __builtin__

import sys
import os
import ntpath
import datetime as dt
import re
import tokenize
import keyword
from collections import OrderedDict

import random
import string

import isort

from VisualFoxpro9Visitor import VisualFoxpro9Visitor

import vfpfunc

class RedirectedBuiltin(object):
    def __init__(self, name):
        self.name = name
        self.func = getattr(__builtin__, name)

    def __call__(self, *args, **kwargs):
        try:
            return self.func(*args, **kwargs)
        except:
            return make_func_code(self.name, *args, **kwargs)

for func in ('chr', 'int', 'str'):
    globals()[func] = RedirectedBuiltin(func)

if sys.version_info >= (3,):
    unicode=str

def isinstance(obj, istype):
    if not __builtin__.isinstance(istype, tuple):
        istype = (istype,)
    istype = tuple(x.func if __builtin__.isinstance(x, RedirectedBuiltin) else x for x in istype)
    return __builtin__.isinstance(obj, istype)

def random_variable(length=10):
    return '_' + ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(length))

class CodeStr(unicode):
    def __repr__(self):
        return unicode(self)

    def __add__(self, val):
        return CodeStr('{} + {}'.format(self, repr(val)))

    def __sub__(self, val):
        return CodeStr('{} - {}'.format(self, repr(val)))

    def __mul__(self, val):
        return CodeStr('{} * {}'.format(self, repr(val)))

def make_func_code(funcname, *args, **kwargs):
    args = [repr(x) for x in args]
    args += ['{}={}'.format(key, repr(kwargs[key])) for key in kwargs]
    return CodeStr('{}({})'.format(funcname, ', '.join(args)))

def string_type(val):
    return isinstance(val, (str, unicode)) and not isinstance(val, CodeStr)

def create_string(val):
    try:
        return str(val)
    except UnicodeEncodeError: #can this happen?
        return val

def add_args_to_code(codestr, args):
    return CodeStr(codestr.format(*[repr(arg) for arg in args]))

class PythonConvertVisitor(VisualFoxpro9Visitor):
    def __init__(self, filename):
        super(PythonConvertVisitor, self).__init__()
        self.filename = filename
        self.imports = []
        self.scope = {}
        self._saved_scope = None
        self._scope_count = 0
        self.withid = ''
        self.class_list = []
        self.function_list = []
        self.used_scope = False

    def visit(self, ctx):
        if ctx:
            return super(type(self), self).visit(ctx)

    def enable_scope(self, enabled):
        self._scope_count = max(self._scope_count + 1 - 2*int(enabled), 0)
        if enabled and self._scope_count == 0:
            if self._saved_scope:
                self.scope = self._saved_scope
            else:
                self.new_scope()
        else:
            if self.scope:
                self._saved_scope = self.scope
            self.scope = None

    def new_scope(self):
        self._scope_count = 0
        self.scope = {}

    def delete_scope(self):
        self._scope_count = 0
        self.scope = None

    def has_scope(self):
        return self.scope is not None

    def visit_with_disabled_scope(self, ctx):
        self.enable_scope(False)
        retval = self.visit(ctx)
        self.enable_scope(True)
        return retval

    def getCtxText(self, ctx):
        start, stop = ctx.getSourceInterval()
        tokens = ctx.parser._input.tokens[start:stop+1]
        return ''.join(token.text for token in tokens)

    def visitPrg(self, ctx):
        self.scope = None

        if ctx.classDef():
            self.class_list = [self.visit(classDef.classDefStart())[0] for classDef in ctx.classDef()]
        if ctx.funcDef():
            self.function_list = [self.visit(funcdef.funcDefStart())[0] for funcdef in ctx.funcDef()]

        self.imports = ['from __future__ import division, print_function']
        defs = []

        for child in ctx.children:
            if isinstance(child, ctx.parser.FuncDefContext):
                self.used_scope = False
                funcname, parameters, funcbody = self.visit(child)
                defs += [CodeStr('def {}({}):'.format(funcname, ', '.join([str(repr(p)) + '=False' for p in parameters]))), funcbody]
            else:
                defs += self.visit(child)

        imports = isort.SortImports(file_contents='\n'.join(set(self.imports))).output.splitlines()
        return  [CodeStr(imp) for imp in imports] + defs

    def visitLine(self, ctx):
        retval = self.visitChildren(ctx)
        if retval is None:
            lines = self.getCtxText(ctx)
            print(lines)
            retval = [CodeStr('#FIX ME: {}'.format(line)) for line in lines.split('\n') if line]
        return retval if isinstance(retval, list) else [retval]

    andfix = re.compile('^&&')
    frontfix = re.compile('^\**')
    endfix = re.compile('\**$')

    def visitLineComment(self, ctx):
        repl = lambda x: '#' * len(x.group())
        comment = ctx.getText().split('\n')[0].strip()
        if len(comment) == 0:
            return ''
        comment = self.andfix.sub('', comment)
        comment = self.frontfix.sub(repl, comment)
        return [CodeStr(self.endfix.sub(repl, comment))]

    def visitCmdStmt(self, ctx):
        return self.visit(ctx.cmd() or ctx.setup())

    def visitLines(self, ctx):
        retval = sum((self.visit(line) for line in ctx.line()), [])
        def badline(line):
            return line.startswith('#') or not line if hasattr(line, 'startswith') else not line
        if not retval or all(badline(l) for l in retval):
            retval.append(CodeStr('pass'))
        return retval

    def visitClassDef(self, ctx):
        assignments = []
        self.used_scope = False
        for stmt in ctx.classDefProperty():
            assignment = self.visit(stmt)
            if isinstance(stmt, ctx.parser.ClassDefLineCommentContext) and assignment:
                assignments.append(assignment)
            else:
                assignments += assignment
        assign_scope = self.used_scope

        comments = []
        for comment in ctx.lineComment():
            comments.append(self.visit(comment) + [comment.start.line])

        funcs = OrderedDict()
        for funcdef in ctx.funcDef():
            self.used_scope = False
            funcname, parameters, funcbody = self.visit(funcdef)
            if funcname == 'init' and assign_scope and not self.used_scope:
                self.used_scope = True
                funcname, parameters, funcbody = self.visit(funcdef)
            parameters = [CodeStr('self')] + [add_args_to_code('{}=False', (p,)) for p in parameters]
            if '.' in funcname:
                newfuncname = funcname.replace('.', '_')
                assignments.append(CodeStr('def {}({}):'.format(newfuncname, ', '.join(parameters))))
                assignments.append(funcbody)
                assignments.append(CodeStr('self.{} = {}'.format(funcname, newfuncname)))
            else:
                funcs.update({funcname: [parameters, funcbody, funcdef.start.line]})

        if assignments and 'init' not in funcs:
            funcs['init'] = [[CodeStr('self')], [], float('inf')]

        try:
            funcbody = funcs['init'][1]
            if funcbody:
                funcs['init'][1] = [funcbody[0]] + assignments + funcbody[1:]
            else:
                self.used_scope = assign_scope
                funcs['init'][1] = self.modify_func_body(assignments)
        except KeyError:
            pass

        classname, supername = self.visit(ctx.classDefStart())
        retval = [CodeStr('class {}({}):'.format(classname, supername))]
        if funcs:
            for funcname in funcs:
                parameters, funcbody, line_number = funcs[funcname]
                while comments and comments[0][1] < line_number:
                    retval.append([comments.pop(0)[0]])
                func = make_func_code(funcname, *parameters)
                retval.append([add_args_to_code('def {}:', (func,)), funcbody])
        else:
            retval.append([CodeStr('pass')])
        while comments:
            retval.append([comments.pop(0)[0]])

        return retval

    def visitClassDefStart(self, ctx):
        names = [CodeStr(self.visit(identifier).title()) for identifier in ctx.identifier()]
        if len(names) < 2:
            names.append('Custom')
        classname, supername = names
        if hasattr(vfpfunc, classname):
            raise Exception(classname + ' is a reserved classname')
        if hasattr(vfpfunc, supername):
            self.imports.append('from vfp2py import vfpfunc')
            supername = add_args_to_code('{}.{}', (CodeStr('vfpfunc'), supername))
        return classname, supername

    def visitClassDefAssign(self, ctx):
        #FIXME - come up with a less hacky way to make this work
        args1 = self.visit_with_disabled_scope(ctx.assign())
        used_scope = self.used_scope
        args2 = self.visit(ctx.assign())
        args = []
        for arg1, arg2 in zip(args1, args2):
            ident = arg1[:arg1.find(' = ')]
            value = arg2[arg2.find(' = '):]
            if 'vfpfunc.variable' in value or 'vfpfunc.function' in value:
                used_scope = True
            args.append(ident + value)
        self.used_scope = used_scope
        return [CodeStr('self.' + arg) for arg in args]

    def visitClassDefAddObject(self, ctx):
        name = self.visit_with_disabled_scope(ctx.identifier())
        keywords = [self.visit_with_disabled_scope(idAttr) for idAttr in ctx.idAttr()[1:]]
        kwargs = {key: self.visit(expr) for key, expr in zip(keywords, ctx.expr())}
        objtype = create_string(self.visit_with_disabled_scope(ctx.idAttr()[0])).title()
        newobj = self.func_call('createobject', objtype, **kwargs)
        return [add_args_to_code('self.{} = {}', (name, newobj))]

    def visitNodefault(self, ctx):
        return []

    def visitFuncDefStart(self, ctx):
        params = self.visit_with_disabled_scope(ctx.parameters()) or []
        params += self.visit_with_disabled_scope(ctx.parameterDef()) or []
        return self.visit(ctx.idAttr2()), params

    def visitParameterDef(self, ctx):
        return self.visit(ctx.parameters())

    def visitParameter(self, ctx):
        return self.visit(ctx.idAttr()[0])

    def visitParameters(self, ctx):
        return [self.visit(parameter) for parameter in ctx.parameter()]

    def modify_func_body(self, body):
        while len(body) > 0 and (not body[-1] or (isinstance(body[-1], CodeStr) and (body[-1] == 'return'))):
            body.pop()
        if len(body) == 0:
            body.append(CodeStr('pass'))
        if not self.used_scope:
            return body
        while CodeStr('pass') in body:
            body.pop(body.index(CodeStr('pass')))
        def fix_returns(lines):
            newbody = []
            for line in lines:
                if isinstance(line, list):
                    newbody.append(fix_returns(line))
                elif isinstance(line, CodeStr) and line.startswith('return '):
                    return_value = CodeStr(line[7:])
                    if 'vfpfunc.variable[' in return_value or 'vfpfunc.function[' in return_value:
                        newbody.append(add_args_to_code('function_return_value = {}', [return_value]))
                        newbody.append(CodeStr('vfpfunc.variable.popscope()'))
                        newbody.append(CodeStr('return function_return_value'))
                    else:
                        newbody.append(CodeStr('vfpfunc.variable.popscope()'))
                        newbody.append(line)
                else:
                    newbody.append(line)
            return newbody
        body = [CodeStr('vfpfunc.variable.pushscope()')] + fix_returns(body)
        if isinstance(body[-1], CodeStr) and body[-1].startswith('return '):
            return body
        body.append(CodeStr('vfpfunc.variable.popscope()'))
        return body

    def visitFuncDef(self, ctx):
        name, parameters = self.visit(ctx.funcDefStart())
        self.new_scope()
        self.scope.update({key: False for key in parameters})
        body = self.visit(ctx.lines())
        self.delete_scope()
        body = self.modify_func_body(body)
        return name, parameters, body

    def visitPrintStmt(self, ctx):
        kwargs = {}
        if len([child for child in ctx.children if child.getText() == '?']) > 1:
            kwargs['end'] = ''
        return [make_func_code('print', *(self.visit(ctx.args()) or []), **kwargs)]

    def visitIfStart(self, ctx):
        return self.visit(ctx.expr())

    def visitIfStmt(self, ctx):
        evaluation = self.visit(ctx.ifStart())

        ifBlock = self.visit(ctx.ifBody)
        retval = [CodeStr('if {}:'.format(evaluation)), ifBlock]

        if ctx.elseBody:
            elseBlock = self.visit(ctx.elseBody)
            retval += [CodeStr('else:'), elseBlock]

        return retval

    def visitCaseStmt(self, ctx):
        n = 0
        retval = []
        for elem in ctx.caseElement():
            if elem.lineComment():
                retval += self.visit(elem.lineComment())
            else:
                expr, lines = self.visit(elem.singleCase())
                if n == 0:
                    retval += [CodeStr('if {}:'.format(expr)), lines]
                else:
                    retval += [CodeStr('elif {}:'.format(expr)), lines]
                n += 1
        if n == 0:
            retval += [CodeStr('if True:'), [CodeStr('pass')]]
        if ctx.otherwise():
            retval += [CodeStr('else:'), self.visit(ctx.otherwise())]
        return retval

    def visitSingleCase(self, ctx):
        return self.visit(ctx.caseExpr()), self.visit(ctx.lines())

    def visitCaseExpr(self, ctx):
        return self.visit(ctx.expr())

    def visitOtherwise(self, ctx):
        return self.visit(ctx.lines())

    def visitForStart(self, ctx):
        loopvar = self.visit_with_disabled_scope(ctx.idAttr())
        self.scope[loopvar] = False
        if ctx.EACH():
            iterator = self.visit(ctx.expr(0))
            return add_args_to_code('for {} in {}:', (loopvar, iterator))
        loop_start = int(self.visit(ctx.loopStart))
        loop_stop = int(self.visit(ctx.loopStop)) + 1
        if ctx.loopStep:
            loop_step = int(self.visit(ctx.loopStep))
            return CodeStr('for {} in range({}, {}, {}):'.format(loopvar, loop_start, loop_stop, loop_step))
        else:
            return CodeStr('for {} in range({}, {}):'.format(loopvar, loop_start, loop_stop))

    def visitForStmt(self, ctx):
        return [self.visit(ctx.forStart()), self.visit(ctx.lines())]

    def visitWhileStart(self, ctx):
        return CodeStr('while {}:'.format(self.visit(ctx.expr())))

    def visitWhileStmt(self, ctx):
        return [self.visit(ctx.whileStart()), self.visit(ctx.lines())]

    def visitWithStmt(self, ctx):
        self.withid = self.visit(ctx.idAttr())
        lines = self.visit(ctx.lines())
        self.withid = ''
        return lines

    def visitScanStmt(self, ctx):
        lines = self.visit(ctx.lines())
        lines.append(make_func_code('vfpfunc.db.skip', None, 1))
        if ctx.FOR():
            condition = self.visit(ctx.expr())
            lines = [add_args_to_code('if not {}:', [condition]), [CodeStr('continue')]] + lines
        save_variable_name = CodeStr(random_variable(length=10) + '_current_record')
        save_recno = CodeStr('{} = {}'.format(save_variable_name, 'vfpfunc.db.recno()'))
        return_recno = make_func_code('vfpfunc.db.goto', None, CodeStr(save_variable_name))
        return [save_recno, CodeStr('while not vfpfunc.db.eof():'), lines, return_recno]

    def visitTryStmt(self, ctx):
        try_lines = self.visit(ctx.tryLines)
        finally_lines = self.visit(ctx.finallyLines) or []
        if not ctx.CATCH():
            return try_lines + finally_lines

        identifier = self.visit(ctx.identifier())
        if identifier:
            self.scope[identifier] = False

        try_lines = [CodeStr('try:'), try_lines]

        if identifier:
            catch_lines = [CodeStr('except Exception as {}:'.format(identifier))]
            catch_lines.append(make_func_code('#vfpfunc.pyexception_to_foxexception', identifier))
        else:
            catch_lines = [CodeStr('except:')]
        catch_lines.append(self.visit(ctx.catchLines))

        finally_lines = [CodeStr('finally:'), finally_lines] if finally_lines else []

        if ctx.identifier():
            del self.scope[identifier]

        return try_lines + catch_lines + finally_lines

    def visitBreakLoop(self, ctx):
        return CodeStr('break')

    def visitContinueLoop(self, ctx):
        return CodeStr('continue')

    def visitDeclaration(self, ctx):
        if ctx.ARRAY() or ctx.DIMENSION() or ctx.DEFINE():
            name = self.visit(ctx.identifier())
            value = self.visit(ctx.arrayIndex())
            if ctx.LOCAL():
                self.scope[name] = False
                array = make_func_code('vfpfunc.Array', *value)
                return [
                    add_args_to_code('{} = {}', [name, array])
                ]
            func = 'vfpfunc.array'
            kwargs = {'public': True} if ctx.PUBLIC() else {}
            self.used_scope = True
            return make_func_code(func, *([str(name)] + value), **kwargs)
        else:
            names = self.visit_with_disabled_scope(ctx.parameters())
            if ctx.LOCAL():
                for name in names:
                    self.scope[name] = False
                return CodeStr(' = '.join([repr(arg) for arg in (names + [False])]) + ' #LOCAL Declaration')
            self.used_scope = True
            func = 'vfpfunc.variable.add_public' if ctx.PUBLIC() else 'vfpfunc.variable.add_private'
            return make_func_code(func, *[str(name) for name in names])

    def visitAssign(self, ctx):
        value = self.visit(ctx.expr())
        args = []
        for var in ctx.idAttr():
            trailer = self.visit(var.trailer()) or []
            if len(trailer) > 0 and isinstance(trailer[-1], list):
                identifier = self.visit(ctx.idAttr()[0].identifier())
                arg = self.createIdAttr(identifier, trailer[:-1] + [[]])
                args.append('{}[{}]'.format(arg[:-2], ','.join(repr(x) for x in trailer[-1])))
            else:
                args.append(self.visit(var))
        if len(args) == 1:
            try:
                if isinstance(value, CodeStr):
                    for op in ('+', '-', '*', '/'):
                        start = str(args[0]) + ' ' + op + ' '
                        if value.startswith(start):
                            return [CodeStr('{} {}= {}'.format(args[0], op, value[len(start):]))]
            except Exception as e:
                pass
        return [CodeStr(' = '.join(args + [repr(value)]))]

    def visitArgs(self, ctx):
        return [self.visit(c) for c in ctx.expr()]

    def visitSpecialArgs(self, ctx):
        return [self.visit(c) for c in ctx.specialExpr()]

    def visitComparison(self, ctx):
        symbol_dict = {ctx.parser.GREATERTHAN: '>',
                       ctx.parser.GTEQ: '>=',
                       ctx.parser.LESSTHAN: '<',
                       ctx.parser.LTEQ: '<=',
                       ctx.parser.NOTEQUALS: '!=',
                       ctx.parser.HASH: '!=',
                       ctx.parser.EQUALS: '==',
                       ctx.parser.DOUBLEEQUALS: '==',
                       ctx.parser.DOLLAR: 'in',
                       ctx.parser.OR: 'or',
                       ctx.parser.AND: 'and'
                      }
        left = self.visit(ctx.expr(0))
        right = self.visit(ctx.expr(1))
        symbol = symbol_dict[ctx.op.type]
        return CodeStr('{} {} {}'.format(repr(left), symbol, repr(right)))

    def visitBooleanOperation(self, ctx):
        return self.visitComparison(ctx)

    def visitUnaryNegation(self, ctx):
        return add_args_to_code('{}' if ctx.op.type == ctx.parser.PLUS_SIGN else '-{}', (self.visit(ctx.expr()),))

    def visitBooleanNegation(self, ctx):
        return CodeStr('not {}'.format(repr(self.visit(ctx.expr()))))

    def func_call(self, funcname, *args, **kwargs):
        if not kwargs and len(args) == 1 and isinstance(args[0], (list, tuple)):
            args = args[0]
        args = list(args)
        if funcname in self.function_list:
            return make_func_code(funcname, *args)
        if funcname == 'chr' and len(args) == 1:
            return chr(int(args[0]))
        if funcname == 'space' and len(args) == 1:
            return int(args[0]) * ' '
        if funcname == 'asc':
            return make_func_code('ord', CodeStr(str(repr(args[0])) + '[0]'))
        if funcname == 'len':
            return make_func_code('len', *args)
        if funcname == 'alen':
            if len(args) == 1:
                return make_func_code('len', *args)
            else:
                args[1] = int(args[1])
                return add_args_to_code('{}.alen({})', args)
        if funcname == 'ascan':
            if len(args) == 3:
                args = [add_args_to_code('{}[{}:]', [args[0], args[2]]), args[1]]
            elif len(args) == 4:
                args = [add_args_to_code('{}[{}:({} + {})]', [args[0], args[2], args[2], args[3]]), args[1]]
            if len(args) == 2:
                return add_args_to_code('{}.index({})', args)
        if funcname == 'empty':
            return add_args_to_code('(not {} if {} is not None else False)', args + args)
        if funcname == 'occurs':
            return add_args_to_code('{}.count({})', reversed(args))
        if funcname in ('at', 'rat'):
            funcname = {
                'at': 'find',
                'rat': 'rfind',
            }[funcname]
            return add_args_to_code('{}.{}({})', [args[1], CodeStr(funcname), args[0]]) + 1
        if funcname in ('repli', 'replicate'):
            args[1:] = [int(arg) for arg in args[1:]]
            return add_args_to_code('({} * {})', args)
        if funcname in ('date', 'datetime', 'time') and len(args) == 0:
            self.imports.append('import datetime as dt')
            if funcname == 'date':
                return make_func_code('dt.datetime.now().date')
            elif funcname == 'datetime':
                return make_func_code('dt.datetime.now')
            elif funcname == 'time':
                return make_func_code('dt.datetime.now().time().strftime', '%H:%M:%S')
        if funcname in ('year', 'month', 'day', 'hour', 'minute', 'sec', 'dow', 'cdow', 'cmonth'):
            self.imports.append('import datetime as dt')
            funcname = {
                'sec': 'second',
                'dow': 'weekday()',
                'cdow': "strftime('%A')",
                'cmonth': "strftime('%B')",
            }.get(funcname, funcname)
            retval = add_args_to_code('{}.{}', [args[0], CodeStr(funcname)])
            if funcname in ('weekday()'):
                self.imports.append('from vfp2py import vfpfunc')
                return make_func_code('vfpfunc.dow_fix', retval, *args[1:])
            return retval
        if funcname == 'dtoc':
            if len(args) == 1 or args[1] == 1:
                if len(args) > 1 and args[1] == 1:
                    args[1] = '%Y%m%d'
                else:
                    args.append('%m/%d/%Y')
                return make_func_code('{}.{}'.format(args[0], 'strftime'), args[1])
        if funcname == 'iif' and len(args) == 3:
{'fraction_non_alphanumeric': 0.14273974334660852, 'fraction_numerical': 0.0048886360284046615, 'mean_word_length': 2.1602850103216356, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '342893', 'n_tokens_mistral': 14347, 'n_tokens_neox': 13975, 'n_words': 3137}