<commit_before>'''
Created on Jul 15, 2011

@author: sean

'''

# Based on pysourcegen from the meta package
# (https://pypi.python.org/pypi/meta)
#
# The meta package has a BSD license, which means that we
# are free to use and modify the source as long as we
# leave the original copyright in place.  Unfortunately,
# the code does not include a copy of the license (it
# simply says that the code has a BSD license) so we
# do not know exactly which variant of the BSD license
# was intended, or who the original copyright holder(s)
# is.  The sole author listed in the source code and
# documentation is <NAME>.
#
# The reason for modifying this module is that it can't
# handle python3 'with' statements properly, and we
# need it to do so.
#
# There are notes on the Python web site that there are
# additional bugs in other constructs, but I haven't
# encountered them yet.  We'll fix as need be.

from __future__ import print_function
import _ast
from meta.asttools import Visitor
from string import Formatter
import sys
from meta.utils import py3op, py2op
from contextlib import contextmanager

if sys.version_info[0] < 3:
    from StringIO import StringIO
else:
    from io import StringIO

@contextmanager
def noctx():
    yield
class ASTFormatter(Formatter):

    def format_field(self, value, format_spec):
        if format_spec == 'node':
            gen = ExprSourceGen()
            gen.visit(value)
            return gen.dumps()
        elif value == '':
            return value
        else:
            return super(ASTFormatter, self).format_field(value, format_spec)

    def get_value(self, key, args, kwargs):
        if key == '':
            return args[0]
        elif key in kwargs:
            return kwargs[key]
        elif isinstance(key, int):
            return args[key]

        key = int(key)
        return args[key]

        raise Exception

def str_node(node):
    gen = ExprSourceGen()
    gen.visit(node)
    return gen.dumps()

def simple_string(value):
    def visitNode(self, node):
        self.print(value, **node.__dict__)
    return visitNode

class ExprSourceGen(Visitor):
    def __init__(self):
        self.out = StringIO()
        self.formatter = ASTFormatter()
        self.indent = '    '
        self.level = 0


    @property
    def indenter(self):
        return Indenter(self)

    @property
    def no_indent(self):
        return NoIndent(self)

    def dump(self, file=sys.stdout):
        self.out.seek(0)
        print(self.out.read(), file=file)

    def dumps(self):
        self.out.seek(0)
        value = self.out.read()
        return value

    def print(self, line, *args, **kwargs):
        line = self.formatter.format(line, *args, **kwargs)

        level = kwargs.get('level')
        prx = self.indent * (level if level else self.level)
        print(prx, line, sep='', end='', file=self.out)

    def print_lines(self, lines,):
        prx = self.indent * self.level
        for line in lines:
            print(prx, line, sep='', file=self.out)

    def visitName(self, node):
        self.print(node.id)
    
    @py2op
    def visitarguments(self, node):
        # ('args', 'vararg', 'kwarg', 'defaults')
        defaults = [None] * (len(node.args) - len(node.defaults))
        defaults.extend(node.defaults)

        i = 0
        args = list(node.args)
        if args:
            i += 1
            arg = args.pop(0)
            default = defaults.pop(0)
            self.visit(arg)
            if default is not None:
                self.print('={:node}', default)

        while args:
            arg = args.pop(0)
            default = defaults.pop(0)
            self.print(', ')
            self.visit(arg)
            if default is not None:
                self.print('={:node}', default)
<commit_msg>add pretty-printer for NameConstant instances<commit_after>'''
Created on Jul 15, 2011

@author: sean

'''

# Based on pysourcegen from the meta package
# (https://pypi.python.org/pypi/meta)
#
# The meta package has a BSD license, which means that we
# are free to use and modify the source as long as we
# leave the original copyright in place.  Unfortunately,
# the code does not include a copy of the license (it
# simply says that the code has a BSD license) so we
# do not know exactly which variant of the BSD license
# was intended, or who the original copyright holder(s)
# is.  The sole author listed in the source code and
# documentation is <NAME>.
#
# The reason for modifying this module is that it can't
# handle python3 'with' statements properly, and we
# need it to do so.
#
# There are notes on the Python web site that there are
# additional bugs in other constructs, but I haven't
# encountered them yet.  We'll fix as need be.

from __future__ import print_function
import _ast
from meta.asttools import Visitor
from string import Formatter
import sys
from meta.utils import py3op, py2op
from contextlib import contextmanager

if sys.version_info[0] < 3:
    from StringIO import StringIO
else:
    from io import StringIO

@contextmanager
def noctx():
    yield
class ASTFormatter(Formatter):

    def format_field(self, value, format_spec):
        if format_spec == 'node':
            gen = ExprSourceGen()
            gen.visit(value)
            return gen.dumps()
        elif value == '':
            return value
        else:
            return super(ASTFormatter, self).format_field(value, format_spec)

    def get_value(self, key, args, kwargs):
        if key == '':
            return args[0]
        elif key in kwargs:
            return kwargs[key]
        elif isinstance(key, int):
            return args[key]

        key = int(key)
        return args[key]

        raise Exception

def str_node(node):
    gen = ExprSourceGen()
    gen.visit(node)
    return gen.dumps()

def simple_string(value):
    def visitNode(self, node):
        self.print(value, **node.__dict__)
    return visitNode

class ExprSourceGen(Visitor):
    def __init__(self):
        self.out = StringIO()
        self.formatter = ASTFormatter()
        self.indent = '    '
        self.level = 0


    @property
    def indenter(self):
        return Indenter(self)

    @property
    def no_indent(self):
        return NoIndent(self)

    def dump(self, file=sys.stdout):
        self.out.seek(0)
        print(self.out.read(), file=file)

    def dumps(self):
        self.out.seek(0)
        value = self.out.read()
        return value

    def print(self, line, *args, **kwargs):
        line = self.formatter.format(line, *args, **kwargs)

        level = kwargs.get('level')
        prx = self.indent * (level if level else self.level)
        print(prx, line, sep='', end='', file=self.out)

    def print_lines(self, lines,):
        prx = self.indent * self.level
        for line in lines:
            print(prx, line, sep='', file=self.out)

    def visitName(self, node):
        self.print(node.id)

    # BBN: special case for Python 3.5
    def visitNameConstant(self, node):
        self.print(str(node.value))
    # End BBN changes
    
    @py2op
    def visitarguments(self, node):
        # ('args', 'vararg', 'kwarg', 'defaults')
        defaults = [None] * (len(node.args) - len(node.defaults))
        defaults.extend(node.defaults)

        i = 0
        args = list(node.args)
        if args:
            i += 1
            arg = args.pop(0)
            default = defaults.pop(0)
            self.visit(arg)
            if default is not None:
                self.print('={:node}', default)

        while args:
            arg = args.pop(0)
            default = defaults.pop(0)
            self.print(', ')
            self.visit(arg)
            if default is not None:
                self.print('={:node}', default)

{'fraction_non_alphanumeric': 0.1033595057278929, 'fraction_numerical': 0.005920967949543055, 'mean_word_length': 2.2965634280865506, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1189367', 'n_tokens_mistral': 2338, 'n_tokens_neox': 2271, 'n_words': 771}