<commit_before>import os
import re

from astvisitor import EmptyCode, descend, parse, ASTVisitor
from store import Class, Function, TestClass, TestMethod, ModuleNotFound, \
     LiveObject, MethodCall, Method
from util import camelize, underscore, sorted


def constructor_as_string(object):
    """For a given object return a string representing a code that will
    construct it.

    >>> constructor_as_string(123)
    '123'
    >>> constructor_as_string('string')
    "'string'"
    >>> obj = LiveObject(None, Class('SomeClass'), None)
    >>> constructor_as_string(obj)
    'SomeClass()'
    >>> obj.add_call(MethodCall(Method('__init__'), {'arg': 'whatever'}, None))
    >>> constructor_as_string(obj)
    "SomeClass(arg='whatever')"
    """
    if isinstance(object, LiveObject):
        args = {}
        # Look for __init__ call and base the constructor on that.
        init_call = object.get_init_call()
        if init_call:
            args = init_call.input
        return call_as_string(object.klass.name, args)
    return repr(object)

def call_as_string(object_name, input):
    """Generate code for calling an object with given input.

    >>> call_as_string('fun', {'a': 1, 'b': 2})
    'fun(a=1, b=2)'
    >>> call_as_string('capitalize', {'str': 'string'})
    "capitalize(str='string')"
    """
    return "%s(%s)" % (object_name, ', '.join(["%s=%s" % (arg, constructor_as_string(value)) for arg, value in input.iteritems()]))

def object2id(object):
    """Convert object to string that can be used as an identifier.
    """
    if object is True:
        return 'true'
    elif object is False:
        return 'false'
    return re.sub(r'\.|\!', '', re.sub(r'\s+', '_', str(object).strip()))

def input_as_string(input):
    """Generate an underscored description of given input arguments.

    >>> input_as_string({})
    ''
    >>> input_as_string({'x': 7, 'y': 13})
    'x_equal_7_and_y_equal_13'
    """
    if len(input) == 1:
        return object2id(input.values()[0])
    return "_and_".join(["%s_equal_%s" % (arg, object2id(value))
                         for arg, value in sorted(input.iteritems())])
<commit_msg>  Filter out all bad characters in generator.object2id.<commit_after>import os
import re

from astvisitor import EmptyCode, descend, parse, ASTVisitor
from store import Class, Function, TestClass, TestMethod, ModuleNotFound, \
     LiveObject, MethodCall, Method
from util import camelize, underscore, sorted


def constructor_as_string(object):
    """For a given object return a string representing a code that will
    construct it.

    >>> constructor_as_string(123)
    '123'
    >>> constructor_as_string('string')
    "'string'"
    >>> obj = LiveObject(None, Class('SomeClass'), None)
    >>> constructor_as_string(obj)
    'SomeClass()'
    >>> obj.add_call(MethodCall(Method('__init__'), {'arg': 'whatever'}, None))
    >>> constructor_as_string(obj)
    "SomeClass(arg='whatever')"
    """
    if isinstance(object, LiveObject):
        args = {}
        # Look for __init__ call and base the constructor on that.
        init_call = object.get_init_call()
        if init_call:
            args = init_call.input
        return call_as_string(object.klass.name, args)
    return repr(object)

def call_as_string(object_name, input):
    """Generate code for calling an object with given input.

    >>> call_as_string('fun', {'a': 1, 'b': 2})
    'fun(a=1, b=2)'
    >>> call_as_string('capitalize', {'str': 'string'})
    "capitalize(str='string')"
    """
    return "%s(%s)" % (object_name, ', '.join(["%s=%s" % (arg, constructor_as_string(value)) for arg, value in input.iteritems()]))

def object2id(object):
    """Convert object to string that can be used as an identifier.
    """
    if object is True:
        return 'true'
    elif object is False:
        return 'false'
    return re.sub(r'[^a-zA-Z0-9_]', '', re.sub(r'\s+', '_', str(object).strip()))

def input_as_string(input):
    """Generate an underscored description of given input arguments.

    >>> input_as_string({})
    ''
    >>> input_as_string({'x': 7, 'y': 13})
    'x_equal_7_and_y_equal_13'
    """
    if len(input) == 1:
        return object2id(input.values()[0])
    return "_and_".join(["%s_equal_%s" % (arg, object2id(value))
                         for arg, value in sorted(input.iteritems())])

{'fraction_non_alphanumeric': 0.15117349286700416, 'fraction_numerical': 0.010354348826507133, 'mean_word_length': 3.4311926605504586, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2607835', 'n_tokens_mistral': 1412, 'n_tokens_neox': 1363, 'n_words': 363}