<commit_before>import types
import ctypes
import _ctypes
import sys

from ctree.types import (
    codegen_type,
    register_type_recognizers,
    register_type_codegenerators,
)

if sys.version_info >= (3, 0):
    register_type_recognizers({
        int: lambda t: ctypes.c_long(t),
        bool: lambda t: ctypes.c_bool(t),
        float: lambda t: ctypes.c_double(t),
        str: lambda t: ctypes.c_char(str.encode(t)) if len(t) == 1 else
        ctypes.c_char_p(str.encode(t)),
        type(None): lambda t: None,
    })

    register_type_codegenerators({
        ctypes.c_int: lambda t: "int",
        ctypes.c_long: lambda t: "long",
        ctypes.c_float: lambda t: "float",
        ctypes.c_double: lambda t: "double",
        ctypes.c_char: lambda t: "char",
        ctypes.c_char_p: lambda t: "char*",
        ctypes.c_void_p: lambda t: "void*",
        ctypes.c_bool: lambda t: "bool",
        ctypes.c_ulong: lambda t: "size_t",
        ctypes.c_wchar_p: lambda t: "char*",
        type(None): lambda n: "void",

        _ctypes.Array: lambda ct: "%s*" % codegen_type(ct._type_()),
        _ctypes._Pointer: lambda ct: "%s*" % codegen_type(ct._type_()),

    })
else:
    register_type_recognizers({
        types.IntType: lambda t: ctypes.c_long(t),
        types.LongType: lambda t: ctypes.c_long(t),
        types.BooleanType: lambda t: ctypes.c_bool(t),
        types.FloatType: lambda t: ctypes.c_double(t),
        types.StringType: lambda t: ctypes.c_char(t) if len(t) == 1 else
        ctypes.c_char_p(t),
        types.NoneType: lambda t: None,
    })

    register_type_codegenerators({
        ctypes.c_int: lambda t: "int",
        ctypes.c_long: lambda t: "long",
        ctypes.c_float: lambda t: "float",
        ctypes.c_double: lambda t: "double",
        ctypes.c_char: lambda t: "char",
        ctypes.c_char_p: lambda t: "char*",
        ctypes.c_void_p: lambda t: "void*",
        ctypes.c_bool: lambda t: "bool",
        ctypes.c_ulong: lambda t: "size_t",
        types.NoneType: lambda n: "void",

        _ctypes.Array: lambda ct: "%s*" % codegen_type(ct._type_()),
        _ctypes._Pointer: lambda ct: "%s*" % codegen_type(ct._type_()),

    })<commit_msg>refactored the type recognizers and code generators in __init__ since a lot of them are redundant between py2 and py3. Also, removed lambdas since lambda t: f(t) is just f<commit_after>import types
import ctypes
import _ctypes
import sys

from ctree.types import (
    codegen_type,
    register_type_recognizers,
    register_type_codegenerators,
)

#Py2 and Py3 common types

register_type_recognizers(
    {
        int: ctypes.c_long,
        bool: ctypes.c_bool,
        float: ctypes.c_double,
        str: lambda t: ctypes.c_char(t.encode()) if len(t) == 1 else ctypes.c_char_p(t.encode()),
        type(None): lambda t: None
    }
)

register_type_codegenerators({
    ctypes.c_int: lambda t: "int",
    ctypes.c_long: lambda t: "long",
    ctypes.c_float: lambda t: "float",
    ctypes.c_double: lambda t: "double",
    ctypes.c_char: lambda t: "char",
    ctypes.c_char_p: lambda t: "char*",
    ctypes.c_void_p: lambda t: "void*",
    ctypes.c_bool: lambda t: "bool",
    ctypes.c_ulong: lambda t: "size_t",
    type(None): lambda n: "void",

    _ctypes.Array: lambda ct: "%s*" % codegen_type(ct._type_()),
    _ctypes._Pointer: lambda ct: "%s*" % codegen_type(ct._type_()),

})

#register version specific nodes

if sys.version_info >= (3, 0):
    pass

else:

    register_type_recognizers({
        long: ctypes.c_long
    })
{'fraction_non_alphanumeric': 0.13284341435839458, 'fraction_numerical': 0.0031091011871113624, 'mean_word_length': 2.9191583610188263, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5788779', 'n_tokens_mistral': 1294, 'n_tokens_neox': 1308, 'n_words': 313}