<commit_before>import numba
import hpat
from numba import types
from numba.typing.templates import infer_global, AbstractTemplate, infer, signature
from numba.extending import (typeof_impl, type_callable, models, register_model,
                                make_attribute_wrapper, lower_builtin, box)
from numba import cgutils
from hpat.str_ext import string_type

class StringArray(object):
    def __init__(self, offsets, data, size):
        self.size = size
        self.offsets = offsets
        self.data = data

    def __repr__(self):
        return 'StringArray({}, {}, {})'.format(self.offsets, self.data, self.size)

class StringArrayType(types.Type):
    def __init__(self):
        super(StringArrayType, self).__init__(
                                    name='StringArrayType()')

string_array_type = StringArrayType()

@typeof_impl.register(StringArray)
def typeof_string_array(val, c):
    return string_array_type

# @type_callable(StringArray)
# def type_string_array_call(context):
#     def typer(offset, data):
#         return string_array_type
#     return typer

@type_callable(StringArray)
def type_string_array_call2(context):
    def typer(string_list=None):
        return string_array_type
    return typer

@register_model(StringArrayType)
class StringArrayModel(models.StructModel):
    def __init__(self, dmm, fe_type):
        members = [
            ('size', types.intp),
            ('offsets', types.Opaque('offsets')),
            ('data', hpat.str_ext.string_type),
            ]
        models.StructModel.__init__(self, dmm, fe_type, members)

@infer
class GetItemStringArray(AbstractTemplate):
    key = "getitem"

    def generic(self, args, kws):
        assert not kws
        [ary, idx] = args
        if isinstance(ary, StringArrayType):
            if isinstance(idx, types.SliceType):
                return signature(string_array_type, *args)
            else:
                assert isinstance(idx, types.Integer)
                return signature(string_type, *args)


@infer
class CmpOpEqStringArray(AbstractTemplate):
    key = '=='

    def generic(self, args, kws):
        assert not kws
        [va, vb] = args
        # if one of the inputs is string array
        if va==string_array_type or vb==string_array_type:
            # inputs should be either string array or string
            assert va == string_array_type or va == string_type
            assert vb == string_array_type or vb == string_type
            return signature(types.Array(types.boolean, 1, 'C'), va, vb)

@infer
class CmpOpNEqStringArray(CmpOpEqStringArray):
    key = '!='

make_attribute_wrapper(StringArrayType, 'size', 'size')
make_attribute_wrapper(StringArrayType, 'offsets', 'offsets')
make_attribute_wrapper(StringArrayType, 'data', 'data')

# @lower_builtin(StringArray, types.Type, types.Type)
# def impl_string_array(context, builder, sig, args):
#     typ = sig.return_type
#     offsets, data = args
#     string_array = cgutils.create_struct_proxy(typ)(context, builder)
#     string_array.offsets = offsets
#     string_array.data = data
#     return string_array._getvalue()

from numba.targets.listobj import ListInstance
from llvmlite import ir as lir
import llvmlite.binding as ll
import hstr_ext<commit_msg>len string array infer type<commit_after>import numba
import hpat
from numba import types
from numba.typing.templates import infer_global, AbstractTemplate, infer, signature
from numba.extending import (typeof_impl, type_callable, models, register_model,
                                make_attribute_wrapper, lower_builtin, box)
from numba import cgutils
from hpat.str_ext import string_type

class StringArray(object):
    def __init__(self, offsets, data, size):
        self.size = size
        self.offsets = offsets
        self.data = data

    def __repr__(self):
        return 'StringArray({}, {}, {})'.format(self.offsets, self.data, self.size)

class StringArrayType(types.Type):
    def __init__(self):
        super(StringArrayType, self).__init__(
                                    name='StringArrayType()')

string_array_type = StringArrayType()

@typeof_impl.register(StringArray)
def typeof_string_array(val, c):
    return string_array_type

# @type_callable(StringArray)
# def type_string_array_call(context):
#     def typer(offset, data):
#         return string_array_type
#     return typer

@type_callable(StringArray)
def type_string_array_call2(context):
    def typer(string_list=None):
        return string_array_type
    return typer

@register_model(StringArrayType)
class StringArrayModel(models.StructModel):
    def __init__(self, dmm, fe_type):
        members = [
            ('size', types.intp),
            ('offsets', types.Opaque('offsets')),
            ('data', hpat.str_ext.string_type),
            ]
        models.StructModel.__init__(self, dmm, fe_type, members)

@infer
class GetItemStringArray(AbstractTemplate):
    key = "getitem"

    def generic(self, args, kws):
        assert not kws
        [ary, idx] = args
        if isinstance(ary, StringArrayType):
            if isinstance(idx, types.SliceType):
                return signature(string_array_type, *args)
            else:
                assert isinstance(idx, types.Integer)
                return signature(string_type, *args)


@infer
class CmpOpEqStringArray(AbstractTemplate):
    key = '=='

    def generic(self, args, kws):
        assert not kws
        [va, vb] = args
        # if one of the inputs is string array
        if va==string_array_type or vb==string_array_type:
            # inputs should be either string array or string
            assert va == string_array_type or va == string_type
            assert vb == string_array_type or vb == string_type
            return signature(types.Array(types.boolean, 1, 'C'), va, vb)

@infer
class CmpOpNEqStringArray(CmpOpEqStringArray):
    key = '!='

@infer_global(len)
class LenStringArray(AbstractTemplate):
    def generic(self, args, kws):
        if not kws and len(args)==1 and args[0]==string_array_type:
            return signature(types.intp, *args)

make_attribute_wrapper(StringArrayType, 'size', 'size')
make_attribute_wrapper(StringArrayType, 'offsets', 'offsets')
make_attribute_wrapper(StringArrayType, 'data', 'data')

# @lower_builtin(StringArray, types.Type, types.Type)
# def impl_string_array(context, builder, sig, args):
#     typ = sig.return_type
#     offsets, data = args
#     string_array = cgutils.create_struct_proxy(typ)(context, builder)
#     string_array.offsets = offsets
#     string_array.data = data
#     return string_array._getvalue()

from numba.targets.listobj import ListInstance
from llvmlite import ir as lir
import llvmlite.binding as ll
import hstr_ext
{'fraction_non_alphanumeric': 0.09730450236966824, 'fraction_numerical': 0.0008886255924170616, 'mean_word_length': 3.236511919698871, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1414249', 'n_tokens_mistral': 2042, 'n_tokens_neox': 2010, 'n_words': 498}