<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

"""

"""
import datetime as dt
import collections

import pandas as pd
import geopandas as gp
import numpy as np
import shapely.geometry
import os

from hub.odhql.exceptions import OdhQLExecutionException

from opendatahub.utils.plugins import RegistrationMixin

import shapely.speedups

shapely.speedups.available and shapely.speedups.enable()


class OdhType(RegistrationMixin):
    dtypes = ()
    ptypes = ()
    name = ''

    by_name = {}
    by_dtype = collections.defaultdict(list)
    by_ptype = collections.defaultdict(list)

    _is_abstract = True

    @classmethod
    def register_child(cls, name, bases, own_dict):
        if not own_dict.get('_is_abstract'):
            instance = cls()

            instance.ptype = cls.ptypes[0]
            instance.dtype = cls.dtypes[0]
            instance.name = cls.name or name.upper()

            setattr(OdhType, instance.name, instance)
            instance.by_name[instance.name] = instance
            for dtype in instance.dtypes:
                OdhType.by_dtype[dtype].append(instance)

            for ptype in instance.ptypes:
                OdhType.by_ptype[ptype].append(instance)

    @classmethod
    def identify_value(cls, value):
        type_ = type(value)
        if issubclass(type_, np.generic):
            return cls.by_dtype[type_]
        else:
            try:
                return cls.by_ptype[type_][0]
            except:
                pass

    @classmethod
    def identify_series(cls, s):
        by_dtype = cls.by_dtype[s.dtype.type]
        if len(by_dtype) == 1:
            return by_dtype[0]

        first = s.first_valid_index()
        if first is not None:
            value = s.iat[first]
            return cls.identify_value(value)
        else:
            return cls.TEXT

    def convert(self, series):
        return series._constructor(series.values.astype(self.dtypes[0]), index=series.index).__finalize__(series)

    def __repr__(self):
        return 'OdhType({})'.format(self.name)

    def __str__(self):
        return self.name

    def __eq__(self, other):
        if not isinstance(other, self.__class__):
            return False
        return self.name == other.name

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(self.name)


class IntegerType(OdhType):
    name = 'INTEGER'
    dtypes = np.int32,
    ptypes = int,

    def convert(self, series):
        return series._constructor(series.values.astype(np.float_).astype(self.dtypes[0]),
                                   index=series.index).__finalize__(series)


class BigIntType(IntegerType):
    name = 'BIGINT'
    dtypes = np.int64,
    ptypes = int,


class SmallIntType(IntegerType):
    name = 'SMALLINT'
    dtypes = np.int16,
    ptypes = int,


class FloatType(OdhType):
    name = 'FLOAT'
    dtypes = np.float64,
    ptypes = float,


class DateTimeType(OdhType):
    name = 'DATETIME'
    dtypes = np.datetime64,
    ptypes = pd.Timestamp,

    def convert(self, series):
        if series.odh_type.ptype not in (int, float):
            raise OdhQLExecutionException('Unable to convert from {} to {}'.format(str(series.odh_type), str(self)))
        return series._constructor(pd.to_datetime(series, unit='s'), index=series.index).__finalize__(series)


class IntervalType(OdhType):
    name = 'INTERVAL'
    dtypes = pd.Timedelta,
    ptypes = dt.timedelta,


class BooleanType(OdhType):
    name = 'BOOLEAN'
    dtypes = np.bool_,
    ptypes = bool,


class TextType(OdhType):
    name = 'TEXT'
    dtypes = np.object_,
    ptypes = unicode, str

    def convert(self, series):
        try:
            return series._constructor(series.values.astype(unicode).astype(object), index=series.index).__finalize__(
                series)
        except:  # fallback, slower
            return series._constructor(series.astype(unicode).astype(object), index=series.index).__finalize__(series)


class GeometryType(OdhType):
    name = 'GEOMETRY'
    dtypes = np.object_,
    ptypes = (
        shapely.geometry.Point, shapely.geometry.LineString, shapely.geometry.Polygon, shapely.geometry.LinearRing,
        shapely.geometry.MultiLineString, shapely.geometry.MultiPoint, shapely.geometry.MultiPolygon
    )


class OdhFrame(pd.DataFrame):
    _metadata = ['name']
<commit_msg>Handle NoneType-columns as text<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from types import NoneType

"""

"""
import datetime as dt
import collections

import pandas as pd
import geopandas as gp
import numpy as np
import shapely.geometry
import os

from hub.odhql.exceptions import OdhQLExecutionException

from opendatahub.utils.plugins import RegistrationMixin

import shapely.speedups

shapely.speedups.available and shapely.speedups.enable()


class OdhType(RegistrationMixin):
    dtypes = ()
    ptypes = ()
    name = ''

    by_name = {}
    by_dtype = collections.defaultdict(list)
    by_ptype = collections.defaultdict(list)

    _is_abstract = True

    @classmethod
    def register_child(cls, name, bases, own_dict):
        if not own_dict.get('_is_abstract'):
            instance = cls()

            instance.ptype = cls.ptypes[0]
            instance.dtype = cls.dtypes[0]
            instance.name = cls.name or name.upper()

            setattr(OdhType, instance.name, instance)
            instance.by_name[instance.name] = instance
            for dtype in instance.dtypes:
                OdhType.by_dtype[dtype].append(instance)

            for ptype in instance.ptypes:
                OdhType.by_ptype[ptype].append(instance)

    @classmethod
    def identify_value(cls, value):
        type_ = type(value)
        if issubclass(type_, np.generic):
            return cls.by_dtype[type_]
        else:
            try:
                return cls.by_ptype[type_][0]
            except:
                pass

    @classmethod
    def identify_series(cls, s):
        by_dtype = cls.by_dtype[s.dtype.type]
        if len(by_dtype) == 1:
            return by_dtype[0]

        first = s.first_valid_index()
        if first is not None:
            value = s.iat[first]
            return cls.identify_value(value)
        else:
            return cls.TEXT

    def convert(self, series):
        return series._constructor(series.values.astype(self.dtypes[0]), index=series.index).__finalize__(series)

    def __repr__(self):
        return 'OdhType({})'.format(self.name)

    def __str__(self):
        return self.name

    def __eq__(self, other):
        if not isinstance(other, self.__class__):
            return False
        return self.name == other.name

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(self.name)


class IntegerType(OdhType):
    name = 'INTEGER'
    dtypes = np.int32,
    ptypes = int,

    def convert(self, series):
        return series._constructor(series.values.astype(np.float_).astype(self.dtypes[0]),
                                   index=series.index).__finalize__(series)


class BigIntType(IntegerType):
    name = 'BIGINT'
    dtypes = np.int64,
    ptypes = int,


class SmallIntType(IntegerType):
    name = 'SMALLINT'
    dtypes = np.int16,
    ptypes = int,


class FloatType(OdhType):
    name = 'FLOAT'
    dtypes = np.float64,
    ptypes = float,


class DateTimeType(OdhType):
    name = 'DATETIME'
    dtypes = np.datetime64,
    ptypes = pd.Timestamp,

    def convert(self, series):
        if series.odh_type.ptype not in (int, float):
            raise OdhQLExecutionException('Unable to convert from {} to {}'.format(str(series.odh_type), str(self)))
        return series._constructor(pd.to_datetime(series, unit='s'), index=series.index).__finalize__(series)


class IntervalType(OdhType):
    name = 'INTERVAL'
    dtypes = pd.Timedelta,
    ptypes = dt.timedelta,


class BooleanType(OdhType):
    name = 'BOOLEAN'
    dtypes = np.bool_,
    ptypes = bool,


class TextType(OdhType):
    name = 'TEXT'
    dtypes = np.object_,
    ptypes = unicode, str, NoneType

    def convert(self, series):
        try:
            return series._constructor(series.values.astype(unicode).astype(object), index=series.index).__finalize__(
                series)
        except:  # fallback, slower
            return series._constructor(series.astype(unicode).astype(object), index=series.index).__finalize__(series)


class GeometryType(OdhType):
    name = 'GEOMETRY'
    dtypes = np.object_,
    ptypes = (
        shapely.geometry.Point, shapely.geometry.LineString, shapely.geometry.Polygon, shapely.geometry.LinearRing,
        shapely.geometry.MultiLineString, shapely.geometry.MultiPoint, shapely.geometry.MultiPolygon
    )


class OdhFrame(pd.DataFrame):
    _metadata = ['name']

{'fraction_non_alphanumeric': 0.10552368095398808, 'fraction_numerical': 0.004049949375632805, 'mean_word_length': 2.576025744167337, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3385956', 'n_tokens_mistral': 2832, 'n_tokens_neox': 2795, 'n_words': 588}