<commit_before>#coding: utf8
from datetime import datetime, date
from decimal import Decimal
import time
import re

# data type transfer variables and functions
FALSE_VALUES = (None, '', 0, '0', 'f', 'F', 'false', 'FALSE')
ISO_DATE = r'^(\d{4})-(\d\d)-(\d\d)$'
ISO_DATETIME = r'^(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(\.\d+)?$'


def to_bool(v):
    """ convert something to a boolean  """
    if not v:           return False
    if is_str(v):       v = v.strip()
    if v in FALSE_VALUES:   return False
    if not v:           return False
    return True


def to_i(v):
    if v is True:   return 1
    if v is False:  return 0
    try:            return int(v)
    except:         return 0


def to_f(v):
    try:    return float(v)
    except: return 0.0


def to_decimal(v):
    if is_decimal(v):   return v
    try:                return Decimal(v)
    except:             return Decimal(0)


datetime2date   = lambda dt: date(dt.year, dt.month, dt.day)
str2binary      = lambda v: v # Used to convert from Strings to BLOBs
binary2str      = lambda v: v # Used to convert from BLOBs to Strings

def str2datetime(s):
    if s is None:     return None
    if not is_str(s): return s
    if not s.strip(): return None
    return __fast_str2datetime(s) or __fallback_str2datetime(s)

def str2date(s):
    if s is None:           return None
    if is_datetime(s):      return datetime_to_date(s)
    if is_date(s):          return s
    if not is_str(s):       return s
    if not s.strip():       return None    
    return __fast_str2date(s) or __fallback_str2date(s)


def __fast_str2datetime(string):
    """ Doesn't handle time zones. """
    if re.match(ISO_DATETIME, string):
        subs = re.split(ISO_DATETIME, string)
        microsec = to_i((to_f(subs[7]) * 1000000))
        return datetime(to_i(subs[1]), to_i(subs[2]), to_i(subs[3]), 
                        to_i(subs[4]), to_i(subs[5]), to_i(subs[6]), 
                        microsec)
    return None


def __fast_str2date(string):
    if re.match(ISO_DATE, string):
        subs = re.split(ISO_DATE, string)
        return date(to_i(subs[1]), to_i(subs[2]), to_i(subs[3]))
    return None


def __fallback_str2date(string):
    string = string.strip()
    string = string.split()[0]
    tm_struct = time.strptime(string, '%Y-%m-%d')
    return date(tm_struct.tm_year, tm_struct.tm_mon, tm_struct.tm_mday)


def __fallback_stri2datetime(string):
    try:
        d = str2date(string)
        return datetime(d.year, d.month, d.day)
    except:
        iso_string = string.strip()
        pos = iso_string.rfind('.')
        if pos != -1:
            iso_string[:pos]
            microsec = microseconds(iso_string[pos:])
        else:
            microsec = 0

        tm_struct = time.strptime(string, '%Y-%m-%d %H:%M:%S')
        return datetime(tm_struct.tm_year, tm_struct.tm_mon, tm_struct.tm_mday, tm_struct.tm_hour, tm_struct.tm_min, tm_struct.tm_sec, microsec)
    

# check the object type functions
is_str      = lambda obj: isinstance(obj, (str, unicode))
is_decimal  = lambda obj: isinstance(obj, Decimal)

is_date     = lambda obj: isinstance(obj, date)
is_datetime = lambda obj: isinstance(obj, datetime)

is_hash     = lambda obj: isinstance(obj, dict)
is_array    = lambda obj: isinstance(obj, (tuple, list, set))


# other functions

def flatten(sequnce):<commit_msg>fix bugs<commit_after>#coding: utf8
from datetime import datetime, date
from decimal import Decimal
import time
import re


# data type transfer variables and functions
FALSE_VALUES = (None, '', 0, '0', 'f', 'F', 'false', 'FALSE')
ISO_DATE = r'^(\d{4})-(\d\d)-(\d\d)$'
ISO_DATETIME = r'^(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(\.\d+)?$'


def to_bool(v):
    """ convert something to a boolean  """
    if not v:           return False
    if is_str(v):       v = v.strip()
    if v in FALSE_VALUES:   return False
    if not v:           return False
    return True


def to_i(v):
    if v is True:   return 1
    if v is False:  return 0
    try:            return int(v)
    except:         return 0


def to_f(v):
    try:    return float(v)
    except: return 0.0


def to_decimal(v):
    if is_decimal(v):   return v
    try:                return Decimal(v)
    except:             return Decimal(0)

datetime2str    = lambda dt, format='%Y-%m-%d %H:%M:%S': dt.strftime(format)
date2str        = lambda d, format='%Y-%m-%d': d.strftime(format)

datetime2date   = lambda dt: date(dt.year, dt.month, dt.day)
str2binary      = lambda v: v # Used to convert from Strings to BLOBs
binary2str      = lambda v: v # Used to convert from BLOBs to Strings

microseconds    = lambda t: to_i((to_f(t) % 1) * 1000000) # '0.123456' -> 123456; '1.123456' -> 123456
    

def str2datetime(s):
    if s is None:     return None
    if not is_str(s): return s
    if not s.strip(): return None
    return __fast_str2datetime(s) or __fallback_str2datetime(s)

def str2date(s):
    if s is None:           return None
    if is_datetime(s):      return datetime2date(s)
    if is_date(s):          return s
    if not is_str(s):       return s
    if not s.strip():       return None    
    return __fast_str2date(s) or __fallback_str2date(s)


def __fast_str2datetime(string):
    """ Doesn't handle time zones. """
    if re.match(ISO_DATETIME, string):
        subs = re.split(ISO_DATETIME, string)
        microsec = to_i((to_f(subs[7]) * 1000000))
        return datetime(to_i(subs[1]), to_i(subs[2]), to_i(subs[3]), 
                        to_i(subs[4]), to_i(subs[5]), to_i(subs[6]), 
                        microsec)
    return None


def __fast_str2date(string):
    if re.match(ISO_DATE, string):
        subs = re.split(ISO_DATE, string)
        return date(to_i(subs[1]), to_i(subs[2]), to_i(subs[3]))
    return None


def __fallback_str2date(string):
    string = string.strip()
    string = string.split()[0]
    tm_struct = time.strptime(string, '%Y-%m-%d')
    return date(tm_struct.tm_year, tm_struct.tm_mon, tm_struct.tm_mday)


def __fallback_str2datetime(string):
    try:
        d = str2date(string)
        return datetime(d.year, d.month, d.day)
    except:
        iso_string = string.strip()
        pos = iso_string.rfind('.')
        if pos != -1:
            iso_string[:pos]
            microsec = microseconds(iso_string[pos:])
        else:
            microsec = 0

        tm_struct = time.strptime(string, '%Y-%m-%d %H:%M:%S')
        return datetime(tm_struct.tm_year, tm_struct.tm_mon, tm_struct.tm_mday, tm_struct.tm_hour, tm_struct.tm_min, tm_struct.tm_sec, microsec)
    

# check the object type functions
is_str      = lambda obj: isinstance(obj, (str, unicode))
is_decimal  = lambda obj: isinstance(obj, Decimal)

is_date     = lambda obj: isinstance(obj, date)
is_datetime = lambda obj: isinstance(obj, datetime)

is_hash     = lambda obj: isinstance(obj, dict)
is_array    = lambda obj: isinstance(obj, (tuple, list, set))


# other functions

def flatten(sequnce):
{'fraction_non_alphanumeric': 0.13106517370083262, 'fraction_numerical': 0.01823140970427792, 'mean_word_length': 2.626756897449245, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5160630', 'n_tokens_mistral': 2563, 'n_tokens_neox': 2473, 'n_words': 645}