<commit_before>#!/usr/bin/env python

import csv
from cStringIO import StringIO
import datetime

import xlrd

from csvkit import typeinference
import utils

class XLSDataError(Exception):
    """
    Exception raised when there is a problem converting XLS data.
    """
    def __init__(self, msg):
        self.msg = msg

def normalize_empty(values):
    """
    Normalize a column which contains only empty cells.
    """
    return [None] * len(values)

def normalize_text(values):
    """
    Normalize a column of text cells.
    """
    return [v if v else None for v in values]

def normalize_numbers(values):
    """
    Normalize a column of numeric cells.
    """
    # Test if all values are whole numbers, if so coerce floats it ints
    integral = True

    for v in values:
        if v and v % 1 != 0:
            integral = False
            break

    if integral:
        return [int(v) if v else None for v in values]
    else:
        # Convert blanks to None
        return [v if v else None for v in values]

def normalize_dates(values, datemode):
    """
    Normalize a column of date cells.
    """
    normal_values = []
    normal_types_set = set()

    for v in values:
        # Convert blanks to None
        if v == '':
            normal_values.append(None)
            continue

        v_tuple = xlrd.xldate_as_tuple(v, datemode)

        if v_tuple == (0, 0, 0, 0, 0, 0):
            # Midnight 
            normal_values.append(datetime.time(*v_tuple[3:]))
            normal_types_set.add('time')
        elif v_tuple[3:] == (0, 0, 0):
            # Date only<commit_msg>Fixed failing test.<commit_after>#!/usr/bin/env python

import csv
from cStringIO import StringIO
import datetime

import xlrd

from csvkit import typeinference
import utils

class XLSDataError(Exception):
    """
    Exception raised when there is a problem converting XLS data.
    """
    def __init__(self, msg):
        self.msg = msg

def normalize_empty(values):
    """
    Normalize a column which contains only empty cells.
    """
    return [None] * len(values)

def normalize_text(values):
    """
    Normalize a column of text cells.
    """
    return [v if v else None for v in values]

def normalize_numbers(values):
    """
    Normalize a column of numeric cells.
    """
    # Test if all values are whole numbers, if so coerce floats it ints
    integral = True

    for v in values:
        if v and v % 1 != 0:
            integral = False
            break

    if integral:
        return [int(v) if v != '' else None for v in values]
    else:
        # Convert blanks to None
        return [v if v else None for v in values]

def normalize_dates(values, datemode):
    """
    Normalize a column of date cells.
    """
    normal_values = []
    normal_types_set = set()

    for v in values:
        # Convert blanks to None
        if v == '':
            normal_values.append(None)
            continue

        v_tuple = xlrd.xldate_as_tuple(v, datemode)

        if v_tuple == (0, 0, 0, 0, 0, 0):
            # Midnight 
            normal_values.append(datetime.time(*v_tuple[3:]))
            normal_types_set.add('time')
        elif v_tuple[3:] == (0, 0, 0):
            # Date only
{'fraction_non_alphanumeric': 0.09324659231722429, 'fraction_numerical': 0.0080545229244114, 'mean_word_length': 2.1471734892787526, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2446371', 'n_tokens_mistral': 1019, 'n_tokens_neox': 964, 'n_words': 349}