<commit_before>"""
Converts uploaded files into a tabular data structure
"""
__author__ = 'fki'
import os
import csv
import codecs
from xlrd import open_workbook, XL_CELL_DATE, xldate_as_tuple
import datetime

import logging
log = logging.getLogger(__name__)


class FileEncoder(object):
    """
    Converts uploaded files into a tabular data structure
    """
    # Register supported extensions with a function
    supported_extensions = {
        '.csv': '_csv_encode',
        '.xlsx': '_xlsx_encode',
        '.xls': '_xlsx_encode'
    }

    # file: InMemoryUploadedFile
    def __init__(self, file):
        """
        Initialize file and file_name
        """
        self.file = file
        self.file_name, self.file_ext = os.path.splitext(file.name)

    def is_supported(self):
        """
        Check if there is a converter for the extension
        """
        if self.file_ext in self.supported_extensions:
            return True
        else:
            return False

    def encode(self):
        """
        Class the responsible function and encodes the file.
        Returns the result
        """
        result = getattr(self, self.supported_extensions[self.file_ext])()
        return result

    def _csv_encode(self):
        """
        Encodes a CSV file.
        """
        r = []
        reader = csv.reader(codecs.iterdecode(self.file, "utf-8"))
        for row in reader:
            log.info(str(row))
            r.append(row)
        return r

    def _xlsx_encode(self):
        """
        Encodes XLS and XLSX files.
        """
        r = []
        wb = open_workbook(file_contents=self.file.read())
        sheet =  wb.sheet_by_index(0)
        for row in range(sheet.nrows):
            values = []
            for col in range(sheet.ncols):
                cell = sheet.cell(row,col)
                # Date cells have to be converted to return as string
                if cell.ctype == XL_CELL_DATE:
                    v = xldate_as_tuple(cell.value,wb.datemode)
                    v = datetime.datetime(*v)
                    v = datetime.date(v.year,v.month,v.day)
                else:
                    v = str(cell.value)
                values.append(v)
            r.append(values)
        return r
<commit_msg>Fixed Bug in XLSX Loading<commit_after>"""
Converts uploaded files into a tabular data structure
"""
__author__ = 'fki'
import os
import csv
import codecs
from xlrd import open_workbook, XL_CELL_DATE, xldate_as_tuple
import datetime

import logging
log = logging.getLogger(__name__)


class FileEncoder(object):
    """
    Converts uploaded files into a tabular data structure
    """
    # Register supported extensions with a function
    supported_extensions = {
        '.csv': '_csv_encode',
        '.xlsx': '_xlsx_encode',
        '.xls': '_xlsx_encode'
    }

    # file: InMemoryUploadedFile
    def __init__(self, file):
        """
        Initialize file and file_name
        """
        self.file = file
        self.file_name, self.file_ext = os.path.splitext(file.name)

    def is_supported(self):
        """
        Check if there is a converter for the extension
        """
        if self.file_ext in self.supported_extensions:
            return True
        else:
            return False

    def encode(self):
        """
        Class the responsible function and encodes the file.
        Returns the result
        """
        result = getattr(self, self.supported_extensions[self.file_ext])()
        return result

    def _csv_encode(self):
        """
        Encodes a CSV file.
        """
        r = []
        reader = csv.reader(codecs.iterdecode(self.file, "utf-8"))
        for row in reader:
            log.info(str(row))
            r.append(row)
        return r

    def _xlsx_encode(self):
        """
        Encodes XLS and XLSX files.
        """
        r = []
        wb = open_workbook(file_contents=self.file.read())
        sheet =  wb.sheet_by_index(0)
        for row in range(sheet.nrows):
            values = []
            for col in range(sheet.ncols):
                cell = sheet.cell(row,col)
                # Date cells have to be converted to return as string
                log.info(cell.value)
                if cell.ctype == XL_CELL_DATE:
                    v = xldate_as_tuple(cell.value,wb.datemode)
                    v = datetime.datetime(*v)
                    v = datetime.date(v.year,v.month,v.day)
                elif isinstance(cell.value, float):
                    if cell.value == int(cell.value):
                        v = int(cell.value)
                    else:
                        v = str(cell.value)
                else:
                    v = str(cell.value)

                values.append(v)
            r.append(values)
        return r

{'fraction_non_alphanumeric': 0.09564315352697095, 'fraction_numerical': 0.0008298755186721991, 'mean_word_length': 1.6872909698996656, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2118929', 'n_tokens_mistral': 1417, 'n_tokens_neox': 1356, 'n_words': 358}