<commit_before>"""
Converts uploaded files into a tabular data structure
"""
__author__ = 'fki'
import codecs
import csv
import datetime
import logging
import os

from xlrd import open_workbook, XL_CELL_DATE, xldate_as_tuple

log = logging.getLogger(__name__)


class FileEncoder(object):
    """
    Converts uploaded files into a tabular data structure
    """
    # Register supported extensions with a function
    supported_extensions = {
        '.csv': '_csv_encode',
        '.tsv': '_tsv_encode',
        '.xlsx': '_xlsx_encode',
        '.xls': '_xlsx_encode'
    }

    # file: InMemoryUploadedFile
    def __init__(self, file):
        """
        Initialize file and file_name
        """
        self.file = file
        self.file_name, self.file_ext = os.path.splitext(file.name)

    def is_supported(self):
        """
        Check if there is a converter for the extension
        """
        if self.file_ext in self.supported_extensions:
            return True
        else:
            return False

    def encode(self):
        """
        Class the responsible function and encodes the file.
        Returns the result
        """
        result = getattr(self, self.supported_extensions[self.file_ext])()
        return result

    def _csv_encode(self, delimiter=' '):
        """
        Encodes a CSV file.
        """
        r = []
        reader = csv.reader(codecs.iterdecode(self.file, "utf-8"), delimiter=delimiter)
        for row in reader:
            log.debug(str(row))
            r.append(row)
        return r

    def _tsv_encode(self):<commit_msg>Fixes delimiter used by FileEncoder's csv decoder<commit_after>"""
Converts uploaded files into a tabular data structure
"""
__author__ = 'fki'
import codecs
import csv
import datetime
import logging
import os

from xlrd import open_workbook, XL_CELL_DATE, xldate_as_tuple

log = logging.getLogger(__name__)


class FileEncoder(object):
    """
    Converts uploaded files into a tabular data structure
    """
    # Register supported extensions with a function
    supported_extensions = {
        '.csv': '_csv_encode',
        '.tsv': '_tsv_encode',
        '.xlsx': '_xlsx_encode',
        '.xls': '_xlsx_encode'
    }

    # file: InMemoryUploadedFile
    def __init__(self, file):
        """
        Initialize file and file_name
        """
        self.file = file
        self.file_name, self.file_ext = os.path.splitext(file.name)

    def is_supported(self):
        """
        Check if there is a converter for the extension
        """
        if self.file_ext in self.supported_extensions:
            return True
        else:
            return False

    def encode(self):
        """
        Class the responsible function and encodes the file.
        Returns the result
        """
        result = getattr(self, self.supported_extensions[self.file_ext])()
        return result

    def _csv_encode(self, delimiter=','):
        """
        Encodes a CSV file.
        """
        r = []
        reader = csv.reader(codecs.iterdecode(self.file, "utf-8"), delimiter=delimiter)
        for row in reader:
            log.debug(str(row))
            r.append(row)
        return r

    def _tsv_encode(self):
{'fraction_non_alphanumeric': 0.09807572936064556, 'fraction_numerical': 0.0006207324643078833, 'mean_word_length': 2.229458917835671, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4133483', 'n_tokens_mistral': 954, 'n_tokens_neox': 924, 'n_words': 261}