<commit_before>import chardet
import codecs
from invisibleroads_macros.disk import get_file_extension
from crosscompute.exceptions import DataTypeError
from crosscompute.scripts.serve import import_upload
from crosscompute.types import DataType
from io import StringIO
from os.path import exists

from .fallbacks import pandas


class TableType(DataType):
    suffixes = 'table',
    formats = 'csv', 'msg', 'json', 'xls', 'xlsx'
    style = 'crosscompute_table:assets/part.min.css'
    script = 'crosscompute_table:assets/part.min.js'
    template = 'crosscompute_table:type.jinja2'
    views = [
        'import_table',
    ]

    @classmethod
    def save(Class, path, table):
        if path.endswith('.csv'):
            table.to_csv(path, encoding='utf-8', index=False)
        elif path.endswith('.msg'):
            table.to_msgpack(path, compress='blosc')
        elif path.endswith('.json'):
            table.to_json(path)
        elif path.endswith('.xls') or path.endswith('.xlsx'):
            table.to_excel(path)
        else:
            raise DataTypeError(
                'file format not supported (%s)' % get_file_extension(path))

    @classmethod
    def load(Class, path):
        if not exists(path):
            raise IOError
        if path.endswith('.csv'):
            try:
                table = pandas.read_csv(
                    path, encoding='utf-8', skipinitialspace=True)
            except UnicodeDecodeError:
                best_encoding = chardet.detect(open(path).read())['encoding']
                best_alpha_count = 0
                for encoding in [best_encoding, 'cp850']:
                    content = codecs.open(path, encoding=encoding).read()
                    alpha_count = sum(x.isalpha() for x in content)
                    if alpha_count > best_alpha_count:
                        best_encoding = encoding
                table = pandas.read_csv(
                    path, encoding=best_encoding, skipinitialspace=True)
        elif path.endswith('.msg'):
            table = pandas.read_msgpack(path)
        elif path.endswith('.json'):
            table = pandas.read_json(path, orient='split')
        elif path.endswith('.xls') or path.endswith('.xlsx'):
            table = pandas.read_excel(path)
        else:
            raise DataTypeError(
                'file format not supported (%s)' % get_file_extension(path))
        return table

    @classmethod
    def parse(Class, text):
        try:
            table = pandas.read_csv(
                StringIO(text), encoding='utf-8', skipinitialspace=True)
        except (TypeError, ValueError):
            raise TypeError('expected_table')
        return table

    @classmethod
    def format(Class, table, format_name='csv'):
        if format_name == 'csv':
            return table.to_csv(encoding='utf-8', index=False)
        elif format_name == 'json':
            return table.to_json(orient='split')

    @classmethod
    def match(Class, table):
        return hasattr(table, 'iterrows')


def import_table(request):
    return import_upload(request, TableType, {
        'class': 'editable-table',
    })<commit_msg>Fix alpha_count<commit_after>import chardet
from invisibleroads_macros.disk import get_file_extension
from crosscompute.exceptions import DataTypeError
from crosscompute.scripts.serve import import_upload
from crosscompute.types import DataType
from io import StringIO
from os.path import exists

from .fallbacks import pandas


class TableType(DataType):
    suffixes = 'table',
    formats = 'csv', 'msg', 'json', 'xls', 'xlsx'
    style = 'crosscompute_table:assets/part.min.css'
    script = 'crosscompute_table:assets/part.min.js'
    template = 'crosscompute_table:type.jinja2'
    views = [
        'import_table',
    ]

    @classmethod
    def save(Class, path, table):
        if path.endswith('.csv'):
            table.to_csv(path, encoding='utf-8', index=False)
        elif path.endswith('.msg'):
            table.to_msgpack(path, compress='blosc')
        elif path.endswith('.json'):
            table.to_json(path)
        elif path.endswith('.xls') or path.endswith('.xlsx'):
            table.to_excel(path)
        else:
            raise DataTypeError(
                'file format not supported (%s)' % get_file_extension(path))

    @classmethod
    def load(Class, path):
        if not exists(path):
            raise IOError
        if path.endswith('.csv'):
            try:
                table = pandas.read_csv(
                    path, encoding='utf-8', skipinitialspace=True)
            except UnicodeDecodeError:
                encoding = _get_encoding(open(path).read())
                table = pandas.read_csv(
                    path, encoding=encoding, skipinitialspace=True)
        elif path.endswith('.msg'):
            table = pandas.read_msgpack(path)
        elif path.endswith('.json'):
            table = pandas.read_json(path, orient='split')
        elif path.endswith('.xls') or path.endswith('.xlsx'):
            table = pandas.read_excel(path)
        else:
            raise DataTypeError(
                'file format not supported (%s)' % get_file_extension(path))
        return table

    @classmethod
    def parse(Class, text):
        try:
            table = pandas.read_csv(
                StringIO(text), encoding='utf-8', skipinitialspace=True)
        except (TypeError, ValueError):
            raise TypeError('expected_table')
        return table

    @classmethod
    def format(Class, table, format_name='csv'):
        if format_name == 'csv':
            return table.to_csv(encoding='utf-8', index=False)
        elif format_name == 'json':
            return table.to_json(orient='split')

    @classmethod
    def match(Class, table):
        return hasattr(table, 'iterrows')


def import_table(request):
    return import_upload(request, TableType, {
        'class': 'editable-table',
    })


def _get_encoding(content):
    best_encoding = chardet.detect(content)['encoding']
    best_alpha_count = 0
    for encoding in [best_encoding, 'cp850']:
        alpha_count = sum(x.isalpha() for x in content.decode(encoding))
        if alpha_count > best_alpha_count:
            best_encoding = encoding
            best_alpha_count = alpha_count
    return best_encoding
{'fraction_non_alphanumeric': 0.10979087452471482, 'fraction_numerical': 0.0028517110266159697, 'mean_word_length': 2.3384452670544684, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7541149', 'n_tokens_mistral': 1840, 'n_tokens_neox': 1764, 'n_words': 406}