<commit_before>from pandoc_converters import pandoc_convert
import chardet 
import re

# TODO: move that in text_base: save images
def convert_from_mimetype(file_name, mime_type, format):
    input = open(file_name, 'r').read()
    return _convert_from_mimetype(input, mime_type, format)

def to_unicode(input):
    if type(input) == str:
        res = None
        for encoding in [chardet.detect(input)['encoding'], 'utf8', 'latin1']:
            try:
                res = unicode(input, encoding)
                break;
            except UnicodeDecodeError:
                pass
        if not res:
            raise Exception('UnicodeDecodeError: could not decode')
        return res
    return input

def _convert_from_mimetype(input, mime_type, format):
    #input = to_unicode(input)
        
    attachs = []
    attachs_dir = None
    ##############################
    if mime_type in ['application/vnd.oasis.opendocument.text',
                     'application/msword',
                     ]:
        
        xhtml_input, attachs = convert_oo_to_html(input)
        converted_input = pandoc_convert(xhtml_input, 'html', format)
        
    ##############################
    # anything looks like code: put them into markdown citation
    elif mime_type.startswith('text/x-') or mime_type in ['application/x-latex',
                                                          'application/x-ruby',
                       ]:
        converted_input = markdown_from_code(input)

    ##############################
    # html
    elif mime_type in ['text/html', 'application/xhtml+xml']:
        if format == 'html':
            converted_input = input
        else:
            converted_input = pandoc_convert(input, 'html', format)
    ##############################
    # anything looks like text -> markdown
    elif mime_type in ['text/plain',
                       'text/english',
                       'text/enriched'<commit_msg>add latex conversion (with pandoc)<commit_after>from pandoc_converters import pandoc_convert
import chardet 
import re

# TODO: move that in text_base: save images
def convert_from_mimetype(file_name, mime_type, format):
    input = open(file_name, 'r').read()
    return _convert_from_mimetype(input, mime_type, format)

def to_unicode(input):
    if type(input) == str:
        res = None
        for encoding in [chardet.detect(input)['encoding'], 'utf8', 'latin1']:
            try:
                res = unicode(input, encoding)
                break;
            except UnicodeDecodeError:
                pass
        if not res:
            raise Exception('UnicodeDecodeError: could not decode')
        return res
    return input

def _convert_from_mimetype(input, mime_type, format):
    #input = to_unicode(input)
        
    attachs = []
    attachs_dir = None
    ##############################
    if mime_type in ['application/vnd.oasis.opendocument.text',
                     'application/msword',
                     ]:
        
        xhtml_input, attachs = convert_oo_to_html(input)
        converted_input = pandoc_convert(xhtml_input, 'html', format)
        
    ##############################
    # latex
    elif mime_type in ['application/x-latex','text/x-tex',]:
        converted_input = pandoc_convert(to_unicode(input), 'latex', format)
    
    ##############################
    # anything looks like code: put them into markdown citation
    elif mime_type.startswith('text/x-') or mime_type in ['application/x-ruby',]:
        converted_input = markdown_from_code(input)

    ##############################
    # html
    elif mime_type in ['text/html', 'application/xhtml+xml']:
        if format == 'html':
            converted_input = input
        else:
            converted_input = pandoc_convert(input, 'html', format)
    ##############################
    # anything looks like text -> markdown
    elif mime_type in ['text/plain',
                       'text/english',
                       'text/enriched'
{'fraction_non_alphanumeric': 0.15511221945137157, 'fraction_numerical': 0.000997506234413965, 'mean_word_length': 2.328630705394191, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1658950', 'n_tokens_mistral': 1152, 'n_tokens_neox': 1105, 'n_words': 261}