<commit_before># -*- coding: utf-8 -*-
#!/usr/bin/env python
#
# Copyright 2012 BigML
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""Miscellaneous utility functions.

"""
import re
import unidecode
from urlparse import urlparse
import locale

DEFAULT_LOCALE = 'en_US.UTF-8'
WINDOWS_DEFAULT_LOCALE = 'English'
LOCALE_SYNONYMS = {'en': ['en_us', 'en_us.utf8', 'english_united states.1252'],
                   'es': ['es_es', 'es_es.utf8', 'spanish_spain.1252'],
                   'sp': ['es_es', 'es_es.utf8', 'spanish_spain.1252'],
                   'fr': [['fr_fr', 'fr_fr.utf8', 'fr_ch.utf8', 'fr_be.utf8',
                           'french_france.1252'],
                          ['fr_ca', 'fr_ca.utf8', 'french_canada.1252']],
                   'de': ['de_de', 'de_de.utf8', 'german_germany.1252'],
                   'ge': ['de_de', 'de_de.utf8', 'german_germany.1252'],
                   'it': ['it_it', 'it_it.utf8', 'italian_italy.1252'],
                   'ca': ['ca_es', 'ca_es.utf8', 'catalan_spain.1252']}

BOLD_REGEX = re.compile(r'''(\*\*)(?=\S)([^\r]*?\S[*_]*)\1''')
ITALIC_REGEX = re.compile(r'''(_)(?=\S)([^\r]*?\S)\1''')
LINKS_REGEX = re.compile((r'''(\[((?:\[[^\]]*\]|[^\[\]])*)\]\([ \t]*()'''
                          r'''<?(.*?)>?[ \t]*((['"])(.*?)\6[ \t]*)?\))'''),
                         re.MULTILINE)
TYPE_MAP = {
    "categorical": str,
    "numeric": locale.atof,
    "text": str
}

PYTHON_TYPE_MAP = {
    "categorical": [unicode, str],
    "numeric": [int, float],
    "text": [unicode, str]
}

PREDICTIONS_FILE_SUFFIX = '_predictions.csv'


def python_map_type(value):
    """Maps a BigML type to equivalent Python types.

    """
    if value in PYTHON_TYPE_MAP:
        return PYTHON_TYPE_MAP[value]
    else:
        return [unicode, str]<commit_msg>Adding more locale literals for equivalences<commit_after># -*- coding: utf-8 -*-
#!/usr/bin/env python
#
# Copyright 2012 BigML
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""Miscellaneous utility functions.

"""
import re
import unidecode
from urlparse import urlparse
import locale

DEFAULT_LOCALE = 'en_US.UTF-8'
WINDOWS_DEFAULT_LOCALE = 'English'
LOCALE_SYNONYMS = {'en': ['en_us', 'en_us.utf8', 'en_us.utf-8',
                          'english_united states.1252'],
                   'es': ['es_es', 'es_es.utf8', 'es_es.utf-8',
                          'spanish_spain.1252'],
                   'sp': ['es_es', 'es_es.utf8', 'es_es.utf-8',
                          'spanish_spain.1252'],
                   'fr': [['fr_fr', 'fr_fr.utf8', 'fr_ch.utf8', 'fr_be.utf8',
                           'fr_fr.utf-8', 'fr_ch.utf-8', 'fr_be.utf-8',
                           'french_france.1252'],
                          ['fr_ca', 'fr_ca.utf8', 'fr_ca.utf-8',
                           'french_canada.1252']],
                   'de': ['de_de', 'de_de.utf8', 'de_de.utf-8',
                          'german_germany.1252'],
                   'ge': ['de_de', 'de_de.utf8', 'de_de.utf-8',
                          'german_germany.1252'],
                   'it': ['it_it', 'it_it.utf8', 'it_it.utf-8',
                          'italian_italy.1252'],
                   'ca': ['ca_es', 'ca_es.utf8', 'ca_es.utf-8',
                          'catalan_spain.1252']}

BOLD_REGEX = re.compile(r'''(\*\*)(?=\S)([^\r]*?\S[*_]*)\1''')
ITALIC_REGEX = re.compile(r'''(_)(?=\S)([^\r]*?\S)\1''')
LINKS_REGEX = re.compile((r'''(\[((?:\[[^\]]*\]|[^\[\]])*)\]\([ \t]*()'''
                          r'''<?(.*?)>?[ \t]*((['"])(.*?)\6[ \t]*)?\))'''),
                         re.MULTILINE)
TYPE_MAP = {
    "categorical": str,
    "numeric": locale.atof,
    "text": str
}

PYTHON_TYPE_MAP = {
    "categorical": [unicode, str],
    "numeric": [int, float],
    "text": [unicode, str]
}

PREDICTIONS_FILE_SUFFIX = '_predictions.csv'


def python_map_type(value):
    """Maps a BigML type to equivalent Python types.

    """
    if value in PYTHON_TYPE_MAP:
        return PYTHON_TYPE_MAP[value]
    else:
        return [unicode, str]
{'fraction_non_alphanumeric': 0.18498606133014736, 'fraction_numerical': 0.026085225009956192, 'mean_word_length': 2.391627278865631, 'pattern_counts': {'":': 12, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1011267', 'n_tokens_mistral': 1844, 'n_tokens_neox': 1752, 'n_words': 399}