<commit_before>"""Azure Translator module."""
import xml.etree.ElementTree as ET

import requests


class Translator(object):
    """
    Translator class, connects to Azure Translator API:

        http://docs.microsofttranslator.com/text-translate.html
    """
    DEFAULT_LANGUAGE = 'en'
    TOKEN_API = 'https://api.cognitive.microsoft.com/sts/v1.0/issueToken'
    TRANSLATE_API = 'https://api.microsofttranslator.com/v2/http.svc/Translate'


    def __init__(self, api_key):
        self.api_key = api_key

    def get_access_token(self):
        """
        Retrieve an access token in order to use the Translator API.
        """
        resp = requests.post(
            self.TOKEN_API,
            headers={
                'Content-Type': 'application/json',
                'Accept': 'application/jwt',
                'Ocp-Apim-Subscription-Key': self.api_key,
            }
        )
        resp.raise_for_status()
        return resp.content

    def translate(self, text, to=DEFAULT_LANGUAGE):
        """
        Translate some text into some language. Target language default to english.

        :param text:               the text to translate
        :param to:                 the target language
        """
        resp = requests.get(
            self.TRANSLATE_API,
            headers={
                'Authorization': 'Bearer {}'.format(self.get_access_token()),
                'Accept': 'application/xml',
            },
            params={
                'text': text,
                'to': to,
            }
        )
        resp.raise_for_status()
        return ET.fromstring(resp.content).text<commit_msg>fix utf8 issue<commit_after>"""Azure Translator module."""
import xml.etree.ElementTree as ET

import requests


class Translator(object):
    """
    Translator class, connects to Azure Translator API:

        http://docs.microsofttranslator.com/text-translate.html
    """
    DEFAULT_LANGUAGE = 'en'
    TOKEN_API = 'https://api.cognitive.microsoft.com/sts/v1.0/issueToken'
    TRANSLATE_API = 'https://api.microsofttranslator.com/v2/http.svc/Translate'


    def __init__(self, api_key):
        self.api_key = api_key

    def get_access_token(self):
        """
        Retrieve an access token in order to use the Translator API.
        """
        resp = requests.post(
            self.TOKEN_API,
            headers={
                'Content-Type': 'application/json',
                'Accept': 'application/jwt',
                'Ocp-Apim-Subscription-Key': self.api_key,
            }
        )
        resp.raise_for_status()
        return resp.content

    def translate(self, text, to=DEFAULT_LANGUAGE):
        """
        Translate some text into some language. Target language default to english.

        :param text:               the text to translate
        :param to:                 the target language
        """
        resp = requests.get(
            self.TRANSLATE_API,
            headers={
                'Authorization': 'Bearer {}'.format(self.get_access_token()),
                'Accept': 'application/xml',
            },
            params={
                'text': text,
                'to': to,
            }
        )
        resp.raise_for_status()
        return ET.fromstring(resp.content.encode('utf-8')).text
{'fraction_non_alphanumeric': 0.11013615733736762, 'fraction_numerical': 0.002420574886535552, 'mean_word_length': 1.7758186397984888, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5497155', 'n_tokens_mistral': 956, 'n_tokens_neox': 908, 'n_words': 199}