<commit_before>import os
import logging
import json
import requests
import urllib

from googleapiclient.discovery import build

logger = logging.getLogger('translation')

MSTRANSLATE_LANGS = ['hi']
GOOGLE_LANGS = ['hi']

def translateGoogle(jobInfo, text, fromLang, toLang = 'en'):
  try:
    logger.info("Started google translation. %s", jobInfo);

    service = build('translate', 'v2',
            developerKey = os.environ['GOOGLE_DEV_KEY'])
    result = service.translations().list(
      source=fromLang,
      target=toLang,
      q=text).execute()['translations'][0]['translatedText']

    logger.info("Completed google translation. %s", jobInfo)
    return result
  except:
    logger.info("Google translation failed. %s", jobInfo)
    return ""

def translateMicrosoft(jobInfo, text, fromLang, toLang = 'en'):
  try:
    logger.info("Started microsoft translation. %s", jobInfo);

    # get the access token
    args = {
      'client_id': os.environ['MSTRANSLATE_CLIENT_ID'],
      'client_secret': os.environ['MSTRANSLATE_CLIENT_SECRET'],
      'scope': 'http://api.microsofttranslator.com',
      'grant_type': 'client_credentials'
    }
    oauth_url = 'https://datamarket.accesscontrol.windows.net/v2/OAuth2-13'
    oauth_response = json.loads(
      requests.post(oauth_url, data=urllib.urlencode(args)).content)

    # make the translate api call
    translation_args = {
      'text': text,
      'to': toLang,
      'from': fromLang
    }
    headers={'Authorization': 'Bearer '+ oauth_response['access_token']}
    translate_url = 'https://api.microsofttranslator.com/V2/Ajax.svc/Translate?'
    translation_result = requests.get(
      translate_url + urllib.urlencode(translation_args),
      headers=headers)

    if translation_result.status_code == 200:
      logger.info("Completed microsoft translation. %s", jobInfo)
      return translation_result.content
    else:
      logger.info(
        "Microsoft translation call returned with status code %i",
        translation_result.status_code)
      return ""
  except Exception as e:
    logging.exception("Microsoft translation failed. %s", jobInfo)
    return ""

def translate(jobId, text, fromLang, toLang = 'en'):
  jobInfo = "fromLang: " + fromLang + " toLang: " + toLang \<commit_msg>Handling unicode input in microsoft translation<commit_after>import os
import logging
import json
import requests
import urllib

from googleapiclient.discovery import build

logger = logging.getLogger('translation')

MSTRANSLATE_LANGS = ['hi']
GOOGLE_LANGS = ['hi']

def translateGoogle(jobInfo, text, fromLang, toLang = 'en'):
  try:
    logger.info("Started google translation. %s", jobInfo);

    service = build('translate', 'v2',
            developerKey = os.environ['GOOGLE_DEV_KEY'])
    result = service.translations().list(
      source=fromLang,
      target=toLang,
      q=text).execute()['translations'][0]['translatedText']

    logger.info("Completed google translation. %s", jobInfo)
    return result
  except:
    logger.info("Google translation failed. %s", jobInfo)
    return ""

def translateMicrosoft(jobInfo, text, fromLang, toLang = 'en'):
  try:
    logger.info("Started microsoft translation. %s", jobInfo);

    # get the access token
    args = {
      'client_id': os.environ['MSTRANSLATE_CLIENT_ID'],
      'client_secret': os.environ['MSTRANSLATE_CLIENT_SECRET'],
      'scope': 'http://api.microsofttranslator.com',
      'grant_type': 'client_credentials'
    }
    oauth_url = 'https://datamarket.accesscontrol.windows.net/v2/OAuth2-13'
    oauth_response = json.loads(
      requests.post(oauth_url, data=urllib.urlencode(args)).content)

    # make the translate api call
    strText = text
    if isinstance(text, unicode):
      strText = text.encode('utf-8')

    translation_args = {
      'text': strText,
      'to': toLang,
      'from': fromLang
    }

    headers={'Authorization': 'Bearer '+ oauth_response['access_token']}
    translate_url = 'https://api.microsofttranslator.com/V2/Ajax.svc/Translate?'
    translation_result = requests.get(
      translate_url + urllib.urlencode(translation_args),
      headers=headers)

    if translation_result.status_code == 200:
      logger.info("Completed microsoft translation. %s", jobInfo)
      return translation_result.content
    else:
      logger.info(
        "Microsoft translation call returned with status code %i",
        translation_result.status_code)
      return ""
  except Exception as e:
    logging.exception("Microsoft translation failed. %s", jobInfo)
    return ""

def translate(jobId, text, fromLang, toLang = 'en'):
  jobInfo = "fromLang: " + fromLang + " toLang: " + toLang \
{'fraction_non_alphanumeric': 0.1251339190057853, 'fraction_numerical': 0.004499678594386115, 'mean_word_length': 3.700906344410876, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '623514', 'n_tokens_mistral': 1451, 'n_tokens_neox': 1435, 'n_words': 316}