<commit_before>#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-
"""
Utilise Yandex Translation Service.

Obtain bilingual semantic human score.
"""

import os
import json
import codecs
import requests

def get_valid_translation(word):
    tr_word_2=get_translation(word)
    if (len(tr_word_2.split()))>1:
       tr_word_2 = None
    return tr_word_2

def get_translation(word):
    base_url = 'https://translate.yandex.net/api/v1.5/tr.json/translate'
    options = {
        'lang': 'en-hi',
        'key': '<Enter Key Here>',
        'text': word
    }
    return json.loads(requests.get(base_url, params=options).text)["text"][0]

def yandex_api(lang_translate, input_score_path, output_score_path):
    """
    Utilise Yandex Translation Service, obtain bilingual semantic human score.

    WordSim score, translated on one column using Yandex.
    Yandex Api Key, lang for translation needs to be provided
    """
    output_data = []
    with codecs.open(input_score_path, 'r', encoding='utf-8') as file:
        with codecs.open(output_score_path, 'w', encoding='utf-8') as outfile:
            for line in file:
                word_1, word_2, score = line.split()
                tr_word_2 = get_valid_translation(word_2)

                if tr_word_2 is not None:
                    try:
                        output_data.append("%s %s %s" % (
                            word_1,
                            tr_word_2,
                            score
                        ))
                    except KeyError:
                        pass
            outfile.write("\n".join(output_data))
<commit_msg>Yandex: Update Documentation to rst for automatic autodoc support of Sphinx.<commit_after>#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-
"""
**Utilise Yandex Translation Service.**

- Obtain bilingual semantic human score.
"""

import os
import json
import codecs
import requests

def get_valid_translation(word):
    """**Ensure the translation is valid.**"""
    tr_word_2=get_translation(word)
    if (len(tr_word_2.split()))>1:
       tr_word_2 = None
    return tr_word_2

def get_translation(word):
    """
    **Obtain translation of specified word from Yandex.**

    - Translation from English -> Hindi.
    """

    base_url = 'https://translate.yandex.net/api/v1.5/tr.json/translate'
    options = {
        'lang': 'en-hi',
        'key': '<Enter Key Here>',
        'text': word
    }
    return json.loads(requests.get(base_url, params=options).text)["text"][0]

def yandex_api(lang_translate, input_score_path, output_score_path):
    """
    **Utilise Yandex Translation Service, obtain bilingual semantic human score.**

    - WordSim score, translated on one column using Yandex.
    - Yandex Api Key, lang for translation needs to be provided
    """
    output_data = []
    with codecs.open(input_score_path, 'r', encoding='utf-8') as file:
        with codecs.open(output_score_path, 'w', encoding='utf-8') as outfile:
            for line in file:
                word_1, word_2, score = line.split()
                tr_word_2 = get_valid_translation(word_2)

                if tr_word_2 is not None:
                    try:
                        output_data.append("%s %s %s" % (
                            word_1,
                            tr_word_2,
                            score
                        ))
                    except KeyError:
                        pass
            outfile.write("\n".join(output_data))

{'fraction_non_alphanumeric': 0.11053689348262992, 'fraction_numerical': 0.011484352569623888, 'mean_word_length': 2.032201914708442, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '345885', 'n_tokens_mistral': 1070, 'n_tokens_neox': 1042, 'n_words': 258}