<commit_before># -*- coding: utf-8 -*-

__module_name__ = "translator"
__module_version__ = "1.0"
__module_description__ = "Translates from one language to others using Google Translate."
__module_author__ = "<NAME>, karona75, briand"

from threading import Thread
from urllib import request as http

# http://translate.google.com/translate_a/t?q=This%20is%20a%20test&client=x&text=&sl=auto&tl=de&ie=UTF-8
# 'User-agent' : 'Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5'

url = 'http://translate.google.com/translate_a/t?q=This%20is%20a%20test&client=x&text=&sl=auto&tl=de&ie=UTF-8'
useragent = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5'
headers = {'User-agent' : useragent}
req = http.Request(url, None, headers)
response = http.urlopen(req)
# response = http.urlopen("http://translate.google.com/translate_a/t?q=This%20is%20a%20test&client=x&text=&sl=auto&tl=de&ie=UTF-8")
#print(response.status_code)
print(response.read())<commit_msg>Added demonstration of threaded translation.<commit_after># -*- coding: utf-8 -*-

__module_name__ = "translator"
__module_version__ = "1.0"
__module_description__ = "Translates from one language to others using Google Translate."
__module_author__ = "<NAME>, karona75, briand"

from threading import Thread
from urllib import request as http
import json
import pprint

url = 'http://translate.google.com/translate_a/t?q=This%20is%20a%20test&client=x&text=&sl=auto&tl=de&ie=UTF-8'
useragent = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5'

def worker(id, url, useragent):
	print('Running translation job ' + str(id))
	headers = {'User-agent': useragent}
	req = http.Request(url, None, headers)
	response = http.urlopen(req)

	data = response.read().decode('utf-8')
	data = json.loads(data)

#	pp = pprint.PrettyPrinter(indent=4)
#	pp.pprint(data)
	print('Ran translation job ' + str(id))

for i in range(10):
	t = Thread(target = worker, args = (i, url, useragent))
	t.start()
{'fraction_non_alphanumeric': 0.16315531602155806, 'fraction_numerical': 0.04850563449289564, 'mean_word_length': 5.587096774193548, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 3, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3474918', 'n_tokens_mistral': 824, 'n_tokens_neox': 775, 'n_words': 160}