<commit_before><?php
/*
    Tatoeba Project, free collaborativ creation of languages corpuses project
    Copyright (C) 2009  <NAME> (<EMAIL>)

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

class GoogleLanguageApiComponent extends Object{
	var $text;
	var $langSrc;
	var $langDest;
	var $translateUrl = "http://ajax.googleapis.com/ajax/services/language/translate?v=1.0";
	
	/* 
	 * This function uses cURL : http://de.php.net/manual/en/curl.installation.php
	 *
	 * The language detection is done by Google's language detect API.
	 * Google documentation : http://code.google.com/apis/ajaxlanguage/documentation/reference.html
	 *
	 * The function returns an associative array with the indices "language", "isReliable" and "confidence".	 
	 */
	function detectLang(){
		$textToAnalyze = urlencode($this->text);
		
		$langDetectUrl = "http://ajax.googleapis.com/ajax/services/language/detect?v=1.0";
		$url = $langDetectUrl . "&q=" . $textToAnalyze;
		
		$ch = curl_init();
		curl_setopt($ch, CURLOPT_URL, $url);
		curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
		curl_setopt($ch, CURLOPT_REFERER, "http://tatoeba.fr");
		$body = curl_exec($ch);
		curl_close($ch);

		// now, process the JSON string
		$json = json_decode($body, true);

		if ($json['responseStatus'] != 200){
			return false;
		}

		return $json['responseData'];
	}
	
	function google2TatoebaCode($googleCode){
		switch($googleCode){
			case 'ar' :		return 'ar';
			case 'bg' :		return 'bg';
			case 'de' :		return 'de';
			case 'en' :		return 'en';
			case 'es' :		return 'es';
			case 'fi' :		return 'fi';
			case 'fr' :		return 'fr';
			case 'iw' : 	return 'he';<commit_msg>Hack for language detection.<commit_after><?php
/*
    Tatoeba Project, free collaborativ creation of languages corpuses project
    Copyright (C) 2009  <NAME> (<EMAIL>)

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

class GoogleLanguageApiComponent extends Object{
	var $text;
	var $langSrc;
	var $langDest;
	var $translateUrl = "http://ajax.googleapis.com/ajax/services/language/translate?v=1.0";
	
	/* 
	 * This function uses cURL : http://de.php.net/manual/en/curl.installation.php
	 *
	 * The language detection is done by Google's language detect API.
	 * Google documentation : http://code.google.com/apis/ajaxlanguage/documentation/reference.html
	 *
	 * The function returns an associative array with the indices "language", "isReliable" and "confidence".	 
	 */
	function detectLang(){
		$textToAnalyze = urlencode($this->text);
		
		$langDetectUrl = "http://ajax.googleapis.com/ajax/services/language/detect?v=1.0";
		$url = $langDetectUrl . "&q=" . $textToAnalyze.$textToAnalyze; 
			// send text in duplicate to re-balance confidence score, because the score
			// depends on the length of the text.
		
		$ch = curl_init();
		curl_setopt($ch, CURLOPT_URL, $url);
		curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
		curl_setopt($ch, CURLOPT_REFERER, "http://tatoeba.fr");
		$body = curl_exec($ch);
		curl_close($ch);

		// now, process the JSON string
		$json = json_decode($body, true);

		if ($json['responseStatus'] != 200){
			return false;
		}

		return $json['responseData'];
	}
	
	function google2TatoebaCode($googleCode){
		switch($googleCode){
			case 'ar' :		return 'ar';
			case 'bg' :		return 'bg';
			case 'de' :		return 'de';
			case 'en' :		return 'en';
			case 'es' :		return 'es';
			case 'fi' :		return 'fi';
			case 'fr' :		return 'fr';
			case 'iw' : 	return 'he';
{'fraction_non_alphanumeric': 0.11752182243985523, 'fraction_numerical': 0.005961251862891207, 'mean_word_length': 3.637709772951629, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1454909', 'n_tokens_mistral': 1607, 'n_tokens_neox': 1422, 'n_words': 490}