<commit_before><?php
/*
 * Created on Jan 27, 2009
 *
 */
 
 
 
 require_once 'lib/ointegratelib.php';
 require_once 'Multilingual/Aligner/SentenceSegmentor.php'; 
 
class Multilingual_MachineTranslation_GoogleTranslateWrapper {

//this array should be updated as Google Translate
//adds more languages

	public $langsSupportedByGoogleTranslate = 
			array (
				'sq' => 'Albanian',
				'ar' => 'Arabic',
				'bg' => 'Bulgarian',
				'ca' => 'Catalan',
				'zh' => 'Chinese',
				'hr' => 'Croatian',
				'cs' => 'Czech',
				'da' => 'Danish',
				'nl' => 'Dutch',
				'en' => 'English',
				'et' => 'Estonian',
				'fil' => 'Filipino',
				'fi' => 'Finnish',
				'fr' => 'French',
				'gl' => 'Galician',
				'de' => 'German',
				'el' => 'Greek',
				'he' => 'Hebrew',
				'hi' => 'Hindi',
				'hu' => 'Hungarian',
				'id' => 'Indonesian',
				'it' => 'Italian',
				'ja' => 'Japanese',
				'ko' => 'Korean',
				'lv' => 'Latvian',
				'lt' => 'Lithuanian',
				'mt' => 'Maltese',
				'no' => 'Norwegian',
				'fa' => 'Persian',
				'pl' => 'Polish',
				'pt' => 'Portuguese',
				'ro' => 'Romanian',
				'ru' => 'Russian',
				'sr' => 'Serbian',
				'sk' => 'Slovak',
				'sl' => 'Slovenian',
				'es' => 'Spanish',
				'sv' => 'Swedish',
				'th' => 'Thai',
				'tr' => 'Turkish',
				'uk' => 'Ukrainian',
				'vi' => 'Vietnamese');

   	var $sourceLang;
   	var $targetLang; 
   	var $markup;
   	var $translatingHTML = true;
   	var $googleAjaxUrl = "http://ajax.googleapis.com/ajax/services/language/translate?v=1.0";

//wiki markup (keep this regex in case we decide to translate wiki markup and not html)    
//	var $markup = "/<[^>]*>|[\s]?[\`\!\@\#\$\%\^\&\*\[\]\:\;\"\'\<\,\>\.\?\/\|\\\=\-\+\_\(\)]{2,}[\s]?|\{[^*]*?\}|\(\([\s\S]*?\)\)|\~[a-z]{2}\~[\s\S]*?\~\/[a-z]{2}\~|\~hs\~|\~\~[\s\S]*?\:|\~\~|[[^\|]*?\||\[[^|\]]*\]|\{[\*]?[^\}]*?[\*]?\}|\*|;/";

	var $wikiMarkup = "/<[^>]*>| ?[\`\!\@\#\$\%\^\&\*\[\]\:\;\"\'\<\,\>\/\|\\\=\-\+\_\(\)]{2,} ?|\(\([\s\S]*?\)\)|\~[a-z]{2}\~[\s\S]*?\~\/[a-z]{2}\~|\~hs\~|\~\~[\s\S]*?\:|\~\~|[[^\|]*?\||\[[^|\]]*\]|\{\*[^\}\*]*?\*\}|\{[^\}]*?\}|^;|!/m";

//Google doesn't return parens upon translation
//Include spaces in markup (Google adds some, and they will be stripped later. Want to preserve the original ones)
	var $htmlMarkup = "/ ?<[^>]*> ?| ?\(|\) ?/";

	var $titleTag = "/(<[Hh][\d][^>]*>(<[^>]*>)*)([^<]*)/";
	
		
   	var $escapeUntranslatableStrings = "<span class='notranslate'>$0</span>";
   	
    var $notranslateTag = "/(<span class='notranslate'>(.*)<\/span>)/U";
	var $notranslateTagWithSpaces = "/ <span class='notranslate'>(.*)<\/span> |^<span class='notranslate'>(.*)<\/span> | <span class='notranslate'>(.*)<\/span>$|^<span class='notranslate'>(.*)<\/span>$|<span class='notranslate'>(.*)<\/span>/Um";
   	var $arrayOfUntranslatableStringsAndTheirIDs = array();
   	var $currentID = 169;
   	
   	function __construct ($sourceLang, $targetLang, $html=true) {
   		$this->sourceLang = $sourceLang;
   		$this->targetLang = $targetLang;
   		if($html) {
   			$this->markup = $this->htmlMarkup;
   		} else {
   			$this->translatingHTML = false;
   			$this->markup = $this->wikiMarkup;
   		}
   	}<commit_msg>Added some more wiki syntax to the regex.<commit_after><?php
/*
 * Created on Jan 27, 2009
 *
 */
 
 
 
 require_once 'lib/ointegratelib.php';
 require_once 'Multilingual/Aligner/SentenceSegmentor.php'; 
 
class Multilingual_MachineTranslation_GoogleTranslateWrapper {

//this array should be updated as Google Translate
//adds more languages

	public $langsSupportedByGoogleTranslate = 
			array (
				'sq' => 'Albanian',
				'ar' => 'Arabic',
				'bg' => 'Bulgarian',
				'ca' => 'Catalan',
				'zh' => 'Chinese',
				'hr' => 'Croatian',
				'cs' => 'Czech',
				'da' => 'Danish',
				'nl' => 'Dutch',
				'en' => 'English',
				'et' => 'Estonian',
				'fil' => 'Filipino',
				'fi' => 'Finnish',
				'fr' => 'French',
				'gl' => 'Galician',
				'de' => 'German',
				'el' => 'Greek',
				'he' => 'Hebrew',
				'hi' => 'Hindi',
				'hu' => 'Hungarian',
				'id' => 'Indonesian',
				'it' => 'Italian',
				'ja' => 'Japanese',
				'ko' => 'Korean',
				'lv' => 'Latvian',
				'lt' => 'Lithuanian',
				'mt' => 'Maltese',
				'no' => 'Norwegian',
				'fa' => 'Persian',
				'pl' => 'Polish',
				'pt' => 'Portuguese',
				'ro' => 'Romanian',
				'ru' => 'Russian',
				'sr' => 'Serbian',
				'sk' => 'Slovak',
				'sl' => 'Slovenian',
				'es' => 'Spanish',
				'sv' => 'Swedish',
				'th' => 'Thai',
				'tr' => 'Turkish',
				'uk' => 'Ukrainian',
				'vi' => 'Vietnamese');

   	var $sourceLang;
   	var $targetLang; 
   	var $markup;
   	var $translatingHTML = true;
   	var $googleAjaxUrl = "http://ajax.googleapis.com/ajax/services/language/translate?v=1.0";

//wiki markup (keep this regex in case we decide to translate wiki markup and not html)    

//	var $wikiMarkup = "/<[^>]*>| ?[\`\!\@\#\$\%\^\&\*\[\]\:\;\"\'\<\,\>\/\|\\\=\-\+\_\(\)]{2,} ?|\(\([\s\S]*?\)\)|\~[a-z]{2,3}\~[\s\S]*?\~\/[a-z]{2,3}\~|\~hs\~|\~\~[\s\S]*?\:|\~\~|[[^\|]*?\||\[[^|\]]*\]|\{\*[^\}\*]*?\*\}|\{[^\}]*?\}|^;|!/m";
	var $wikiMarkup = "/<[^>]*>| ?[\`\!\@\#\$\%\^\&\*\[\]\:\;\"\'\<\,\>\/\|\\\=\-\+\_\(\)]{2,} ?|\(\([\s\S]*?\)\)|\~\/?[a-z]{2,3}\~|\~hs\~|\~\~[\s\S]*?\:|\~\~|[[^\|]*?\||\[[^|\]]*\]|\{\*[^\}\*]*?\*\}|\{[^\}]*?\}|^;|!/m";

//Google doesn't return parens upon translation
//Include spaces in markup (Google adds some, and they will be stripped later. Want to preserve the original ones)
	var $htmlMarkup = "/ ?<[^>]*> ?| ?\(|\) ?/";

	var $titleTag = "/(<[Hh][\d][^>]*>(<[^>]*>)*)([^<]*)/";
	
		
   	var $escapeUntranslatableStrings = "<span class='notranslate'>$0</span>";
   	
    var $notranslateTag = "/(<span class='notranslate'>(.*)<\/span>)/U";
	var $notranslateTagWithSpaces = "/ <span class='notranslate'>(.*)<\/span> |^<span class='notranslate'>(.*)<\/span> | <span class='notranslate'>(.*)<\/span>$|^<span class='notranslate'>(.*)<\/span>$|<span class='notranslate'>(.*)<\/span>/Um";
   	var $arrayOfUntranslatableStringsAndTheirIDs = array();
   	var $currentID = 169;
   	
   	function __construct ($sourceLang, $targetLang, $html=true) {
   		$this->sourceLang = $sourceLang;
   		$this->targetLang = $targetLang;
   		if($html) {
   			$this->markup = $this->htmlMarkup;
   		} else {
   			$this->translatingHTML = false;
   			$this->markup = $this->wikiMarkup;
   		}
   	}
{'fraction_non_alphanumeric': 0.3143754909662215, 'fraction_numerical': 0.005970149253731343, 'mean_word_length': 2.915129151291513, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 153, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '17966', 'n_tokens_mistral': 3076, 'n_tokens_neox': 2609, 'n_words': 440}