<commit_before>/*
 * 	Copyright (C) 2012-2013 DFKI GmbH
 * 	Deutsches Forschungszentrum fuer Kuenstliche Intelligenz
 * 	German Research Center for Artificial Intelligence
 * 	http://www.dfki.de
 * 
 * 	Permission is hereby granted, free of charge, to any person obtaining a 
 * 	copy of this software and associated documentation files (the 
 * 	"Software"), to deal in the Software without restriction, including 
 * 	without limitation the rights to use, copy, modify, merge, publish, 
 * 	distribute, sublicense, and/or sell copies of the Software, and to 
 * 	permit persons to whom the Software is furnished to do so, subject to 
 * 	the following conditions:
 * 
 * 	The above copyright notice and this permission notice shall be included 
 * 	in all copies or substantial portions of the Software.
 * 
 * 	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
 * 	OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
 * 	MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 
 * 	IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 
 * 	CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 
 * 	TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 
 * 	SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */



/**
* @module mmir.semantic
* 
*/

/**
* The GrammarConverter object initializes the grammar for the voice recognition.
* 
* @class GrammarConverter
* @constructor
* @category core
* 
* @requires mmir.CommonUtils.isArray
* @requires jQuery.ajax
*/
define(['commonUtils', 'jquery'], function(commonUtils, $){

IS_DEBUG_ENABLED = true;//FIXME need to remove this!

/**
 * @class GrammarConverter
 */
var GrammarConverter = function(){
	
//	this.THE_INTERNAL_GRAMMAR_CONVERTER_INSTANCE_NAME = "theGrammarConverterInstance";
//	this.grammar_tokens = "/~ --- Token definitions --- ~/\n\n/~ Characters to be ignored ~/\n!   ' |\\t' ;\n\n/~ Non-associative tokens ~/\n";
//	this.grammar_utterances = "";
//	this.grammar_phrases = "phrases:";
//	this.token_variables = "[*\n  var " + this.variable_prefix
//			+ "result = '';\n";
//	this.tokens_array = new Array();
	
	this.variable_prefix = "_$";
	this.variable_regexp = /"(_\$[^\"]*)"/igm;// /"_$([^\"]*)/igm;
	
	//regular expression for detecting encoded chars (see mask/unmask functions)
	this.enc_regexp_str = "~~([0-9|A-F|a-f]{4})~~";

	this.jscc_grammar_definition = "";
	this.js_grammar_definition = "";
	this.json_grammar_definition = null;
	this.stop_words_regexp;
	
	//default setting for masking value Strings in JSON values (see maskJSON() / unmaskJSON)
	this.maskValues = true;
	//default setting for masking property-name Strings in JSON values (see maskJSON() / unmaskJSON)
	// WARNING: this is actually EXPERIMENTAL; it should be set to false, since JS/CC may not be able to handle masked ID names...
	this.maskNames = false;
	//default setting for loading JSON files:
	// if set to true, old-style umlauts encodings (e.g. __oe__) will converted after loading the file
	// Enable this, if you need to use old-style encoded grammars ... still, the better option would
	//  be to convert the old-style grammar (i.e. use un-encoded umlauts in the JSON grammar file).
	this.convertOldFormat = false;
	

	//alternative reg-exp for stop-words (a different method for detecting/removing stopwords must be used!)
	this.stop_words_regexp_alt;
	
};

GrammarConverter.prototype.loadGrammar = function(successCallback, errorCallback, grammarUrl, doLoadSynchronously){
	var self = this;
	var success = function(data){
		
		//DISABLED: old-style masking for umlauts:
//		data = self.recodeJSON(data, self.encodeUmlauts);
		
		//if auto-upgrading is enabled:
		//   decode old-style umlaut masking before continuing
		if(self.convertOldFormat){
			data = self.recodeJSON(data, self.decodeUmlauts);
		}
		
		self.json_grammar_definition = data;
		
		if (typeof successCallback == "function") {
			successCallback(self);
		}
	};
	var error = function(data){
		alert("failed to load the grammar! error: "+ JSON.stringify(data));
		if (typeof errorCallback == "function") {
			errorCallback(self);
		}
	};
	this.loadResource(success, error, grammarUrl, doLoadSynchronously);
};

GrammarConverter.prototype.loadResource = function(successCallback, errorCallback, resourceUrl, doLoadSynchronously){

	var theUrl = resourceUrl;
	if(!theUrl){
		if(errorCallback){
			console.error('GrammarConverter.loadResource: missing URL!');
			errorCallback(this);
		}
		return;///////////////// EARLY EXIT //////////////////////
	}
	<commit_msg>GrammarConverter.loadGrammar: the callback functions are now called in context of the GrammarConverter instance and with additional arguments (e.g. in case of error)<commit_after>/*
 * 	Copyright (C) 2012-2013 DFKI GmbH
 * 	Deutsches Forschungszentrum fuer Kuenstliche Intelligenz
 * 	German Research Center for Artificial Intelligence
 * 	http://www.dfki.de
 * 
 * 	Permission is hereby granted, free of charge, to any person obtaining a 
 * 	copy of this software and associated documentation files (the 
 * 	"Software"), to deal in the Software without restriction, including 
 * 	without limitation the rights to use, copy, modify, merge, publish, 
 * 	distribute, sublicense, and/or sell copies of the Software, and to 
 * 	permit persons to whom the Software is furnished to do so, subject to 
 * 	the following conditions:
 * 
 * 	The above copyright notice and this permission notice shall be included 
 * 	in all copies or substantial portions of the Software.
 * 
 * 	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
 * 	OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
 * 	MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 
 * 	IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 
 * 	CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 
 * 	TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 
 * 	SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */



/**
* @module mmir.semantic
* 
*/

/**
* The GrammarConverter object initializes the grammar for the voice recognition.
* 
* @class GrammarConverter
* @constructor
* @category core
* 
* @requires mmir.CommonUtils.isArray
* @requires jQuery.ajax
*/
define(['commonUtils', 'jquery'], function(commonUtils, $){

IS_DEBUG_ENABLED = true;//FIXME need to remove this!

/**
 * @class GrammarConverter
 */
var GrammarConverter = function(){
	
//	this.THE_INTERNAL_GRAMMAR_CONVERTER_INSTANCE_NAME = "theGrammarConverterInstance";
//	this.grammar_tokens = "/~ --- Token definitions --- ~/\n\n/~ Characters to be ignored ~/\n!   ' |\\t' ;\n\n/~ Non-associative tokens ~/\n";
//	this.grammar_utterances = "";
//	this.grammar_phrases = "phrases:";
//	this.token_variables = "[*\n  var " + this.variable_prefix
//			+ "result = '';\n";
//	this.tokens_array = new Array();
	
	this.variable_prefix = "_$";
	this.variable_regexp = /"(_\$[^\"]*)"/igm;// /"_$([^\"]*)/igm;
	
	//regular expression for detecting encoded chars (see mask/unmask functions)
	this.enc_regexp_str = "~~([0-9|A-F|a-f]{4})~~";

	this.jscc_grammar_definition = "";
	this.js_grammar_definition = "";
	this.json_grammar_definition = null;
	this.stop_words_regexp;
	
	//default setting for masking value Strings in JSON values (see maskJSON() / unmaskJSON)
	this.maskValues = true;
	//default setting for masking property-name Strings in JSON values (see maskJSON() / unmaskJSON)
	// WARNING: this is actually EXPERIMENTAL; it should be set to false, since JS/CC may not be able to handle masked ID names...
	this.maskNames = false;
	//default setting for loading JSON files:
	// if set to true, old-style umlauts encodings (e.g. __oe__) will converted after loading the file
	// Enable this, if you need to use old-style encoded grammars ... still, the better option would
	//  be to convert the old-style grammar (i.e. use un-encoded umlauts in the JSON grammar file).
	this.convertOldFormat = false;
	

	//alternative reg-exp for stop-words (a different method for detecting/removing stopwords must be used!)
	this.stop_words_regexp_alt;
	
};

GrammarConverter.prototype.loadGrammar = function(successCallback, errorCallback, grammarUrl, doLoadSynchronously){
	var self = this;
	var success = function(data, status, xhr){
		
		//DISABLED: old-style masking for umlauts:
//		data = self.recodeJSON(data, self.encodeUmlauts);
		
		//if auto-upgrading is enabled:
		//   decode old-style umlaut masking before continuing
		if(self.convertOldFormat){
			data = self.recodeJSON(data, self.decodeUmlauts);
		}
		
		self.json_grammar_definition = data;
		
		if (typeof successCallback == "function") {
			successCallback.call(this, self, xhr);
		}
	};
	var error = function(xhr, status, data){
		alert("failed to load the grammar! error: "+ JSON.stringify(data));
		if (typeof errorCallback == "function") {
			errorCallback.call(this, self);
		}
	};
	this.loadResource(success, error, grammarUrl, doLoadSynchronously);
};

GrammarConverter.prototype.loadResource = function(successCallback, errorCallback, resourceUrl, doLoadSynchronously){

	var theUrl = resourceUrl;
	if(!theUrl){
		console.error('GrammarConverter.loadResource: missing URL!');
		if(errorCallback){
			errorCallback.call(this, this);
		}
		return;///////////////// EARLY EXIT //////////////////////
	}
	
{'fraction_non_alphanumeric': 0.11824288303657106, 'fraction_numerical': 0.0023456658492376587, 'mean_word_length': 3.8727272727272726, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5974695', 'n_tokens_mistral': 3056, 'n_tokens_neox': 2832, 'n_words': 941}