<commit_before>package org.jpmml.itest;

import java.io.StringReader;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.dmg.pmml.FieldName;
import org.dmg.pmml.OpType;
import org.dmg.pmml.PMML;
import org.jpmml.evaluator.EvaluationException;
import org.jpmml.evaluator.Evaluator;
import org.jpmml.evaluator.ModelEvaluatorFactory;
import org.jpmml.evaluator.RegressionModelEvaluator;
import org.jpmml.manager.PMMLManager;
import org.jpmml.translator.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class BaseModelTest {
	private static final Logger logger = LoggerFactory.getLogger(BaseModelTest.class);
	
	protected double getMissingVarProbability() {
		return 0.1;
	}

	public void runSingleModelEvaluation(PMML pmmlDoc,
			String codeTemplate, 
			ManualModelImplementation manual,
			Map<String, Object> variableValues
			) throws Exception {

		// creating evaluator
		PMMLManager pmmlManager = new PMMLManager(pmmlDoc);
		Evaluator evaluator = (Evaluator)pmmlManager.getModelManager(null, ModelEvaluatorFactory.getInstance());

		// translate and compile
		CompiledModel compiledModel = createCompiledModel(pmmlDoc, codeTemplate);
		
		executeAndCompareOutput(0, compiledModel, evaluator, manual, variableValues);
	}
	
	public void testModelEvaluation(PMML pmmlDoc,
			String codeTemplate, 
			ManualModelImplementation manual,
			Map<String, List<?>> variables,
			final int iterations) throws Exception {

		// creating evaluator
		PMMLManager pmmlManager = new PMMLManager(pmmlDoc);
		Evaluator evaluator = (Evaluator)pmmlManager.getModelManager(null, ModelEvaluatorFactory.getInstance());

		// translate and compile
		CompiledModel compiledModel = createCompiledModel(pmmlDoc, codeTemplate);
		
		
		for (int i=0;i<iterations; i++) {
			// generate random variables
			Map<String, Object> nameToValue = new HashMap<String, Object>();
			for (Map.Entry<String, List<?>> e : variables.entrySet()) {
				// skip variable in 10% cases
 				if (Math.random()>getMissingVarProbability()) {
 					// use one of predefined values
 					if (e.getValue()!=null) {
 						int index = (int)(Math.random()*e.getValue().size());
 						if (index == e.getValue().size()) {
 							index--;
 						}
 						Object value = e.getValue().get(index);
 						nameToValue.put(e.getKey(), value);
 					}
 					else {
 						// otherwise generate random number between 0 and 100
 						nameToValue.put(e.getKey(), 100.0*Math.random());
 					}
 				}
 				try {
				executeAndCompareOutput(i, compiledModel, evaluator, manual, nameToValue);
 				}
 				catch (EvaluationException ee) {
 					if (ee.getMessage().startsWith("Missing parameter ")
 							&& evaluator instanceof RegressionModelEvaluator) {
 						// This is fine, the way we generate our test
 						// doesn't fit with the input for the regression model.
 						// So in order to keep this way of thinking, we just
 						// remove the exception for this particular case.
 					}
 					else {
 						// Otherwise, this exception is not normal, and should be sent
 						// to the user.
 						throw ee;
 					}
 				}
			}
		}		
	}
	
	private CompiledModel createCompiledModel(PMML pmmlDoc, String codeTemplate) throws Exception {
		//InputStream is = getClass().getResourceAsStream("/codetemplate.vm");
		
		String className = "TestModel" + System.currentTimeMillis();
				
		String javaSource = PmmlToJavaTranslator.generateJavaCode(pmmlDoc, className, 
				new StringReader(codeTemplate), 
				new TranslationContext() {
					// override missing value method, since in our template numeric variables represented with Double class
					public String getMissingValue(OpType variableType) {
						if (variableType==OpType.CONTINUOUS) return "null";
						return super.getMissingValue(variableType);
					}
					
					public String getModelResultTrackingVariable() {
						return "resultExplanation";
					}
				}
			);

		Class<?> modelClass = PmmlToJavaTranslator.createModelClass(className, "org.jpmml.itest", javaSource);
		
		return (CompiledModel)modelClass.newInstance();
	}

	public void executeAndCompareOutput(int iteration, 
			CompiledModel pmmlModel,
			Evaluator evaluator,
			ManualModelImplementation manual,
			Map<String, Object> nameToValue) {

		Object value1 = pmmlModel.execute(nameToValue);
		Object value2 = manual.execute(nameToValue);
		
		compareValues(iteration, nameToValue, value1, value2, pmmlModel.getResultExplanation(), manual.getResultExplanation());

		// if we get here then value1==value2
		// now evaluate value3 and compare against value1
		Object value3 = evaluateModel(evaluator, nameToValue);
		
		// Fake for the result explanation, because evaluator.getResultExplanation doesn't exist.
		compareValues(iteration, nameToValue, value2, value3, null, null);
	}
	
	protected Object evaluateModel(Evaluator evaluator, Map<String, Object> nameToValue) {
		Map<FieldName, Object> fieldToValue = new HashMap<FieldName, Object>();
		for (Map.Entry<String, Object> entry : nameToValue.entrySet()) {
			fieldToValue.put(new FieldName(entry.getKey()), entry.getValue());
		}
		return evaluator.evaluate(fieldToValue);
	}

	private void compareValues(int iteration, Map<String, Object> nameToValue, 
			Object value1,
			Object value2, String explanation1, String explanation2 
			) {
		if ((value1==null && value2!=null)
				|| (value1!=null && value2==null)
				|| (explanation1 == null && explanation2 != null)
				|| (explanation1 != null && explanation2 == null)
				|| (value1!=null && value2!=null && !value1.equals(value2))
				|| (explanation1 != null && explanation2 != null && !explanation1.equals(explanation2))) {
			logger.info("Test failed. Value1 = " + value1 + "; value2 = " + value2 + "; explanation1 = "
				+ explanation1 + "; explanation2 = " + explanation2);
			for (Map.Entry<String, Object> e : nameToValue.entrySet()) {
				logger.info(e.getKey() + " = " + e.getValue());
			}
		}
		
		if (value1 != null) {
			assert value1.equals(value2);
		}
		else if (value2 != null) {
			assert value2.equals(value1);
		}
		else {
			assert value1 == value2;
		}<commit_msg>Add an information on which test is run on BaseModelTest.<commit_after>package org.jpmml.itest;

import java.io.StringReader;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.dmg.pmml.FieldName;
import org.dmg.pmml.OpType;
import org.dmg.pmml.PMML;
import org.jpmml.evaluator.EvaluationException;
import org.jpmml.evaluator.Evaluator;
import org.jpmml.evaluator.ModelEvaluatorFactory;
import org.jpmml.evaluator.RegressionModelEvaluator;
import org.jpmml.manager.PMMLManager;
import org.jpmml.translator.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class BaseModelTest {
	private static final Logger logger = LoggerFactory.getLogger(BaseModelTest.class);
	
	protected double getMissingVarProbability() {
		return 0.1;
	}

	public void runSingleModelEvaluation(PMML pmmlDoc,
			String codeTemplate, 
			ManualModelImplementation manual,
			Map<String, Object> variableValues
			) throws Exception {

		// creating evaluator
		PMMLManager pmmlManager = new PMMLManager(pmmlDoc);
		Evaluator evaluator = (Evaluator)pmmlManager.getModelManager(null, ModelEvaluatorFactory.getInstance());

		// translate and compile
		CompiledModel compiledModel = createCompiledModel(pmmlDoc, codeTemplate);
		
		executeAndCompareOutput(0, compiledModel, evaluator, manual, variableValues);
	}
	
	public void testModelEvaluation(PMML pmmlDoc,
			String codeTemplate, 
			ManualModelImplementation manual,
			Map<String, List<?>> variables,
			final int iterations) throws Exception {

		// creating evaluator
		PMMLManager pmmlManager = new PMMLManager(pmmlDoc);
		Evaluator evaluator = (Evaluator)pmmlManager.getModelManager(null, ModelEvaluatorFactory.getInstance());

		// translate and compile
		CompiledModel compiledModel = createCompiledModel(pmmlDoc, codeTemplate);
		
		
		for (int i=0;i<iterations; i++) {
			// generate random variables
			Map<String, Object> nameToValue = new HashMap<String, Object>();
			for (Map.Entry<String, List<?>> e : variables.entrySet()) {
				// skip variable in 10% cases
 				if (Math.random()>getMissingVarProbability()) {
 					// use one of predefined values
 					if (e.getValue()!=null) {
 						int index = (int)(Math.random()*e.getValue().size());
 						if (index == e.getValue().size()) {
 							index--;
 						}
 						Object value = e.getValue().get(index);
 						nameToValue.put(e.getKey(), value);
 					}
 					else {
 						// otherwise generate random number between 0 and 100
 						nameToValue.put(e.getKey(), 100.0*Math.random());
 					}
 				}
 				try {
				executeAndCompareOutput(i, compiledModel, evaluator, manual, nameToValue);
 				}
 				catch (EvaluationException ee) {
 					if (ee.getMessage().startsWith("Missing parameter ")
 							&& evaluator instanceof RegressionModelEvaluator) {
 						// This is fine, the way we generate our test
 						// doesn't fit with the input for the regression model.
 						// So in order to keep this way of thinking, we just
 						// remove the exception for this particular case.
 					}
 					else {
 						// Otherwise, this exception is not normal, and should be sent
 						// to the user.
 						throw ee;
 					}
 				}
			}
		}		
	}
	
	private CompiledModel createCompiledModel(PMML pmmlDoc, String codeTemplate) throws Exception {
		//InputStream is = getClass().getResourceAsStream("/codetemplate.vm");
		
		String className = "TestModel" + System.currentTimeMillis();
				
		String javaSource = PmmlToJavaTranslator.generateJavaCode(pmmlDoc, className, 
				new StringReader(codeTemplate), 
				new TranslationContext() {
					// override missing value method, since in our template numeric variables represented with Double class
					public String getMissingValue(OpType variableType) {
						if (variableType==OpType.CONTINUOUS) return "null";
						return super.getMissingValue(variableType);
					}
					
					public String getModelResultTrackingVariable() {
						return "resultExplanation";
					}
				}
			);

		Class<?> modelClass = PmmlToJavaTranslator.createModelClass(className, "org.jpmml.itest", javaSource);
		
		return (CompiledModel)modelClass.newInstance();
	}

	public void executeAndCompareOutput(int iteration, 
			CompiledModel pmmlModel,
			Evaluator evaluator,
			ManualModelImplementation manual,
			Map<String, Object> nameToValue) {

		Object value1 = pmmlModel.execute(nameToValue);
		Object value2 = manual.execute(nameToValue);
		
		compareValues(iteration, nameToValue, value1, value2, pmmlModel.getResultExplanation(), manual.getResultExplanation(),
				false);

		// if we get here then value1==value2
		// now evaluate value3 and compare against value1
		Object value3 = evaluateModel(evaluator, nameToValue);
		
		// Fake for the result explanation, because evaluator.getResultExplanation doesn't exist.
		compareValues(iteration, nameToValue, value2, value3, null, null, true);
	}
	
	protected Object evaluateModel(Evaluator evaluator, Map<String, Object> nameToValue) {
		Map<FieldName, Object> fieldToValue = new HashMap<FieldName, Object>();
		for (Map.Entry<String, Object> entry : nameToValue.entrySet()) {
			fieldToValue.put(new FieldName(entry.getKey()), entry.getValue());
		}
		return evaluator.evaluate(fieldToValue);
	}

	private void compareValues(int iteration, Map<String, Object> nameToValue, 
			Object value1,
			Object value2, String explanation1, String explanation2, boolean secondTest 
			) {
		if ((value1==null && value2!=null)
				|| (value1!=null && value2==null)
				|| (explanation1 == null && explanation2 != null)
				|| (explanation1 != null && explanation2 == null)
				|| (value1!=null && value2!=null && !value1.equals(value2))
				|| (explanation1 != null && explanation2 != null && !explanation1.equals(explanation2))) {
			logger.info((secondTest ? "Second " : "First ") + "test failed. Value1 = " + value1 + "; value2 = " + value2 + "; explanation1 = "
				+ explanation1 + "; explanation2 = " + explanation2);
			for (Map.Entry<String, Object> e : nameToValue.entrySet()) {
				logger.info(e.getKey() + " = " + e.getValue());
			}
		}
		
		if (value1 != null) {
			assert value1.equals(value2);
		}
		else if (value2 != null) {
			assert value2.equals(value1);
		}
		else {
			assert value1 == value2;
		}
{'fraction_non_alphanumeric': 0.1118117875333495, 'fraction_numerical': 0.010186757215619695, 'mean_word_length': 3.8719968491532097, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6341964', 'n_tokens_mistral': 4461, 'n_tokens_neox': 3716, 'n_words': 902}