<commit_before>/*
 * Copyright (c) 2013 <NAME>
 *
 * This file is part of JPMML-Evaluator
 *
 * JPMML-Evaluator is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * JPMML-Evaluator is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with JPMML-Evaluator.  If not, see <http://www.gnu.org/licenses/>.
 */
package org.jpmml.evaluator;

import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import com.google.common.base.Equivalence;
import com.google.common.base.Function;
import com.google.common.collect.MapDifference;
import com.google.common.collect.Maps;
import org.dmg.pmml.FieldName;

public class BatchUtil {

	private BatchUtil(){
	}

	static
	public List<Conflict> evaluate(Batch batch, Set<FieldName> ignoredFields, final double precision, final double zeroThreshold) throws Exception {
		Evaluator evaluator = batch.getEvaluator();

		List<? extends Map<FieldName, ?>> input = batch.getInput();
		List<? extends Map<FieldName, ?>> output = batch.getOutput();

		List<FieldName> groupFields = evaluator.getGroupFields();
		List<FieldName> targetFields = evaluator.getTargetFields();

		if(groupFields.size() == 1){
			FieldName groupField = groupFields.get(0);

			input = EvaluatorUtil.groupRows(groupField, input);
		} else

		if(groupFields.size() > 1){
			throw new EvaluationException();
		}

		Equivalence<Object> equivalence = new Equivalence<Object>(){

			@Override
			public boolean doEquivalent(Object expected, Object actual){
				actual = EvaluatorUtil.decode(actual);

				return VerificationUtil.acceptable(TypeUtil.parseOrCast(TypeUtil.getDataType(actual), expected), actual, precision, zeroThreshold);
			}

			@Override
			public int doHash(Object object){
				return object.hashCode();
			}
		};

		if(output.size() > 0){

			if(input.size() != output.size()){
				throw new EvaluationException();
			}

			List<Conflict> conflicts = new ArrayList<>();

			for(int i = 0; i < input.size(); i++){
				Map<FieldName, ?> arguments = input.get(i);

				Map<FieldName, ?> result = evaluator.evaluate(arguments);

				// Delete the default target field
				if(targetFields.size() == 0){
					result = new LinkedHashMap<>(result);

					result.remove(evaluator.getTargetField());
				} // End if

				if(ignoredFields != null && ignoredFields.size() > 0){
					result = new LinkedHashMap<>(result);

					Set<FieldName> fields = result.keySet();
					fields.removeAll(ignoredFields);
				}

				MapDifference<FieldName, ?> difference = Maps.<FieldName, Object>difference(output.get(i), result, equivalence);
				if(!difference.areEqual()){
					Conflict conflict = new Conflict(i, arguments, difference);

					conflicts.add(conflict);
				}
			}

			return conflicts;
		} else

		{
			for(int i = 0; i < input.size(); i++){
				Map<FieldName, ?> arguments = input.get(i);

				evaluator.evaluate(arguments);
			}

			return Collections.emptyList();
		}
	}

	/**
	 * <p>
	 * Evaluates the model using empty arguments.
	 * </p>
	 *
	 * @return The value of the target field.
	 */
	static
	public Object evaluateDefault(Batch batch) throws Exception {
		Evaluator evaluator = batch.getEvaluator();

		Map<FieldName, ?> arguments = Collections.emptyMap();

		Map<FieldName, ?> result = evaluator.evaluate(arguments);

		return result.get(evaluator.getTargetField());
	}

	static
	public List<Map<FieldName, String>> parseRecords(List<List<String>> table, Function<String, String> function){
		List<Map<FieldName, String>> records = new ArrayList<>();

		List<String> headerRow = table.get(0);

		for(int i = 1; i < table.size(); i++){
			List<String> bodyRow = table.get(i);

			if(headerRow.size() != bodyRow.size()){
				throw new IllegalArgumentException("Expected " + headerRow.size() + " cells, but got " + bodyRow.size() + " cells (data record " + (i - 1) + ")");
			}

			Map<FieldName, String> record = new LinkedHashMap<>();

			for(int j = 0; j < headerRow.size(); j++){
				record.put(FieldName.create(headerRow.get(j)), function.apply(bodyRow.get(j)));
			}
<commit_msg>Checking the column layout of input CSV file<commit_after>/*
 * Copyright (c) 2013 <NAME>
 *
 * This file is part of JPMML-Evaluator
 *
 * JPMML-Evaluator is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * JPMML-Evaluator is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with JPMML-Evaluator.  If not, see <http://www.gnu.org/licenses/>.
 */
package org.jpmml.evaluator;

import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import com.google.common.base.Equivalence;
import com.google.common.base.Function;
import com.google.common.collect.MapDifference;
import com.google.common.collect.Maps;
import org.dmg.pmml.FieldName;

public class BatchUtil {

	private BatchUtil(){
	}

	static
	public List<Conflict> evaluate(Batch batch, Set<FieldName> ignoredFields, final double precision, final double zeroThreshold) throws Exception {
		Evaluator evaluator = batch.getEvaluator();

		List<? extends Map<FieldName, ?>> input = batch.getInput();
		List<? extends Map<FieldName, ?>> output = batch.getOutput();

		List<FieldName> groupFields = evaluator.getGroupFields();
		List<FieldName> targetFields = evaluator.getTargetFields();

		if(groupFields.size() == 1){
			FieldName groupField = groupFields.get(0);

			input = EvaluatorUtil.groupRows(groupField, input);
		} else

		if(groupFields.size() > 1){
			throw new EvaluationException();
		}

		Equivalence<Object> equivalence = new Equivalence<Object>(){

			@Override
			public boolean doEquivalent(Object expected, Object actual){
				actual = EvaluatorUtil.decode(actual);

				return VerificationUtil.acceptable(TypeUtil.parseOrCast(TypeUtil.getDataType(actual), expected), actual, precision, zeroThreshold);
			}

			@Override
			public int doHash(Object object){
				return object.hashCode();
			}
		};

		if(output.size() > 0){

			if(input.size() != output.size()){
				throw new EvaluationException();
			}

			List<Conflict> conflicts = new ArrayList<>();

			for(int i = 0; i < input.size(); i++){
				Map<FieldName, ?> arguments = input.get(i);

				Map<FieldName, ?> result = evaluator.evaluate(arguments);

				// Delete the default target field
				if(targetFields.size() == 0){
					result = new LinkedHashMap<>(result);

					result.remove(evaluator.getTargetField());
				} // End if

				if(ignoredFields != null && ignoredFields.size() > 0){
					result = new LinkedHashMap<>(result);

					Set<FieldName> fields = result.keySet();
					fields.removeAll(ignoredFields);
				}

				MapDifference<FieldName, ?> difference = Maps.<FieldName, Object>difference(output.get(i), result, equivalence);
				if(!difference.areEqual()){
					Conflict conflict = new Conflict(i, arguments, difference);

					conflicts.add(conflict);
				}
			}

			return conflicts;
		} else

		{
			for(int i = 0; i < input.size(); i++){
				Map<FieldName, ?> arguments = input.get(i);

				evaluator.evaluate(arguments);
			}

			return Collections.emptyList();
		}
	}

	/**
	 * <p>
	 * Evaluates the model using empty arguments.
	 * </p>
	 *
	 * @return The value of the target field.
	 */
	static
	public Object evaluateDefault(Batch batch) throws Exception {
		Evaluator evaluator = batch.getEvaluator();

		Map<FieldName, ?> arguments = Collections.emptyMap();

		Map<FieldName, ?> result = evaluator.evaluate(arguments);

		return result.get(evaluator.getTargetField());
	}

	static
	public List<Map<FieldName, String>> parseRecords(List<List<String>> table, Function<String, String> function){
		List<Map<FieldName, String>> records = new ArrayList<>();

		List<String> headerRow = table.get(0);

		Set<String> uniqueHeaderRow = new LinkedHashSet<>(headerRow);

		if(uniqueHeaderRow.size() < headerRow.size()){
			List<String> duplicateCells = new ArrayList<>();

			for(int j = 0; j < headerRow.size(); j++){
				String cell = headerRow.get(j);

				if(Collections.frequency(headerRow, cell) != 1){
					duplicateCells.add(cell);
				}
			}

			throw new IllegalArgumentException("Expected unique cell names, but got non-unique cell name(s) " + duplicateCells);
		}

		for(int i = 1; i < table.size(); i++){
			List<String> bodyRow = table.get(i);

			if(headerRow.size() != bodyRow.size()){
				throw new IllegalArgumentException("Expected " + headerRow.size() + " cells, but got " + bodyRow.size() + " cells (data record " + (i - 1) + ")");
			}

			Map<FieldName, String> record = new LinkedHashMap<>();

			for(int j = 0; j < headerRow.size(); j++){
				record.put(FieldName.create(headerRow.get(j)), function.apply(bodyRow.get(j)));
			}

{'fraction_non_alphanumeric': 0.13202479338842976, 'fraction_numerical': 0.00371900826446281, 'mean_word_length': 4.0579937304075235, 'pattern_counts': {'":': 0, '<': 93, '<?xml version=': 0, '>': 89, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2403426', 'n_tokens_mistral': 3339, 'n_tokens_neox': 2866, 'n_words': 796}