<commit_before>package org.encog.app.analyst.csv.transform;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.encog.app.analyst.AnalystError;
import org.encog.app.analyst.csv.basic.LoadedRow;
import org.encog.parse.expression.ExpressionHolder;
import org.encog.parse.expression.ExpressionTreeElement;
import org.encog.parse.expression.ExpressionTreeFunction;
import org.encog.parse.expression.extension.ExpressionExtension;
import org.encog.util.csv.CSVFormat;
import org.encog.util.csv.ReadCSV;

public class ProcessExtension implements ExpressionExtension {

	private Map<String,Integer> map = new HashMap<String,Integer>();
	private int forwardWindowSize;
	private int backwardWindowSize;
	private int totalWindowSize;
	private List<LoadedRow> data = new ArrayList<LoadedRow>();
	private final CSVFormat format;
	
	ProcessExtension(CSVFormat theFormat) {
		this.format = theFormat;
	}
	
	@Override
	public ExpressionTreeFunction factorFunction(ExpressionHolder theOwner,
			String theName, List<ExpressionTreeElement> theArgs) {
		if (theName.equals("field")) {
			return new FunctionField(this, theOwner, theArgs);
		} if (theName.equals("fieldmax")) {
			return new FunctionFieldMax(this, theOwner, theArgs);
		} else {
			return null;
		}
	}

	public String getField(String fieldName, int fieldIndex) {
		if( !map.containsKey(fieldName) ) {
			throw new AnalystError("Unknown input field: " + fieldName);<commit_msg>Added PIP function to process<commit_after>package org.encog.app.analyst.csv.transform;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.encog.app.analyst.AnalystError;
import org.encog.app.analyst.csv.basic.LoadedRow;
import org.encog.parse.expression.ExpressionHolder;
import org.encog.parse.expression.ExpressionTreeElement;
import org.encog.parse.expression.ExpressionTreeFunction;
import org.encog.parse.expression.extension.ExpressionExtension;
import org.encog.util.csv.CSVFormat;
import org.encog.util.csv.ReadCSV;

public class ProcessExtension implements ExpressionExtension {

	private Map<String,Integer> map = new HashMap<String,Integer>();
	private int forwardWindowSize;
	private int backwardWindowSize;
	private int totalWindowSize;
	private List<LoadedRow> data = new ArrayList<LoadedRow>();
	private final CSVFormat format;
	
	ProcessExtension(CSVFormat theFormat) {
		this.format = theFormat;
	}
	
	@Override
	public ExpressionTreeFunction factorFunction(ExpressionHolder theOwner,
			String theName, List<ExpressionTreeElement> theArgs) {
		if (theName.equals("field")) {
			return new FunctionField(this, theOwner, theArgs);
		} if (theName.equals("fieldmax")) {
			return new FunctionFieldMax(this, theOwner, theArgs);
		} if (theName.equals("fieldmaxpip")) {
			return new FunctionFieldMaxPIP(this, theOwner, theArgs);
		} else {
			return null;
		}
	}

	public String getField(String fieldName, int fieldIndex) {
		if( !map.containsKey(fieldName) ) {
			throw new AnalystError("Unknown input field: " + fieldName);
{'fraction_non_alphanumeric': 0.10307941653160453, 'fraction_numerical': 0.0, 'mean_word_length': 5.827433628318584, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '680492', 'n_tokens_mistral': 963, 'n_tokens_neox': 913, 'n_words': 206}