<commit_before>/*
 * (C) 2010-2012 ICM UW. All rights reserved.
 */

package pl.edu.icm.coansys.classification.documents.pig.proceeders;


import java.io.IOException;
import java.util.Arrays;

import org.apache.pig.EvalFunc;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataType;
import org.apache.pig.data.DefaultDataBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;

import pl.edu.icm.coansys.classification.documents.auxil.StackTraceExtractor;

/**
*
* @author pdendek
*/
public class THRES_FOR_CATEG2 extends EvalFunc<Tuple>{

	@Override
	public Schema outputSchema(Schema p_input){
		try{
			return Schema.generateNestedSchema(DataType.TUPLE, 
					DataType.CHARARRAY, DataType.INTEGER, DataType.DOUBLE);
		}catch(FrontendException e){
			throw new IllegalStateException(e);
		}
	}

	//C1: {categ: chararray,count: long,occ_pos: long,occ_neg: long}
	public Tuple exec(Tuple input) throws IOException {
		if (input == null || input.size() == 0)
			return null;
		try{
			Integer num = (Integer) input.get(0);
			String categ = (String) input.get(1);
			DataBag db = (DataBag) input.get(2);
			
			long[] pos = new long[num+1];//no of neighbours +1 for 0 count
			long[] neg = new long[num+1];
			Arrays.fill(pos, 0);
			Arrays.fill(neg, 0);
			for(Tuple t : db){
				long i  = (Long)t.get(1);
				long i1 = (Long)t.get(2);
				long i2 = (Long)t.get(3);
				
				pos[(int) i]= i1;
				neg[(int) i]= i2;
			}

			int thres = -1;
			double bestF1 = 0;
			
			
			for(int i = 1; i<num;i++){
				int TP = countLess(i,pos);
				int TN = countEqMore(i,pos);<commit_msg>bugfix<commit_after>/*
 * (C) 2010-2012 ICM UW. All rights reserved.
 */

package pl.edu.icm.coansys.classification.documents.pig.proceeders;


import java.io.IOException;
import java.util.Arrays;

import org.apache.pig.EvalFunc;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataType;
import org.apache.pig.data.DefaultDataBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;

import pl.edu.icm.coansys.classification.documents.auxil.StackTraceExtractor;

/**
*
* @author pdendek
*/
public class THRES_FOR_CATEG2 extends EvalFunc<Tuple>{

	@Override
	public Schema outputSchema(Schema p_input){
		try{
			return Schema.generateNestedSchema(DataType.TUPLE, 
					DataType.CHARARRAY, DataType.INTEGER, DataType.DOUBLE);
		}catch(FrontendException e){
			throw new IllegalStateException(e);
		}
	}

	//C1: {categ: chararray,count: long,occ_pos: long,occ_neg: long}
	public Tuple exec(Tuple input) throws IOException {
		if (input == null || input.size() == 0)
			return null;
		try{
			Integer num = (Integer) input.get(0);
			String categ = (String) input.get(1);
			DataBag db = (DataBag) input.get(2);
			
			long[] pos = new long[num+1];//no of neighbours +1 for 0 count
			long[] neg = new long[num+1];
			Arrays.fill(pos, 0);
			Arrays.fill(neg, 0);
			for(Tuple t : db){
				int i  = (Integer)t.get(1);
				long i1 = (Long)t.get(2);
				long i2 = (Long)t.get(3);
				
				pos[i]= i1;
				neg[i]= i2;
			}

			int thres = -1;
			double bestF1 = 0;
			
			
			for(int i = 1; i<num;i++){
				int TP = countLess(i,pos);
				int TN = countEqMore(i,pos);
{'fraction_non_alphanumeric': 0.14480953765629542, 'fraction_numerical': 0.01802849665600465, 'mean_word_length': 3.7058823529411766, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2283461', 'n_tokens_mistral': 1460, 'n_tokens_neox': 1279, 'n_words': 275}