<commit_before>/*
 *  Copyright (c) 1995-2015, The University of Sheffield. See the file
 *  COPYRIGHT.txt in the software or at http://gate.ac.uk/gate/COPYRIGHT.txt
 *
 *  This file is part of GATE (see http://gate.ac.uk/), and is free
 *  software, licenced under the GNU Library General Public License,
 *  Version 2, June 1991 (in the distribution as file licence.html,
 *  and also available at http://gate.ac.uk/gate/licence.html).
 *
 */
package gate.plugin.evaluation.resources;

import gate.plugin.evaluation.api.ContainmentType;
import gate.plugin.evaluation.api.NilTreatment;
import gate.Annotation;
import gate.AnnotationSet;
import gate.Controller;
import gate.Factory;
import gate.FeatureMap;
import gate.Resource;
import gate.Utils;
import gate.annotation.AnnotationSetImpl;
import gate.annotation.ImmutableAnnotationSetImpl;
import gate.creole.AbstractLanguageAnalyser;
import gate.creole.ControllerAwarePR;
import gate.creole.CustomDuplication;
import gate.creole.ExecutionException;
import gate.creole.ResourceInstantiationException;
import gate.creole.metadata.CreoleParameter;
import gate.creole.metadata.CreoleResource;
import gate.creole.metadata.Optional;
import gate.creole.metadata.RunTime;
import gate.plugin.evaluation.api.AnnotationDifferTagging;
import gate.plugin.evaluation.api.AnnotationDifferTagging.CandidateList;
import gate.plugin.evaluation.api.ByThEvalStatsTagging;
import gate.plugin.evaluation.api.EvalStatsTagging;
import gate.plugin.evaluation.api.EvalStatsTaggingMacro;
import gate.plugin.evaluation.api.FeatureComparison;
import gate.plugin.evaluation.api.ThresholdsToUse;
import gate.util.Files;
import gate.util.GateRuntimeException;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.PrintStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.log4j.Logger;

// TODO: add a list for getting the distribution of ranks/score of first lenient/exact match
// if we process lists. 
// This should get output as a separate tsv both per document and over all documents
// Maybe only if enabled with a parameter.
// If we process lists, also output per-document information about what the best is we can do
// in that document with the candidate lists we have.

// TODO: add a datastructure for incremental update of the contingency tables needed for 
// McNemar etc, if we have a reference set.
// These should get output at the end with values over all documents

// TODO: think about how this should deal with parallelization and custom duplication.
// At the moment this will not work properly or even fail when run from duplicated pipelines. 
// In the best case, each duplicate will just report the statistics on its own subset of documents,
// in the worst case, some resources will override each other (e.g. when writing from each 
// duplicate to the tsv files). 
// The proper way to deal with this requires that we have a standard way of how to share 
// data between those custom duplicated instances of the PR which belong to the same job (in a 
// larger VM, there could duplicates that belong to different jobs between which we do not want
// to share data).
// For now, we just prevent the multithreaded use of this PR by throwing an exception if an 
// attempt is made to custom-duplicate it. 
// Later on, we could change this such that all duplicated instances just call the methods of
// the original in a synchronized way.
// Finally, once we can calculate overall stats on all stats objects (including the byThreshold
// objects) we can try to support multi-threading in the following way:
// = all per-document io needs to go through a single instance synchronized method, or each 
//   instance creates their own output file
// = each duplicate creates their own stats objects over all documents it sees
// = there is a way to access all the stats objects of all the other duplicates somehow, or 
//   ideally there is a way to do this for just the "original" 
// = the controllerExecutionFinished method either knows if it is running for the original or 
//   can somehow make sure that the actual finishing code is only run for one of the duplicates.
//   That finishing code that would get the stats obejcts from all duplicates and merge/sum them,
//   then output the final statistics. 


// TODO(!!!): Add a java class for holding counts or instances of pairings between the reference set
// and the response set so we can calculate p-values for the SingleResponse accuracy analysis. 
// This could be used to get p-values for the McNemar test and the paired t-test.

/**
 *
 * @author <NAME>
 */
@CreoleResource(name = "EvaluatePRF",
        comment = "Calculate P/R/F evalutation measures for documents")<commit_msg>Add comment for a TODO.<commit_after>/*
 *  Copyright (c) 1995-2015, The University of Sheffield. See the file
 *  COPYRIGHT.txt in the software or at http://gate.ac.uk/gate/COPYRIGHT.txt
 *
 *  This file is part of GATE (see http://gate.ac.uk/), and is free
 *  software, licenced under the GNU Library General Public License,
 *  Version 2, June 1991 (in the distribution as file licence.html,
 *  and also available at http://gate.ac.uk/gate/licence.html).
 *
 */
package gate.plugin.evaluation.resources;

import gate.plugin.evaluation.api.ContainmentType;
import gate.plugin.evaluation.api.NilTreatment;
import gate.Annotation;
import gate.AnnotationSet;
import gate.Controller;
import gate.Factory;
import gate.FeatureMap;
import gate.Resource;
import gate.Utils;
import gate.annotation.AnnotationSetImpl;
import gate.annotation.ImmutableAnnotationSetImpl;
import gate.creole.AbstractLanguageAnalyser;
import gate.creole.ControllerAwarePR;
import gate.creole.CustomDuplication;
import gate.creole.ExecutionException;
import gate.creole.ResourceInstantiationException;
import gate.creole.metadata.CreoleParameter;
import gate.creole.metadata.CreoleResource;
import gate.creole.metadata.Optional;
import gate.creole.metadata.RunTime;
import gate.plugin.evaluation.api.AnnotationDifferTagging;
import gate.plugin.evaluation.api.AnnotationDifferTagging.CandidateList;
import gate.plugin.evaluation.api.ByThEvalStatsTagging;
import gate.plugin.evaluation.api.EvalStatsTagging;
import gate.plugin.evaluation.api.EvalStatsTaggingMacro;
import gate.plugin.evaluation.api.FeatureComparison;
import gate.plugin.evaluation.api.ThresholdsToUse;
import gate.util.Files;
import gate.util.GateRuntimeException;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.PrintStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.log4j.Logger;

// TODO: add a list for getting the distribution of ranks/score of first lenient/exact match
// if we process lists. 
// This should get output as a separate tsv both per document and over all documents
// Maybe only if enabled with a parameter.
// If we process lists, also output per-document information about what the best is we can do
// in that document with the candidate lists we have.

// TODO: add a datastructure for incremental update of the contingency tables needed for 
// McNemar etc, if we have a reference set.
// These should get output at the end with values over all documents

// TODO: think about how this should deal with parallelization and custom duplication.
// At the moment this will not work properly or even fail when run from duplicated pipelines. 
// In the best case, each duplicate will just report the statistics on its own subset of documents,
// in the worst case, some resources will override each other (e.g. when writing from each 
// duplicate to the tsv files). 
// The proper way to deal with this requires that we have a standard way of how to share 
// data between those custom duplicated instances of the PR which belong to the same job (in a 
// larger VM, there could duplicates that belong to different jobs between which we do not want
// to share data).
// For now, we just prevent the multithreaded use of this PR by throwing an exception if an 
// attempt is made to custom-duplicate it. 
// Later on, we could change this such that all duplicated instances just call the methods of
// the original in a synchronized way.
// Finally, once we can calculate overall stats on all stats objects (including the byThreshold
// objects) we can try to support multi-threading in the following way:
// = all per-document io needs to go through a single instance synchronized method, or each 
//   instance creates their own output file
// = each duplicate creates their own stats objects over all documents it sees
// = there is a way to access all the stats objects of all the other duplicates somehow, or 
//   ideally there is a way to do this for just the "original" 
// = the controllerExecutionFinished method either knows if it is running for the original or 
//   can somehow make sure that the actual finishing code is only run for one of the duplicates.
//   That finishing code that would get the stats obejcts from all duplicates and merge/sum them,
//   then output the final statistics. 


// TODO(!!!): Add a java class for holding counts or instances of pairings between the reference set
// and the response set so we can calculate p-values for the SingleResponse accuracy analysis. 
// This could be used to get p-values for the McNemar test and the paired t-test.

// TODO: make the output directory URL more easily parametrizable by making the runtime parameter
// an OR parameter with a String version which can then be expandable like the other String
// parameters.

/**
 *
 * @author <NAME>
 */
@CreoleResource(name = "EvaluatePRF",
        comment = "Calculate P/R/F evalutation measures for documents")
{'fraction_non_alphanumeric': 0.07145720585287632, 'fraction_numerical': 0.0028061735818801364, 'mean_word_length': 4.213688610240334, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2011410', 'n_tokens_mistral': 2679, 'n_tokens_neox': 2591, 'n_words': 1216}