<commit_before>package eu.newsreader.eventcoreference.naf;

import eu.kyotoproject.kaf.KafSense;
import eu.newsreader.eventcoreference.coref.ComponentMatch;
import eu.newsreader.eventcoreference.objects.CompositeEvent;
import eu.newsreader.eventcoreference.objects.NafMention;
import eu.newsreader.eventcoreference.objects.SemObject;
import eu.newsreader.eventcoreference.objects.SourceMeta;
import eu.newsreader.eventcoreference.output.JenaSerialization;
import eu.newsreader.eventcoreference.util.ReadSourceMetaFile;
import eu.newsreader.eventcoreference.util.Util;
import org.apache.jena.atlas.logging.Log;
import vu.wntools.wordnet.WordnetData;
import vu.wntools.wordnet.WordnetLmfSaxParser;

import java.io.*;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

/**
 * Created with IntelliJ IDEA.
 * User: kyoto
 * Date: 11/14/13
 * Time: 7:54 PM
 * To change this template use File | Settings | File Templates.
 */
public class MatchEventObjects {
    static int nEventMatches = 0;
    static boolean CROSSDOC = false;
    static boolean FIXURI = false;
    static boolean GZIP = false;
    static boolean SUBFOLDER = false;
    static int DEBUG = 0;
    static ArrayList<String> tokenIds = new ArrayList<String>();
    public static String MATCHTYPE= "ililemma";  // ili OR lemma OR ililemma OR none OR split
    public static boolean HYPERS = false;
    public static boolean LCS = false;
    public static boolean ILIURI = false;
    public static boolean VERBOSEMENTIONS = false;
    public static String CHAINING = "3";
    static ArrayList<String> crossDocCorefSet = new ArrayList<String>(); /// just for debugging
    static public String TIME = "";

    static final String usage = "MatchEventObjects reads obj files stored in time-folders with CompositeEventObjects and outputs a single RDF-TRiG file\n" +
            "The parameters are:\n" +
            "--event-folder  <path>     <Path to the event folder that has subfolders for each time-description, e.g. \"e-2012-03-29\". Object file (*.obj) should be stored in these subfolders\n" +
            "--gz                       <OPTIONAL for reading gzipped object files with gz extension and writing trig.gz" +
            "--wordnet-lmf   <path>     <(OPTIONAL, not yet used) Path to a WordNet-LMF file\n>" +
            "--concept-match <int>      <threshold for conceptual matches of events, default is 50>\n" +
            "--phrase-match  <int>      <threshold for phrase matches of events, default is 50>\n" +
            "--hypers                   <(OPTIONAL) use hypernyms to match events\n"+
            "--lcs                      <(OPTIONAL) use lowest-common-subsumers to match events\n"+
            "--chaining                 <Determines the chaining function used: 1, 2, 3, 4. Default value is 3\n"+
            "--match-type    <string>   <(OPTIONAL) Indicates what is used to match events across resources. Default value is \"LEMMA\". Values:\"LEMMA\", \"ILI\", \"ILILEMMA\">\n" +
            "--ili                      <(OPTIONAL) Path to ILI.ttl file to convert wordnet-synsets identifiers to ILI identifiers>\n" +
            "--source-data   <path>     <(OPTIONAL, Deprecated) Path to LexisNexis meta data on owners and authors to enrich the provenance>\n" +
            "--roles  <string>          <String with PropbBank roles, separated by \",\" for which there minimally needs to be a match, e.g. \"a0,a1\". This is especially relevant for sourceEvent, grammaticalEvent. If value is \"all\", then all participants need to match. This can be used for futureEvent"+
            "--verbose                  <(OPTIONAL) representation of mentions is extended with token ids, terms ids and sentence number\n"+
            "--time     <string>        <(OPTIONAL) year, month or day indicate granularity of temporal match. If empty time is not matched\n"+
            "--ili-uri                  <(OPTIONAL) If used, the ILI-identifiers are used to represents events. This is necessary for cross-lingual extraction>\n" +
            "--subfolder                <(OPTIONAL) Processes any subfolder>\n" +
            "--debug                    <(OPTIONAL) default=0, 1=minimal, 2=max>\n";

    static String testArguments = " --event-folder /Users/piek/Desktop/Yassine/s1b2b/events/all --concept-match 80 --phrase-match 68 --ili /Code/vu/newsreader/vua-resources/ili.ttl.gz --hypers --lcs --chaining 3 --match-type ILILEMMA --verbose --debug 0 --time day --roles a0,a1,a2 --token-id /Users/piek/Desktop/Yassine/s1b2b.key.tokens";
    static public void main (String [] args) {
        Log.setLog4j("jena-log4j.properties");
        ArrayList<String> roleNeededArrayList = new ArrayList<String>();
        HashMap<String, SourceMeta> sourceMetaHashMap = null;
        WordnetData wordnetData = null;
        int conceptMatchThreshold = 50;
        int phraseMatchThreshold = 50;
        String pathToEventFolder = "";
        String pathToSourceDataFile = "";
         args = testArguments.split(" ");
        if (args.length==1 && args[0].equals("--help")) {
            System.out.println("usage = " + usage);
            return;
        }
        if (args.length==1 && args[0].equals("--usage")) {
            System.out.println("usage = " + usage);
            return;
        }
        for (int i = 0; i < args.length; i++) {
            String arg = args[i];
            if (arg.equals("--event-folder") && args.length>(i+1)) {<commit_msg>-- added a new parameter --no-paragraph-time . Default behaviour is that 2 sentences before and 1 sentence after are checked for time anchors if no time anhors are found. The parameter --no-paragraph-time turns this off which means the document creation time is used as an anchor.<commit_after>package eu.newsreader.eventcoreference.naf;

import eu.kyotoproject.kaf.KafSense;
import eu.newsreader.eventcoreference.coref.ComponentMatch;
import eu.newsreader.eventcoreference.objects.CompositeEvent;
import eu.newsreader.eventcoreference.objects.NafMention;
import eu.newsreader.eventcoreference.objects.SemObject;
import eu.newsreader.eventcoreference.objects.SourceMeta;
import eu.newsreader.eventcoreference.output.JenaSerialization;
import eu.newsreader.eventcoreference.util.ReadSourceMetaFile;
import eu.newsreader.eventcoreference.util.Util;
import org.apache.jena.atlas.logging.Log;
import vu.wntools.wordnet.WordnetData;
import vu.wntools.wordnet.WordnetLmfSaxParser;

import java.io.*;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

/**
 * Created with IntelliJ IDEA.
 * User: kyoto
 * Date: 11/14/13
 * Time: 7:54 PM
 * To change this template use File | Settings | File Templates.
 */
public class MatchEventObjects {
    static int nEventMatches = 0;
    static boolean CROSSDOC = false;
    static boolean FIXURI = false;
    static boolean GZIP = false;
    static boolean SUBFOLDER = false;
    static int DEBUG = 0;
    static ArrayList<String> tokenIds = new ArrayList<String>();
    public static String MATCHTYPE= "ililemma";  // ili OR lemma OR ililemma OR none OR split
    public static boolean HYPERS = false;
    public static boolean LCS = false;
    public static boolean ILIURI = false;
    public static boolean VERBOSEMENTIONS = false;
    public static String CHAINING = "3";
    static ArrayList<String> crossDocCorefSet = new ArrayList<String>(); /// just for debugging
    static public String TIME = "";

    static final String usage = "MatchEventObjects reads obj files stored in time-folders with CompositeEventObjects and outputs a single RDF-TRiG file\n" +
            "The parameters are:\n" +
            "--event-folder  <path>     <Path to the event folder that has subfolders for each time-description, e.g. \"e-2012-03-29\". Object file (*.obj) should be stored in these subfolders\n" +
            "--gz                       <OPTIONAL for reading gzipped object files with gz extension and writing trig.gz" +
            "--wordnet-lmf   <path>     <(OPTIONAL, not yet used) Path to a WordNet-LMF file\n>" +
            "--concept-match <int>      <threshold for conceptual matches of events, default is 50>\n" +
            "--phrase-match  <int>      <threshold for phrase matches of events, default is 50>\n" +
            "--hypers                   <(OPTIONAL) use hypernyms to match events\n"+
            "--lcs                      <(OPTIONAL) use lowest-common-subsumers to match events\n"+
            "--chaining                 <Determines the chaining function used: 1, 2, 3, 4. Default value is 3\n"+
            "--match-type    <string>   <(OPTIONAL) Indicates what is used to match events across resources. Default value is \"LEMMA\". Values:\"LEMMA\", \"ILI\", \"ILILEMMA\">\n" +
            "--ili                      <(OPTIONAL) Path to ILI.ttl file to convert wordnet-synsets identifiers to ILI identifiers>\n" +
            "--source-data   <path>     <(OPTIONAL, Deprecated) Path to LexisNexis meta data on owners and authors to enrich the provenance>\n" +
            "--roles  <string>          <String with PropbBank roles, separated by \",\" for which there minimally needs to be a match, e.g. \"a0,a1\". This is especially relevant for sourceEvent, grammaticalEvent. If value is \"all\", then all participants need to match. This can be used for futureEvent"+
            "--verbose                  <(OPTIONAL) representation of mentions is extended with token ids, terms ids and sentence number\n"+
            "--time     <string>        <(OPTIONAL) year, month or day indicate granularity of temporal match. If empty time is not matched\n"+
            "--ili-uri                  <(OPTIONAL) If used, the ILI-identifiers are used to represents events. This is necessary for cross-lingual extraction>\n" +
            "--subfolder                <(OPTIONAL) Processes any subfolder>\n" +
            "--debug                    <(OPTIONAL) default=0, 1=minimal, 2=max>\n";

    static String testArguments = " --event-folder /Users/piek/Desktop/Yassine/s1b2b/events/all --concept-match 80 --phrase-match 68 --ili /Code/vu/newsreader/vua-resources/ili.ttl.gz --hypers --lcs --chaining 3 --match-type ILILEMMA --verbose --debug 0 --time day --roles a0,a1,a2 --token-id /Users/piek/Desktop/Yassine/s1b2b.key.tokens";
    static public void main (String [] args) {
        Log.setLog4j("jena-log4j.properties");
        ArrayList<String> roleNeededArrayList = new ArrayList<String>();
        HashMap<String, SourceMeta> sourceMetaHashMap = null;
        WordnetData wordnetData = null;
        int conceptMatchThreshold = 50;
        int phraseMatchThreshold = 50;
        String pathToEventFolder = "";
        String pathToSourceDataFile = "";
        if (args.length==0) args = testArguments.split(" ");
        if (args.length==1 && args[0].equals("--help")) {
            System.out.println("usage = " + usage);
            return;
        }
        if (args.length==1 && args[0].equals("--usage")) {
            System.out.println("usage = " + usage);
            return;
        }
        for (int i = 0; i < args.length; i++) {
            String arg = args[i];
            if (arg.equals("--event-folder") && args.length>(i+1)) {
{'fraction_non_alphanumeric': 0.11150193457615196, 'fraction_numerical': 0.010640168835736898, 'mean_word_length': 2.6628019323671497, 'pattern_counts': {'":': 0, '<': 69, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5013706', 'n_tokens_mistral': 3344, 'n_tokens_neox': 3217, 'n_words': 1032}