<commit_before>package eu.fbk.dkm.premon.premonitor;

import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;

import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;

import org.openrdf.model.Resource;
import org.openrdf.model.Statement;
import org.openrdf.model.URI;
import org.openrdf.model.Value;
import org.openrdf.model.ValueFactory;
import org.openrdf.model.impl.ValueFactoryImpl;
import org.openrdf.model.vocabulary.DCTERMS;
import org.openrdf.model.vocabulary.RDF;
import org.openrdf.rio.RDFHandler;
import org.openrdf.rio.RDFHandlerException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import eu.fbk.dkm.utils.CommandLine;
import eu.fbk.rdfpro.AbstractRDFHandler;
import eu.fbk.rdfpro.RDFHandlers;
import eu.fbk.rdfpro.RDFProcessor;
import eu.fbk.rdfpro.RDFProcessors;
import eu.fbk.rdfpro.RDFSource;
import eu.fbk.rdfpro.RDFSources;
import eu.fbk.rdfpro.util.QuadModel;
import eu.fbk.rdfpro.util.Statements;

/**
 * Created by alessio on 29/10/15.
 */

public class Premonitor {

    private static final String DEFAULT_PATH = ".";
    private static final String DEFAULT_LANGUAGE = "en";

    private static final String DEFAULT_PB_FOLDER = "pb";
    private static final String DEFAULT_PB_SOURCE = "pb-17";
    private static final String DEFAULT_PB_SOURCE_ON = "pb-ontonotes-5";

    private static final String DEFAULT_NB_FOLDER = "nb";
    private static final String DEFAULT_NB_SOURCE = "nb-10";

    private static final URI META_GRAPH = Statements.VALUE_FACTORY
            .createURI("http://premon.fbk.eu/resource/graph-meta");
    
	private static final Logger LOGGER = LoggerFactory.getLogger(Premonitor.class);

	private static final ValueFactory VALUE_FACTORY;
	public static final Map<String, URI> LANGUAGE_CODES_TO_URIS;
	static {
		VALUE_FACTORY = ValueFactoryImpl.getInstance();
		final Map<String, URI> codesToURIs = Maps.newHashMap();
		for (final String language : Locale.getISOLanguages()) {
			final Locale locale = new Locale(language);
			final URI uri = VALUE_FACTORY.createURI("http://lexvo.org/id/iso639-3/",
					locale.getISO3Language());
			codesToURIs.put(language, uri);
		}
		LANGUAGE_CODES_TO_URIS = ImmutableMap.copyOf(codesToURIs);
	}

	public static void main(String[] args) {

		List<Statement> statements = new ArrayList<>();
		RDFHandler handler = RDFHandlers.wrap(statements);

		try {
			final CommandLine cmd = CommandLine
					.parser()
					.withName("./premonitor")
					.withHeader("Transform linguistic resources into RDF")
					.withOption("i", "input", String.format("input folder (default %s)", DEFAULT_PATH), "FOLDER", CommandLine.Type.DIRECTORY_EXISTING, true, false, false)
					.withOption("w", "output", "Output file (mandatory)", "FILE", CommandLine.Type.FILE, true, false, true)
					.withOption("l", "language", String.format("Language for literals, default %s", DEFAULT_LANGUAGE), "ISO-CODE", CommandLine.Type.STRING, true, false, false)
					.withOption("s", "single", "Extract single lemma", "LEMMA", CommandLine.Type.STRING, true, false, false)

					.withOption(null, "pb-non-verbs", "Extract also non-verbs (only for OntoNotes)")
					.withOption(null, "pb-ontonotes", "Specify that this is an OntoNotes version of ProbBank")
					.withOption(null, "pb-examples", "Extract examples for PropBank")
					.withOption(null, "pb-folder", "PropBank frames folder", "FOLDER", CommandLine.Type.DIRECTORY_EXISTING, true, false, false)
					.withOption(null, "pb-no-def", "Skip PropBank definitions")
					.withOption(null, "pb-source", String.format("PropBank source, default %s/%s", DEFAULT_PB_SOURCE, DEFAULT_PB_SOURCE_ON), "SOURCE", CommandLine.Type.STRING, true, false, false)

					.withOption(null, "nb-examples", "Extract examples for PropBank")
					.withOption(null, "nb-folder", "PropBank frames folder", "FOLDER", CommandLine.Type.DIRECTORY_EXISTING, true, false, false)
					.withOption(null, "nb-no-def", "Skip PropBank definitions")
					.withOption(null, "nb-source", String.format("PropBank source, default %s/%s", DEFAULT_PB_SOURCE, DEFAULT_PB_SOURCE_ON), "SOURCE", CommandLine.Type.STRING, true, false, false)
<commit_msg>NB fix<commit_after>package eu.fbk.dkm.premon.premonitor;

import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;

import com.google.common.collect.HashMultimap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;

import org.openrdf.model.Resource;
import org.openrdf.model.Statement;
import org.openrdf.model.URI;
import org.openrdf.model.Value;
import org.openrdf.model.ValueFactory;
import org.openrdf.model.impl.ValueFactoryImpl;
import org.openrdf.model.vocabulary.DCTERMS;
import org.openrdf.model.vocabulary.RDF;
import org.openrdf.rio.RDFHandler;
import org.openrdf.rio.RDFHandlerException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import eu.fbk.dkm.utils.CommandLine;
import eu.fbk.rdfpro.AbstractRDFHandler;
import eu.fbk.rdfpro.RDFHandlers;
import eu.fbk.rdfpro.RDFProcessor;
import eu.fbk.rdfpro.RDFProcessors;
import eu.fbk.rdfpro.RDFSource;
import eu.fbk.rdfpro.RDFSources;
import eu.fbk.rdfpro.util.QuadModel;
import eu.fbk.rdfpro.util.Statements;

/**
 * Created by alessio on 29/10/15.
 */

public class Premonitor {

    private static final String DEFAULT_PATH = ".";
    private static final String DEFAULT_LANGUAGE = "en";

    private static final String DEFAULT_PB_FOLDER = "pb";
    private static final String DEFAULT_PB_SOURCE = "pb-17";
    private static final String DEFAULT_PB_SOURCE_ON = "pb-ontonotes-5";

    private static final String DEFAULT_NB_FOLDER = "nb";
    private static final String DEFAULT_NB_SOURCE = "nb-10";

    private static final URI META_GRAPH = Statements.VALUE_FACTORY
            .createURI("http://premon.fbk.eu/resource/graph-meta");
    
	private static final Logger LOGGER = LoggerFactory.getLogger(Premonitor.class);

	private static final ValueFactory VALUE_FACTORY;
	public static final Map<String, URI> LANGUAGE_CODES_TO_URIS;
	static {
		VALUE_FACTORY = ValueFactoryImpl.getInstance();
		final Map<String, URI> codesToURIs = Maps.newHashMap();
		for (final String language : Locale.getISOLanguages()) {
			final Locale locale = new Locale(language);
			final URI uri = VALUE_FACTORY.createURI("http://lexvo.org/id/iso639-3/",
					locale.getISO3Language());
			codesToURIs.put(language, uri);
		}
		LANGUAGE_CODES_TO_URIS = ImmutableMap.copyOf(codesToURIs);
	}

	public static void main(String[] args) {

		List<Statement> statements = new ArrayList<>();
		RDFHandler handler = RDFHandlers.wrap(statements);

		try {
			final CommandLine cmd = CommandLine
					.parser()
					.withName("./premonitor")
					.withHeader("Transform linguistic resources into RDF")
					.withOption("i", "input", String.format("input folder (default %s)", DEFAULT_PATH), "FOLDER", CommandLine.Type.DIRECTORY_EXISTING, true, false, false)
					.withOption("w", "output", "Output file (mandatory)", "FILE", CommandLine.Type.FILE, true, false, true)
					.withOption("l", "language", String.format("Language for literals, default %s", DEFAULT_LANGUAGE), "ISO-CODE", CommandLine.Type.STRING, true, false, false)
					.withOption("s", "single", "Extract single lemma", "LEMMA", CommandLine.Type.STRING, true, false, false)

					.withOption(null, "pb-non-verbs", "Extract also non-verbs (only for OntoNotes)")
					.withOption(null, "pb-ontonotes", "Specify that this is an OntoNotes version of ProbBank")
					.withOption(null, "pb-examples", "Extract examples for PropBank")
					.withOption(null, "pb-folder", "PropBank frames folder", "FOLDER", CommandLine.Type.DIRECTORY_EXISTING, true, false, false)
					.withOption(null, "pb-no-def", "Skip PropBank definitions")
					.withOption(null, "pb-source", String.format("PropBank source, default %s/%s", DEFAULT_PB_SOURCE, DEFAULT_PB_SOURCE_ON), "SOURCE", CommandLine.Type.STRING, true, false, false)

					.withOption(null, "nb-examples", "Extract examples for NomBank")
					.withOption(null, "nb-folder", "NomBank frames folder", "FOLDER", CommandLine.Type.DIRECTORY_EXISTING, true, false, false)
					.withOption(null, "nb-no-def", "Skip NomBank definitions")
					.withOption(null, "nb-source", String.format("NomBank source, default %s", DEFAULT_NB_SOURCE), "SOURCE", CommandLine.Type.STRING, true, false, false)

{'fraction_non_alphanumeric': 0.12703989120580236, 'fraction_numerical': 0.004079782411604714, 'mean_word_length': 4.922818791946309, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5524799', 'n_tokens_mistral': 3156, 'n_tokens_neox': 2917, 'n_words': 652}