<commit_before>package edu.jhu.thrax;

import edu.jhu.thrax.util.ConfFileParser;

import org.apache.hadoop.conf.Configuration;

import java.util.Map;
import java.util.HashMap;
import java.io.IOException;

/**
 */
public class ThraxConfig {

    /**
     * Field delimiter.
     */
    public static final String DELIMITER = "|||";
    /**
     * Regular expression for the field delimiter.
     */
    public static final String DELIMITER_REGEX = "\\|\\|\\|";

    /**
     * Grammar type for extraction.
     */
    public static String GRAMMAR = "hiero";
    /**
     * Output format. This doesn't do anything. The only format is Joshua.
     */
    public static String OUTPUT_FORMAT = "joshua";

    /**
     * Whether or not to allow adjacent NTs in extracted rules.
     */
    public static boolean ADJACENT = false;
    /**
     * Whether or not to allow unaligned words on the boundary of initial
     * phrases during rule extraction.
     */
    public static boolean LOOSE = false;
    /**
     * Maximum length of initial phrase pairs.
     */
    public static int INITIAL_PHRASE_LIMIT = 10;
    /**
     * Maximum number of symbols on the source side of non-lexical rules.
     */
    public static int NONLEX_SOURCE_LENGTH_LIMIT = 5;
    /**
     * Maximum number of terminals on the source side of non-lexical rules.
     */
    public static int NONLEX_SOURCE_WORD_LIMIT = 5;
    /**
     * Maximum number of symbols on the target side of non-lexical rules.
     */
    public static int NONLEX_TARGET_LENGTH_LIMIT = 5;
    /**
     * Maximum number of terminals on the target side of non-lexical rules.
     */
    public static int NONLEX_TARGET_WORD_LIMIT = 5;
    /**
     * Maximum number of source side terminals in purely lexical rules.
     */
    public static int LEX_SOURCE_LENGTH_LIMIT = 12;
    /**
     * Maximum number of target side terminals in purely lexical rules.
     */
    public static int LEX_TARGET_LENGTH_LIMIT = 12;
    /**
     * Maximum number of nonterminal symbols in extracted rules.
     */
    public static int ARITY = 2;
    /**
     * Minimum number of aligned terminal symbols required in extracted rules.
     */
    public static int LEXICALITY = 1;

    /**
     * Contains whitespace-separated names of features for rule extraction.
     */
    public static String FEATURES = "";

    /**
     * Maximum number of constituent labels to extract for each span in SAMT.
     * Constituent labels are labels that consist of one syntactic symbol.
     */
    public static int MAX_CONSTITUENT_LABELS = -1;
    /**
     * Maximum number of CCG-style labels to extract for each span in SAMT.
     * CCG-style labels do not correspond to a syntactic constituent, but
     * rather encode symbols that are missing particular parts on either left
     * or right.
     */
    public static int MAX_CCG_LABELS = -1;
    /**
     * Maximum number of concatenated labels to extract for each span in SAMT.
     */
    public static int MAX_CAT_LABELS = -1;

    /**
     * Default nonterminal symbol. This is the only NT in Hiero grammars, and
     * it is used when no other symbol is possible in an SAMT grammar.
     */
    public static String DEFAULT_NT = "X";

    /**
     * Label feature scores with feature names?
     */
    public static boolean LABEL_FEATURE_SCORES = false;

    /**
     * How to handle unary categories in SAMT labelling.
     */
    public static String UNARY_CATEGORY_HANDLER = "all";

    /**
     * Allow double concatenation in SAMT labelling.
     */
    public static boolean ALLOW_DOUBLE_CONCAT = false;

    /**
     * Phrase penalty for each rule.
     */
    public static double PHRASE_PENALTY = 0.0;
    /**
     * Whether to allow X on the left-hand side of a lexical rule.
     */
    public static boolean ALLOW_X_NONLEX = false;
    /**
     * Allow abstract rules to be extracted.
     */
    public static boolean ALLOW_ABSTRACT = false;<commit_msg>default phrase penalty set to 2.718<commit_after>package edu.jhu.thrax;

import edu.jhu.thrax.util.ConfFileParser;

import org.apache.hadoop.conf.Configuration;

import java.util.Map;
import java.util.HashMap;
import java.io.IOException;

/**
 */
public class ThraxConfig {

    /**
     * Field delimiter.
     */
    public static final String DELIMITER = "|||";
    /**
     * Regular expression for the field delimiter.
     */
    public static final String DELIMITER_REGEX = "\\|\\|\\|";

    /**
     * Grammar type for extraction.
     */
    public static String GRAMMAR = "hiero";
    /**
     * Output format. This doesn't do anything. The only format is Joshua.
     */
    public static String OUTPUT_FORMAT = "joshua";

    /**
     * Whether or not to allow adjacent NTs in extracted rules.
     */
    public static boolean ADJACENT = false;
    /**
     * Whether or not to allow unaligned words on the boundary of initial
     * phrases during rule extraction.
     */
    public static boolean LOOSE = false;
    /**
     * Maximum length of initial phrase pairs.
     */
    public static int INITIAL_PHRASE_LIMIT = 10;
    /**
     * Maximum number of symbols on the source side of non-lexical rules.
     */
    public static int NONLEX_SOURCE_LENGTH_LIMIT = 5;
    /**
     * Maximum number of terminals on the source side of non-lexical rules.
     */
    public static int NONLEX_SOURCE_WORD_LIMIT = 5;
    /**
     * Maximum number of symbols on the target side of non-lexical rules.
     */
    public static int NONLEX_TARGET_LENGTH_LIMIT = 5;
    /**
     * Maximum number of terminals on the target side of non-lexical rules.
     */
    public static int NONLEX_TARGET_WORD_LIMIT = 5;
    /**
     * Maximum number of source side terminals in purely lexical rules.
     */
    public static int LEX_SOURCE_LENGTH_LIMIT = 12;
    /**
     * Maximum number of target side terminals in purely lexical rules.
     */
    public static int LEX_TARGET_LENGTH_LIMIT = 12;
    /**
     * Maximum number of nonterminal symbols in extracted rules.
     */
    public static int ARITY = 2;
    /**
     * Minimum number of aligned terminal symbols required in extracted rules.
     */
    public static int LEXICALITY = 1;

    /**
     * Contains whitespace-separated names of features for rule extraction.
     */
    public static String FEATURES = "";

    /**
     * Maximum number of constituent labels to extract for each span in SAMT.
     * Constituent labels are labels that consist of one syntactic symbol.
     */
    public static int MAX_CONSTITUENT_LABELS = -1;
    /**
     * Maximum number of CCG-style labels to extract for each span in SAMT.
     * CCG-style labels do not correspond to a syntactic constituent, but
     * rather encode symbols that are missing particular parts on either left
     * or right.
     */
    public static int MAX_CCG_LABELS = -1;
    /**
     * Maximum number of concatenated labels to extract for each span in SAMT.
     */
    public static int MAX_CAT_LABELS = -1;

    /**
     * Default nonterminal symbol. This is the only NT in Hiero grammars, and
     * it is used when no other symbol is possible in an SAMT grammar.
     */
    public static String DEFAULT_NT = "X";

    /**
     * Label feature scores with feature names?
     */
    public static boolean LABEL_FEATURE_SCORES = false;

    /**
     * How to handle unary categories in SAMT labelling.
     */
    public static String UNARY_CATEGORY_HANDLER = "all";

    /**
     * Allow double concatenation in SAMT labelling.
     */
    public static boolean ALLOW_DOUBLE_CONCAT = false;

    /**
     * Phrase penalty for each rule.
     */
    public static double PHRASE_PENALTY = 2.718;
    /**
     * Whether to allow X on the left-hand side of a lexical rule.
     */
    public static boolean ALLOW_X_NONLEX = false;
    /**
     * Allow abstract rules to be extracted.
     */
    public static boolean ALLOW_ABSTRACT = false;
{'fraction_non_alphanumeric': 0.0801970194493559, 'fraction_numerical': 0.005051780752715332, 'mean_word_length': 2.3683538919608678, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '801566', 'n_tokens_mistral': 2363, 'n_tokens_neox': 2218, 'n_words': 856}