<commit_before>
/*
 * Carrot2 project.
 *
 * Copyright (C) 2002-2006, <NAME>, <NAME>.
 * Portions (C) Contributors listed in "carrot2.CONTRIBUTORS" file.
 * All rights reserved.
 *
 * Refer to the full license file "carrot2.LICENSE"
 * in the root folder of the repository checkout or at:
 * http://www.carrot2.org/carrot2.LICENSE
 */

package com.dawidweiss.carrot.filter.stc;


/**
 * Names of properties used to drive the algorithm and
 * their default values.
 * 
 * @author <NAME>
 * @version $Revision$
 */
public class StcConstants {
    
    /**
     * No instantiation of this class.
     */
    private StcConstants() {
    }
    
    /**
     * Minimum score threshold for base clusters (clusters
     * with lower score are discarded before merging).
     * 
     * <p>Allowed range: 0.0 -- inf., property type: <code>String</code>
     *    (a numeric value)</p>
     */
    public final static String MIN_BASE_CLUSTER_SCORE 
        = "stc.threshold.minBaseClusterScore";

    /** Default value of {@link #MIN_BASE_CLUSTER_SCORE}. */
    public final static double DEFAULT_MIN_BASE_CLUSTER_SCORE = 2.0f;

    /**
     * Maximum highest scoring base clusters promoted for merging.
     * 
     * <p>Allowed range: integer, property type <code>String</code>
     *    (a numeric value)</p>
     */
    public final static String MAX_BASE_CLUSTERS_COUNT
        = "stc.threshold.maxBaseClusters";

    /** Default value of {@link #MAX_BASE_CLUSTERS_COUNT}. */
    public final static int DEFAULT_MAX_BASE_CLUSTERS_COUNT = 300;

    /**
     * Minimum number of documents in a base cluster to be promoted
     * to merging phase. 
     * 
     * <p>Allowed range: 2 -- inf., property type <code>String</code>
     *    (a numeric value)</p>
     */
    public final static String MIN_BASE_CLUSTER_SIZE
        = "stc.threshold.minBaseClusterSize";

    /** Default value of {@link #MIN_BASE_CLUSTER_SIZE}. */
    public final static int DEFAULT_MIN_BASE_CLUSTER_SIZE = 2;

    /**
     * A word is marked as ignored (irrelevant) if it is in fewer
     * documents than this threshold.
     * 
     * <p>Allowed range: 2 -- inf., property type <code>String</code>
     *    (a numeric value)</p>
     */
    public final static String IGNORED_WORD_IF_IN_FEWER_DOCS
        = "stc.threshold.ignoreWordIfInFewerDocs";

    /** Default value of {@link #IGNORED_WORD_IF_IN_FEWER_DOCS}. */
    public final static int DEFAULT_IGNORED_WORD_IF_IN_FEWER_DOCS = 2;
    
    /**
     * A word is marked as ignored (irrelevant) if it is in a higher
     * ratio of documents than this.
     * 
     * <p>Allowed range: 0 -- 1, property type <code>String</code>
     *    (a numeric value)</p>
     */
    public final static String IGNORED_WORD_IF_IN_MORE_DOCS
        = "stc.threshold.ignoreWordIfInHigherDocsPercent";

    /** Default value of {@link #IGNORED_WORD_IF_IN_MORE_DOCS}. */
    public final static double DEFAULT_IGNORED_WORD_IF_IN_MORE_DOCS = 0.9f;

    /**
     * Two base clusters are merged if their documents overlap
     * at least in this ratio.
     * 
     * <p>Allowed range: 0 -- 1, property type <code>String</code>
     *    (a numeric value)</p>
     */
    public final static String MERGE_THRESHOLD
        = "stc.threshold.mergeThreshold";

    /** Default value of {@link #MERGE_THRESHOLD}. */
    public final static double DEFAULT_MERGE_THRESHOLD = 0.6f;
    
    /**
     * Maximum number of merged clusters returned.
     * 
     * <p>Allowed range: integer, 1 -- inf., property type <code>String</code>
     *    (a numeric value)</p>
     */
    public final static String MAX_CLUSTERS
        = "stc.threshold.maxClusters";

    /** Default value of {@link #MAX_CLUSTERS}. */
    public final static int DEFAULT_MAX_CLUSTERS = 15;

    /**
     * Maximum overlap between phrases selected for cluster labels. If the overlap
     * exceeds this level, one phrase is removed from the label.
     * 
     * <p>Allowed range: 0 -- 1, property type <code>String</code> (a numeric value)</p>
     */
    public final static String MAX_PHRASE_OVERLAP = "stc.max.phrase.overlap";

    /**
     * Default value of {@link #MAX_PHRASE_OVERLAP}.
     */
    public final static double DEFAULT_MAX_PHRASE_OVERLAP = 0.6d;

    /**
     * Minimum general phrase coverage to appear in cluster description.
     * 
     * <p>Allowed range: 0 -- 1, property type <code>String</code> (a numeric value)</p>
     */
    public static final String MOST_GENERAL_PHRASE_COVERAGE = "stc.most.general.phrase.coverage";
    
    /**
     * Default value of {@link #MOST_GENERAL_PHRASE_COVERAGE}.
     */
    public static final double DEFAULT_MOST_GENERAL_PHRASE_COVERAGE = 0.2d; 
}
<commit_msg>Defaults changed a bit (after manually tuning)<commit_after>
/*
 * Carrot2 project.
 *
 * Copyright (C) 2002-2006, <NAME>, <NAME>.
 * Portions (C) Contributors listed in "carrot2.CONTRIBUTORS" file.
 * All rights reserved.
 *
 * Refer to the full license file "carrot2.LICENSE"
 * in the root folder of the repository checkout or at:
 * http://www.carrot2.org/carrot2.LICENSE
 */

package com.dawidweiss.carrot.filter.stc;


/**
 * Names of properties used to drive the algorithm and
 * their default values.
 * 
 * @author <NAME>
 * @version $Revision$
 */
public class StcConstants {
    
    /**
     * No instantiation of this class.
     */
    private StcConstants() {
    }
    
    /**
     * Minimum score threshold for base clusters (clusters
     * with lower score are discarded before merging).
     * 
     * <p>Allowed range: 0.0 -- inf., property type: <code>String</code>
     *    (a numeric value)</p>
     */
    public final static String MIN_BASE_CLUSTER_SCORE 
        = "stc.threshold.minBaseClusterScore";

    /** Default value of {@link #MIN_BASE_CLUSTER_SCORE}. */
    public final static double DEFAULT_MIN_BASE_CLUSTER_SCORE = 2.0f;

    /**
     * Maximum highest scoring base clusters promoted for merging.
     * 
     * <p>Allowed range: integer, property type <code>String</code>
     *    (a numeric value)</p>
     */
    public final static String MAX_BASE_CLUSTERS_COUNT
        = "stc.threshold.maxBaseClusters";

    /** Default value of {@link #MAX_BASE_CLUSTERS_COUNT}. */
    public final static int DEFAULT_MAX_BASE_CLUSTERS_COUNT = 300;

    /**
     * Minimum number of documents in a base cluster to be promoted
     * to merging phase. 
     * 
     * <p>Allowed range: 2 -- inf., property type <code>String</code>
     *    (a numeric value)</p>
     */
    public final static String MIN_BASE_CLUSTER_SIZE
        = "stc.threshold.minBaseClusterSize";

    /** Default value of {@link #MIN_BASE_CLUSTER_SIZE}. */
    public final static int DEFAULT_MIN_BASE_CLUSTER_SIZE = 2;

    /**
     * A word is marked as ignored (irrelevant) if it is in fewer
     * documents than this threshold.
     * 
     * <p>Allowed range: 2 -- inf., property type <code>String</code>
     *    (a numeric value)</p>
     */
    public final static String IGNORED_WORD_IF_IN_FEWER_DOCS
        = "stc.threshold.ignoreWordIfInFewerDocs";

    /** Default value of {@link #IGNORED_WORD_IF_IN_FEWER_DOCS}. */
    public final static int DEFAULT_IGNORED_WORD_IF_IN_FEWER_DOCS = 2;
    
    /**
     * A word is marked as ignored (irrelevant) if it is in a higher
     * ratio of documents than this.
     * 
     * <p>Allowed range: 0 -- 1, property type <code>String</code>
     *    (a numeric value)</p>
     */
    public final static String IGNORED_WORD_IF_IN_MORE_DOCS
        = "stc.threshold.ignoreWordIfInHigherDocsPercent";

    /** Default value of {@link #IGNORED_WORD_IF_IN_MORE_DOCS}. */
    public final static double DEFAULT_IGNORED_WORD_IF_IN_MORE_DOCS = 0.9f;

    /**
     * Two base clusters are merged if their documents overlap
     * at least in this ratio.
     * 
     * <p>Allowed range: 0 -- 1, property type <code>String</code>
     *    (a numeric value)</p>
     */
    public final static String MERGE_THRESHOLD
        = "stc.threshold.mergeThreshold";

    /** Default value of {@link #MERGE_THRESHOLD}. */
    public final static double DEFAULT_MERGE_THRESHOLD = 0.6f;
    
    /**
     * Maximum number of merged clusters returned.
     * 
     * <p>Allowed range: integer, 1 -- inf., property type <code>String</code>
     *    (a numeric value)</p>
     */
    public final static String MAX_CLUSTERS
        = "stc.threshold.maxClusters";

    /** Default value of {@link #MAX_CLUSTERS}. */
    public final static int DEFAULT_MAX_CLUSTERS = 15;

    /**
     * Maximum overlap between phrases selected for cluster labels. If the overlap
     * exceeds this level, one phrase is removed from the label.
     * 
     * <p>Allowed range: 0 -- 1, property type <code>String</code> (a numeric value)</p>
     */
    public final static String MAX_PHRASE_OVERLAP = "stc.max.phrase.overlap";

    /**
     * Default value of {@link #MAX_PHRASE_OVERLAP}.
     */
    public final static double DEFAULT_MAX_PHRASE_OVERLAP = 0.3d;

    /**
     * Minimum general phrase coverage to appear in cluster description.
     * 
     * <p>Allowed range: 0 -- 1, property type <code>String</code> (a numeric value)</p>
     */
    public static final String MOST_GENERAL_PHRASE_COVERAGE = "stc.most.general.phrase.coverage";
    
    /**
     * Default value of {@link #MOST_GENERAL_PHRASE_COVERAGE}.
     */
    public static final double DEFAULT_MOST_GENERAL_PHRASE_COVERAGE = 0.5d; 
}

{'fraction_non_alphanumeric': 0.10676835081029552, 'fraction_numerical': 0.00910920453341807, 'mean_word_length': 2.678223607323724, 'pattern_counts': {'":': 0, '<': 81, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7665078', 'n_tokens_mistral': 3111, 'n_tokens_neox': 2945, 'n_words': 860}