<commit_before>import java.util.*;
import java.io.*;

public class Util
{
    public static final char RULE_NAME_DELIMITER = ':';
    public static final char OR_DELIMITER = '|';

    public static final Vector<Rule> loadRules (String inputFile, boolean debug)
    {
        /*
         * Open the data file and read it in.
         */

        BufferedReader reader = null;
        Vector<Rule> values = new Vector<Rule>();

        try
        {
            String line = null;

            reader = new BufferedReader(new FileReader(inputFile));
         
            while (((line = reader.readLine()) != null) && (line.length() > 0))
            {
                int nameDelim = line.indexOf(RULE_NAME_DELIMITER);
                String name = line.substring(0, nameDelim);

                int orDelim = line.indexOf(OR_DELIMITER);  // there's only ever a maximum of one of these in a rule
            }
        }
        catch (Throwable ex)
        {
            ex.printStackTrace();
        }
        finally
        {
            try
            {
                reader.close();
            }<commit_msg>Update Util.java<commit_after>import java.util.*;
import java.io.*;

public class Util
{
    public static final char RULE_NAME_DELIMITER = ':';
    public static final char OR_DELIMITER = '|';
    public static final char RULE_DELIMITER = ' ';

    public static final Vector<Rule> loadRules (String inputFile, boolean debug)
    {
        /*
         * Open the data file and read it in.
         */

        BufferedReader reader = null;
        Vector<Rule> values = new Vector<Rule>();

        try
        {
            String line = null;

            reader = new BufferedReader(new FileReader(inputFile));
         
            while (((line = reader.readLine()) != null) && (line.length() > 0))
            {
                int nameDelim = line.indexOf(RULE_NAME_DELIMITER);
                String name = line.substring(0, nameDelim);

                int orDelim = line.indexOf(OR_DELIMITER);  // there's only ever a maximum of one of these in a rule
                String dataOne = line.substring(RULE_NAME_DELIMITER +1, ((orDelim == -1) ? line.length() : orDelim));
                String dataTwo = ((orDelim == -1) ? null : line.substring(OR_DELIMITER +1));

            }
        }
        catch (Throwable ex)
        {
            ex.printStackTrace();
        }
        finally
        {
            try
            {
                reader.close();
            }
{'fraction_non_alphanumeric': 0.1036, 'fraction_numerical': 0.0032, 'mean_word_length': 1.6161087866108788, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3930791', 'n_tokens_mistral': 697, 'n_tokens_neox': 670, 'n_words': 189}