<commit_before>package lex;


import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;

class LexSourceParser {
    private static final int DEFINITIONS = 1, RULES = 2, USER_ROUTINES = 3;
    private static final String escapeChars = "\\.()[]+-*|@?!";  // ^$

    private int lineNumber = 0;
    private int section;

    // definitions in section DEFINITIONS
    private Map<String, String> definitions = new HashMap<>();

    LexSourceParser() {
    }

    void parse(String sourcePath) {
        try (BufferedReader br = new BufferedReader(new FileReader(sourcePath))) {
            section = DEFINITIONS;
            StringBuilder target = new StringBuilder();
            Map<String, String> rules = new LinkedHashMap<>();

            boolean inComment = false, inRule = false;

            // used in SECTION RULES
            StringBuilder ruleAction = new StringBuilder();
            String ruleDef = "";

            // one line of source
            String line;
            while ((line = br.readLine()) != null) {
                ++lineNumber;
                // section user routines
                if (section == USER_ROUTINES) {
                    break;
                }

                // empty line
                if (line.isEmpty()) {
                    continue;
                }

                if (inComment) {
                    // copy comment to target file
                    if (line.startsWith("%}")) {
                        inComment = false;
                    }
                    // multi-line comment ends
                    else {
                        target.append(line).append('\n');
                    }
                    continue;
                }
                // multi-line comment begins
                else if (line.startsWith("%{")) {
                    inComment = true;
                    continue;
                }
                // section delimiter
                else if (line.startsWith("%%")) {
                    ++section;
                    continue;
                }

                // in rule definition
                if (inRule) {
                    if (isBlank(line.charAt(0))) {
                        ruleAction.append(line).append('\n');
                        continue;
                    } else {
                        if (ruleAction.toString().isEmpty()) {
                            handleError("null rule action", lineNumber - 1);
                        }
                        rules.put(ruleDef, ruleAction.toString());
                        inRule = false;
                    }
                }
                // single line comment
                else if (line.startsWith(" ") || line.startsWith("\t")) {
                    target.append(line.trim()).append('\n');
                    continue;
                }

                // section definitions
                String[] pair = splitByTab(line);
                if (section == DEFINITIONS) {
                    if (pair == null) {
                        handleError("invalid definition", lineNumber);
                    } else {
                        String right = toRe(pair[1]);
                        definitions.put(pair[0], right);
                    }
                }

                if (section == RULES) {
                    if (pair == null) {
                        handleError("invalid rules", lineNumber);
                    } else {
                        inRule = true;
                        ruleDef = toRe(pair[0]);
                        ruleAction = new StringBuilder(pair[1]);
                    }
                }

            }
            /* section user routines, copy all lines left to target file. */
            while (line != null) {
                target.append(line).append('\n');
                line = br.readLine();
                ++lineNumber;
            }
//            ReParser reParser = new ReParser();
//            for (String s : rules.keySet()) {
//                System.out.println();
//                System.out.println("\t" + s);
//                reParser.processRe(s);
//            }

//            int i = 0;
//            for (String s : rules.keySet()) {
//                assert i < rules.size();
//                if (i == 0) {
//                    System.out.println();
//                    System.out.println("\t" + s);
//                    reParser.processRe(s);
//                    break;
//                }
//                i--;
//            }
            for (String s : definitions.keySet()) {
                System.out.println(s + "->" + definitions.get(s));
            }

            for (String s : rules.keySet()) {
                System.out.println(s + "->" + rules.get(s));
            }

            System.out.println(target.toString());
        } catch (FileNotFoundException e) {
            handleError("can't open file \"" + sourcePath + "\" to read", 0);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
<commit_msg>add get method<commit_after>package lex;


import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;

class LexSourceParser {
    private static final int DEFINITIONS = 1, RULES = 2, USER_ROUTINES = 3;
    private static final String escapeChars = "\\.()[]+-*|@?!";  // ^$

    private int lineNumber = 0;
    private int section;

    // definitions in section DEFINITIONS
    private Map<String, String> definitions;
    private Map<String, String> rules;
    private StringBuilder userRoutines;

    LexSourceParser() {
    }

    Map<String, String> getRules() {
        return rules;
    }

    String getUserRoutines() {
        return userRoutines == null ? null : userRoutines.toString();
    }

    void parse(String sourcePath) {
        try (BufferedReader br = new BufferedReader(new FileReader(sourcePath))) {
            definitions = new HashMap<>();
            rules = new LinkedHashMap<>();
            userRoutines = new StringBuilder();

            section = DEFINITIONS;
            boolean inComment = false, inRule = false;

            // used in SECTION RULES
            StringBuilder ruleAction = new StringBuilder();
            String ruleDef = "";

            // one line of source
            String line;
            while ((line = br.readLine()) != null) {
                ++lineNumber;
                // section user routines
                if (section == USER_ROUTINES) {
                    break;
                }

                // empty line
                if (line.isEmpty()) {
                    continue;
                }

                if (inComment) {
                    // copy comment to userRoutines file
                    if (line.startsWith("%}")) {
                        inComment = false;
                    }
                    // multi-line comment ends
                    else {
                        userRoutines.append(line).append('\n');
                    }
                    continue;
                }
                // multi-line comment begins
                else if (line.startsWith("%{")) {
                    inComment = true;
                    continue;
                }
                // section delimiter
                else if (line.startsWith("%%")) {
                    ++section;
                    continue;
                }

                // in rule definition
                if (inRule) {
                    if (isBlank(line.charAt(0))) {
                        ruleAction.append(line).append('\n');
                        continue;
                    } else {
                        if (ruleAction.toString().isEmpty()) {
                            handleError("null rule action", lineNumber - 1);
                        }
                        rules.put(ruleDef, ruleAction.toString());
                        inRule = false;
                    }
                }
                // single line comment
                else if (line.startsWith(" ") || line.startsWith("\t")) {
                    userRoutines.append(line.trim()).append('\n');
                    continue;
                }

                // section definitions
                String[] pair = splitByTab(line);
                if (section == DEFINITIONS) {
                    if (pair == null) {
                        handleError("invalid definition", lineNumber);
                    } else {
                        String right = toRe(pair[1]);
                        definitions.put(pair[0], right);
                    }
                }


                if (section == RULES) {
                    if (pair == null) {
                        handleError("invalid rules", lineNumber);
                    } else {
                        inRule = true;
                        ruleDef = toRe(pair[0]);
                        ruleAction = new StringBuilder(pair[1]);
                    }
                }

            }
            /* section user routines, copy all lines left to userRoutines file. */
            while (line != null) {
                userRoutines.append(line).append('\n');
                line = br.readLine();
                ++lineNumber;
            }
//            ReParser reParser = new ReParser();
//            for (String s : rules.keySet()) {
//                System.out.println();
//                System.out.println("\t" + s);
//                reParser.processRe(s);
//            }

//            int i = 0;
//            for (String s : rules.keySet()) {
//                assert i < rules.size();
//                if (i == 0) {
//                    System.out.println();
//                    System.out.println("\t" + s);
//                    reParser.processRe(s);
//                    break;
//                }
//                i--;
//            }
            for (String s : definitions.keySet()) {
                System.out.println(s + "->" + definitions.get(s));
            }

            for (String s : rules.keySet()) {
                System.out.println(s + "->" + rules.get(s));
            }

            System.out.println(userRoutines.toString());
        } catch (FileNotFoundException e) {
            handleError("can't open file \"" + sourcePath + "\" to read", 0);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }


{'fraction_non_alphanumeric': 0.1147204331186409, 'fraction_numerical': 0.0024269579016148604, 'mean_word_length': 1.0627647285329227, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6585705', 'n_tokens_mistral': 2687, 'n_tokens_neox': 2479, 'n_words': 591}