<commit_before>package stream.flarebot.flarebot.util;

import stream.flarebot.flarebot.FlareBot;
import stream.flarebot.flarebot.commands.Command;
import stream.flarebot.flarebot.commands.general.CommandUsageCommand;
import stream.flarebot.flarebot.commands.general.TagsCommand;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class UsageParser {

    public static void test(String... ss) {
        for (String s : matchUsage(new CommandUsageCommand(), ss)) {
            FlareBot.getInstance().getImportantLogChannel().sendMessage(s).queue();
        }
    }

    public static List<String> matchUsage(Command c, String[] args) {
        List<String> strings = new ArrayList<>();
        if (args.length == 0) {
            Collections.addAll(strings, c.getUsage().split("\n"));
            return strings;
        }
        for (String usage : c.getUsage().split("\n")) {
            Pattern p = Pattern.compile("`.+`");
            String symbols = usage.replace("{%}" + c.getCommand(), "").trim(); // Strip command out
            Matcher m = p.matcher(symbols);
            if (m.find()) {
                symbols = m.group().replace("`", "").trim();
            } else {
                continue;
            }
            Map<Integer, Pair<Symbol, String>> map = findSymbols(symbols);
            boolean applicable = false;
            if (args.length > map.size()) { // If there are more args than symbols it wouldn't be applicable
                continue;
            }
            for (Map.Entry<Integer, Pair<Symbol, String>> entry : map.entrySet()) {
                if (entry.getValue().getKey() == Symbol.SINGLE_SUB_COMMAND) {
                    if (args[entry.getKey()].equalsIgnoreCase(entry.getValue().getValue())) {
                        applicable = true; // Sub command matches arg
                    } else {
                        break; // We don't want to check any other args if this fails
                    }
                } else if (entry.getValue().getKey() == Symbol.MULTIPLE_SUB_COMMAND) {
                    for (String cmd : entry.getValue().getValue().split("\\|")) {
                        if (args[entry.getKey()].equalsIgnoreCase(cmd)) {
                            applicable = true;
                            break;
                        }
                    }
                    if (!applicable) break; // CHeck nothing else if this fails
                } else {
                    applicable = true;
                }
            }

            if (applicable) strings.add(usage);
        }
        return strings;
    }

    private static Map<Integer, Pair<Symbol, String>> findSymbols(String string) {
        Map<Integer, Pair<Symbol, String>> map = new HashMap<>();
        int i = 0;
        for (String s : string.split(" ")) {
            for (Symbol sy : Symbol.values()) {
                if (sy.matches(s)) map.put(i, new Pair<>(sy, s));
            }
            i++;
        }
        return map;
    }


    public enum Symbol {
        SINGLE_SUB_COMMAND(Pattern.compile("^[A-Za-z]+$")),
        MULTIPLE_SUB_COMMAND(Pattern.compile("^[A-z]+(\\|+[A-z]+)+$")),
        REQUIRED_ARG(Pattern.compile("^<.+>$")),
        OPTIONAL_ARG(Pattern.compile("^\\[.+\\]$"));

        private Pattern regex;

        Symbol(Pattern regex) {
            this.regex = regex;
        }

        public Pattern getRegex() {
            return regex;
        }

        public boolean matches(String arg) {
            return getRegex().matcher(arg).matches();
        }

    }

}
<commit_msg>Fix bugs kthnx<commit_after>package stream.flarebot.flarebot.util;

import org.apache.commons.lang3.StringUtils;
import stream.flarebot.flarebot.FlareBot;
import stream.flarebot.flarebot.commands.Command;
import stream.flarebot.flarebot.commands.general.CommandUsageCommand;
import stream.flarebot.flarebot.commands.general.PollCommand;
import stream.flarebot.flarebot.commands.general.TagsCommand;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class UsageParser {

    public static void test(String... ss) {
        for (String s : matchUsage(new PollCommand(), ss)) {
            FlareBot.getInstance().getImportantLogChannel().sendMessage(s).queue();
        }
    }

    public static List<String> matchUsage(Command c, String[] args) {
        List<String> strings = new ArrayList<>();
        String[] usages = c.getUsage().split("\n");
        usages = Arrays.stream(usages).filter(StringUtils::isNotEmpty).toArray(String[]::new);
        if (args.length == 0) {
            Collections.addAll(strings, usages);
            return strings;
        }
        for (String usage : usages) {
            Pattern p = Pattern.compile("`.+`");
            String symbols = usage.replace("{%}" + c.getCommand(), "").trim(); // Strip command out
            Matcher m = p.matcher(symbols);
            if (m.find()) {
                symbols = m.group().replace("`", "").trim();
            } else {
                continue;
            }
            Map<Integer, Pair<Symbol, String>> map = findSymbols(symbols);
            boolean applicable = false;
            if (args.length > map.size()) { // If there are more args than symbols it wouldn't be applicable
                continue;
            }
            for (Map.Entry<Integer, Pair<Symbol, String>> entry : map.entrySet()) {
                if (entry.getValue().getKey() == Symbol.SINGLE_SUB_COMMAND) {
                    if (args[entry.getKey()].equalsIgnoreCase(entry.getValue().getValue())) {
                        applicable = true; // Sub command matches arg
                    } else {
                        applicable = false;
                        break; // We don't want to check any other args if this fails
                    }
                } else if (entry.getValue().getKey() == Symbol.MULTIPLE_SUB_COMMAND) {
                    boolean valid = false;
                    for (String cmd : entry.getValue().getValue().split("\\|")) {
                        if (args[entry.getKey()].equalsIgnoreCase(cmd)) {
                            applicable = true;
                            valid = true;
                            break;
                        }
                    }
                    if (!valid) {
                        applicable = false;
                        break; // CHeck nothing else if this fails
                    }
                } else {
                    applicable = true;
                }
                if (!(args.length - 1 >= entry.getKey())) {
                    break;
                }
            }

            if (applicable) strings.add(usage);
        }
        return strings;
    }

    private static Map<Integer, Pair<Symbol, String>> findSymbols(String string) {
        Map<Integer, Pair<Symbol, String>> map = new HashMap<>();
        int i = 0;
        for (String s : string.split(" ")) {
            for (Symbol sy : Symbol.values()) {
                if (sy.matches(s)) map.put(i, new Pair<>(sy, s));
            }
            i++;
        }
        return map;
    }


    public enum Symbol {
        SINGLE_SUB_COMMAND(Pattern.compile("^[A-Za-z]+$")),
        MULTIPLE_SUB_COMMAND(Pattern.compile("^[A-z]+(\\|+[A-z]+)+$")),
        REQUIRED_ARG(Pattern.compile("^<.+>$")),
        OPTIONAL_ARG(Pattern.compile("^\\[.+\\]$"));

        private Pattern regex;

        Symbol(Pattern regex) {
            this.regex = regex;
        }

        public Pattern getRegex() {
            return regex;
        }

        public boolean matches(String arg) {
            return getRegex().matcher(arg).matches();
        }

    }

}

{'fraction_non_alphanumeric': 0.13367641521275261, 'fraction_numerical': 0.0007531065645788879, 'mean_word_length': 1.7083616587355541, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3225918', 'n_tokens_mistral': 2201, 'n_tokens_neox': 2083, 'n_words': 473}