<commit_before>package nl.rubensten.pp2lal2pp.parser;

import nl.rubensten.pp2lal2pp.Constants;
import nl.rubensten.pp2lal2pp.PP2LAL2PPException;
import nl.rubensten.pp2lal2pp.ParseException;
import nl.rubensten.pp2lal2pp.lang.Operator;
import nl.rubensten.pp2lal2pp.util.Regex;

import java.util.*;

/**
 * @author <NAME>
 */
public class Tokeniser implements Iterable<String> {

    /**
     * The list of tokens.
     */
    private List<String> tokens;

    /**
     * The original input.
     */
    private String original;

    public Tokeniser(String code) {
        this.original = code;

        String code2 = Regex.replaceAll("((((?<!'),))|(,(?!'))|((?<!'),(?!')))", code, " , ");
        code2 = Regex.replaceAll("((((?<!')\\)))|(\\)(?!'))|((?<!')\\)(?!')))", code2, " ) ");
        code2 = Regex.replaceAll("((((?<!')\\())|(\\((?!'))|((?<!')\\((?!')))", code2, " ( ");
        code2 = Regex.replaceAll("((((?<!')\\-(?![=\\d])))|(\\-(?!['=\\d]))|((?<!')\\-(?!['=\\d])" +
                "))", code2, " - ");
        code2 = Regex.replaceAll("((((?<!')\\}))|(\\}(?!'))|((?<!')\\}(?!')))", code2, " } ");
        code2 = Regex.replaceAll("((((?<!')\\{))|(\\{(?!'))|((?<!')\\{(?!')))", code2, " { ");

        for (Operator op : Operator.values()) {
            String c = op.getRegexSign();
            code2 = Regex.replaceAll("((((?<!')" + c + "))|(" + c + "(?!'))|((?<!')" + c + "(?!')" +
                    "))", code2, "櫓\uF214(" + op.name() + ")");
        }

        for (Operator op : Operator.values()) {
            code2 = Regex.replace("櫓\uF214(" + op.name() + ")", code2, " " + op.getSign() + " ");
        }

        String[] strings = Regex.split("\\s+", code2);

        tokens = new ArrayList<>();
        for (String string : strings) {
            if (!string.isEmpty()) {
                tokens.add(string);
            }
        }
        tokens = Collections.unmodifiableList(tokens);<commit_msg>Fixed comments not working when the hash is directly in front of text.<commit_after>package nl.rubensten.pp2lal2pp.parser;

import nl.rubensten.pp2lal2pp.Constants;
import nl.rubensten.pp2lal2pp.PP2LAL2PPException;
import nl.rubensten.pp2lal2pp.ParseException;
import nl.rubensten.pp2lal2pp.lang.Operator;
import nl.rubensten.pp2lal2pp.util.Regex;

import java.util.*;

/**
 * @author <NAME>
 */
public class Tokeniser implements Iterable<String> {

    /**
     * The list of tokens.
     */
    private List<String> tokens;

    /**
     * The original input.
     */
    private String original;

    public Tokeniser(String code) {
        this.original = code;

        String code2 = Regex.replaceAll("((((?<!'),))|(,(?!'))|((?<!'),(?!')))", code, " , ");
        code2 = Regex.replaceAll("((((?<!')#))|(#(?!'))|((?<!')#(?!')))", code2, " ) ");
        code2 = Regex.replaceAll("((((?<!')\\)))|(\\)(?!'))|((?<!')\\)(?!')))", code2, " ) ");
        code2 = Regex.replaceAll("((((?<!')\\())|(\\((?!'))|((?<!')\\((?!')))", code2, " ( ");
        code2 = Regex.replaceAll("((((?<!')\\-(?![=\\d])))|(\\-(?!['=\\d]))|((?<!')\\-(?!['=\\d])" +
                "))", code2, " - ");
        code2 = Regex.replaceAll("((((?<!')\\}))|(\\}(?!'))|((?<!')\\}(?!')))", code2, " } ");
        code2 = Regex.replaceAll("((((?<!')\\{))|(\\{(?!'))|((?<!')\\{(?!')))", code2, " { ");

        for (Operator op : Operator.values()) {
            String c = op.getRegexSign();
            code2 = Regex.replaceAll("((((?<!')" + c + "))|(" + c + "(?!'))|((?<!')" + c + "(?!')" +
                    "))", code2, "櫓\uF214(" + op.name() + ")");
        }

        for (Operator op : Operator.values()) {
            code2 = Regex.replace("櫓\uF214(" + op.name() + ")", code2, " " + op.getSign() + " ");
        }

        String[] strings = Regex.split("\\s+", code2);

        tokens = new ArrayList<>();
        for (String string : strings) {
            if (!string.isEmpty()) {
                tokens.add(string);
            }
        }
        tokens = Collections.unmodifiableList(tokens);
{'fraction_non_alphanumeric': 0.29955067398901647, 'fraction_numerical': 0.018472291562656017, 'mean_word_length': 2.478298611111111, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3312084', 'n_tokens_mistral': 1512, 'n_tokens_neox': 1469, 'n_words': 210}