<commit_before>package seedu.todo.controllers;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang.StringUtils;

import edu.emory.mathcs.backport.java.util.Arrays;
import seedu.todo.commons.exceptions.UnmatchedQuotesException;

public class Tokenizer {

    private final static String QUOTE = "\"";

    private static class TokenizedString {
        public String string;
        public boolean isToken;
        public boolean isQuote;

        TokenizedString(String string, boolean isToken, boolean isQuote) {
            this.string = string;
            this.isToken = isToken;
            this.isQuote = isQuote;
        }

        @Override
        public String toString() {
            return String.format("TokenizedString(%s, %s, %s)", this.string, isToken, isQuote);
        }
    }

    public static HashMap<String, String[]> tokenize(HashMap<String, String[]> tokenDefinitions, String inputCommand)
            throws UnmatchedQuotesException {
        
        // Generate token -> tokenType mapping and list of tokens
        List<String> tokens = new ArrayList<String>();
        HashMap<String, String> getTokenType = new HashMap<String, String>();
        for (Map.Entry<String, String[]> tokenDefinition : tokenDefinitions.entrySet()) {
            String tokenType = tokenDefinition.getKey();
            for (String token : tokenDefinition.getValue()) {
                tokens.add(token);
                getTokenType.put(token, tokenType);
            }
        }

        if (inputCommand.length() == 0)
            return null;

        // Split inputCommand into arraylist of chunks

        if (StringUtils.countMatches(inputCommand, QUOTE) % 2 == 1)
            throw new UnmatchedQuotesException("Unmatched double-quotes detected.");

        // --- Split by quotes
        String[] splitString = inputCommand.split(QUOTE);

        // If first char is QUOTE, then first element is a quoted string.
        List<TokenizedString> tokenizedSplitString = new ArrayList<TokenizedString>();
        for (int i = 0; i < splitString.length; i++) {
            tokenizedSplitString.add(new TokenizedString(splitString[i].trim(), false, (i % 2 == 1)));
        }
        
        // --- Split by tokens
        for (int i = 0; i < tokenizedSplitString.size(); i++) { // Java doesn't eager-evaluate the terminating condition
            TokenizedString currString = tokenizedSplitString.get(i);
            if (currString.isQuote || currString.isToken)
                continue;
            
            // Try to match all the tokens
            for (String token : tokens) {
                Matcher m = Pattern.compile(String.format("\\b%s\\b", token)).matcher(currString.string);
                if (!m.find())
                    continue;
                
                // Found. Replace current element with split elements.
                String preString = currString.string.substring(0, m.start()).trim();
                String postString = currString.string.substring(m.end(), currString.string.length()).trim();
                
                tokenizedSplitString.remove(i);
                List<TokenizedString> replacedSplitStrings = new ArrayList<TokenizedString>();
                if (!preString.isEmpty())
                    replacedSplitStrings.add(new TokenizedString(preString, false, false));
                replacedSplitStrings.add(new TokenizedString(token, true, false));
                if (!postString.isEmpty())
                    replacedSplitStrings.add(new TokenizedString(postString, false, false));
                tokenizedSplitString.addAll(i, replacedSplitStrings);
                
                // Update currString and resume.
                currString = tokenizedSplitString.get(i);
                if (currString.isQuote || currString.isToken)
                    break;
            }   
        }
        
        // Get arraylist of indices
        // Get dictionary of tokenType -> index
        
        // Return dictionary of tokenType -> {token, tokenField}
        
        
        return null;
    }

}
<commit_msg>Only match one occurrence per token, store in hashmap.<commit_after>package seedu.todo.controllers;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang.StringUtils;

import edu.emory.mathcs.backport.java.util.Arrays;
import seedu.todo.commons.exceptions.UnmatchedQuotesException;

public class Tokenizer {

    private final static String QUOTE = "\"";

    private static class TokenizedString {
        public String string;
        public boolean isToken;
        public boolean isQuote;

        TokenizedString(String string, boolean isToken, boolean isQuote) {
            this.string = string;
            this.isToken = isToken;
            this.isQuote = isQuote;
        }

        @Override
        public String toString() {
            return String.format("TokenizedString(%s, %s, %s)", this.string, isToken, isQuote);
        }
    }

    public static HashMap<String, String[]> tokenize(HashMap<String, String[]> tokenDefinitions, String inputCommand)
            throws UnmatchedQuotesException {
        
        // Generate token -> tokenType mapping and list of tokens
        List<String> tokens = new ArrayList<String>();
        HashMap<String, String> getTokenType = new HashMap<String, String>();
        for (Map.Entry<String, String[]> tokenDefinition : tokenDefinitions.entrySet()) {
            String tokenType = tokenDefinition.getKey();
            for (String token : tokenDefinition.getValue()) {
                tokens.add(token);
                getTokenType.put(token, tokenType);
            }
        }

        if (inputCommand.length() == 0)
            return null;

        // Split inputCommand into arraylist of chunks

        if (StringUtils.countMatches(inputCommand, QUOTE) % 2 == 1)
            throw new UnmatchedQuotesException("Unmatched double-quotes detected.");

        // --- Split by quotes
        String[] splitString = inputCommand.split(QUOTE);

        // If first char is QUOTE, then first element is a quoted string.
        List<TokenizedString> tokenizedSplitString = new ArrayList<TokenizedString>();
        for (int i = 0; i < splitString.length; i++) {
            tokenizedSplitString.add(new TokenizedString(splitString[i].trim(), false, (i % 2 == 1)));
        }
        
        // --- Split by tokens
        Map<String, Integer> tokenIndices = new HashMap<String, Integer>();
        for (int i = 0; i < tokenizedSplitString.size(); i++) { // Java doesn't eager-evaluate the terminating condition
            TokenizedString currString = tokenizedSplitString.get(i);
            if (currString.isQuote)
                continue;
            if (currString.isToken) {
                tokenIndices.put(currString.string, i);
                tokens.remove(currString.string);
                continue;
            }
            
            // Try to match all the tokens
            for (String token : tokens) {
                Matcher m = Pattern.compile(String.format("\\b%s\\b", token)).matcher(currString.string);
                if (!m.find())
                    continue;
                
                // Found. Replace current element with split elements.
                String preString = currString.string.substring(0, m.start()).trim();
                String postString = currString.string.substring(m.end(), currString.string.length()).trim();
                
                tokenizedSplitString.remove(i);
                List<TokenizedString> replacedSplitStrings = new ArrayList<TokenizedString>();
                if (!preString.isEmpty())
                    replacedSplitStrings.add(new TokenizedString(preString, false, false));
                replacedSplitStrings.add(new TokenizedString(token, true, false));
                if (!postString.isEmpty())
                    replacedSplitStrings.add(new TokenizedString(postString, false, false));
                tokenizedSplitString.addAll(i, replacedSplitStrings);
                
                // Update currString and resume.
                currString = tokenizedSplitString.get(i);
                if (currString.isQuote)
                    break;
                if (currString.isToken) {
                    tokenIndices.put(currString.string, i);
                    tokens.remove(currString.string);
                    break;
                }
            }   
        }
        
        // Get arraylist of indices
        // Get dictionary of tokenType -> index
        
        // Return dictionary of tokenType -> {token, tokenField}
        
        
        return null;
    }

}

{'fraction_non_alphanumeric': 0.10044742729306488, 'fraction_numerical': 0.0017897091722595079, 'mean_word_length': 1.9933043187144293, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2646512', 'n_tokens_mistral': 2310, 'n_tokens_neox': 2209, 'n_words': 562}