<commit_before>package org.springframework.roo.shell;

import java.util.LinkedHashMap;
import java.util.Map;

import org.springframework.roo.support.util.Assert;

public class ParserUtils {
	
	/**
	 * Converts a particular buffer into a tokenized structure.
	 * 
	 * Properly treats double quotes (") as option delimiters.
	 * 
	 * Expects option names to be preceded by a single or double dash. We call this an "option marker".
	 * 
	 * Treats spaces as the default option tokenizer.
	 * 
	 * Any token without an option marker is considered the default. The default is returned in the Map as an
	 * element with an empty string key (""). There can only be a single default.
	 * 
	 * @param remainingBuffer to tokenize
	 * @return a Map where keys are the option names (minus any dashes) and values are the option values (any double-quotes are removed)
	 * 
	 */
	public static Map<String,String> tokenize(String remainingBuffer) {
		Assert.notNull(remainingBuffer, "Remaining buffer cannot be null, although it can be empty");
		Map<String,String> result = new LinkedHashMap<String, String>();
		StringBuilder currentOption = new StringBuilder();
		StringBuilder currentValue = new StringBuilder();
		boolean inQuotes = false;
		
		// Verify correct number of double quotes are present
		int count = 0;
		for (char c : remainingBuffer.toCharArray()) {
			if ('"' == c) {
				count++;
			}
		}
		Assert.isTrue(count % 2 == 0, "Cannot have an unbalanced number of quotation marks");
		
		if ("".equals(remainingBuffer.trim())) {
			// They've not specified anything, so exit now
			return result;
		}
		
		String[] split = remainingBuffer.split(" ");
		for (int i = 0 ; i < split.length; i++) {
			String currentToken = split[i];
			
			if (inQuotes == true) {
				// We're only interested in this token series ending
				if (currentToken.endsWith("\"")) {
					// The current token series has ended
					String tokenLessDelimiters = currentToken.substring(0, currentToken.length()-1);
					currentValue.append(" ").append(tokenLessDelimiters);
					inQuotes = false;
					
					// Store this now-ended token series
					store(result, currentOption, currentValue);
					currentOption = new StringBuilder();
					currentValue = new StringBuilder();
					
				} else {
					// The current token series has not ended
					currentValue.append(" ").append(currentToken);
				}
				continue;
			}
			
			if (currentToken.startsWith("\"")) {
				// We're about to start a new delimited token
				String tokenLessDelimiters = currentToken.substring(1);
				currentValue.append(tokenLessDelimiters);
				inQuotes = true;
				continue;
			}
			
			if (currentToken.startsWith("-")) {
				// We're about to start a new option marker
				// First strip of the - or -- or however many there are
				int lastIndex = currentToken.lastIndexOf("-");
				String tokenLessDelimiters = currentToken.substring(lastIndex+1);
				currentOption.append(tokenLessDelimiters);
				
				// Store this token if it's the last one, or the next token starts with a "-"
				if (i+1 == split.length) {
					// We're at the end of the tokens, so store this one and stop processing<commit_msg>ROO-23: Error on parsing command in Roo shell<commit_after>package org.springframework.roo.shell;

import java.util.LinkedHashMap;
import java.util.Map;

import org.springframework.roo.support.util.Assert;

public class ParserUtils {
	
	/**
	 * Converts a particular buffer into a tokenized structure.
	 * 
	 * Properly treats double quotes (") as option delimiters.
	 * 
	 * Expects option names to be preceded by a single or double dash. We call this an "option marker".
	 * 
	 * Treats spaces as the default option tokenizer.
	 * 
	 * Any token without an option marker is considered the default. The default is returned in the Map as an
	 * element with an empty string key (""). There can only be a single default.
	 * 
	 * @param remainingBuffer to tokenize
	 * @return a Map where keys are the option names (minus any dashes) and values are the option values (any double-quotes are removed)
	 * 
	 */
	public static Map<String,String> tokenize(String remainingBuffer) {
		Assert.notNull(remainingBuffer, "Remaining buffer cannot be null, although it can be empty");
		Map<String,String> result = new LinkedHashMap<String, String>();
		StringBuilder currentOption = new StringBuilder();
		StringBuilder currentValue = new StringBuilder();
		boolean inQuotes = false;
		
		// Verify correct number of double quotes are present
		int count = 0;
		for (char c : remainingBuffer.toCharArray()) {
			if ('"' == c) {
				count++;
			}
		}
		Assert.isTrue(count % 2 == 0, "Cannot have an unbalanced number of quotation marks");
		
		if ("".equals(remainingBuffer.trim())) {
			// They've not specified anything, so exit now
			return result;
		}
		
		String[] split = remainingBuffer.split(" ");
		for (int i = 0 ; i < split.length; i++) {
			String currentToken = split[i];
			
			if (inQuotes == true) {
				// We're only interested in this token series ending
				if (currentToken.endsWith("\"")) {
					// The current token series has ended
					String tokenLessDelimiters = currentToken.substring(0, currentToken.length()-1);
					currentValue.append(" ").append(tokenLessDelimiters);
					inQuotes = false;
					
					// Store this now-ended token series
					store(result, currentOption, currentValue);
					currentOption = new StringBuilder();
					currentValue = new StringBuilder();
					
				} else {
					// The current token series has not ended
					currentValue.append(" ").append(currentToken);
				}
				continue;
			}
			
			if (currentToken.startsWith("\"")) {
				// We're about to start a new delimited token
				String tokenLessDelimiters = currentToken.substring(1);
				currentValue.append(tokenLessDelimiters);
				inQuotes = true;
				continue;
			}
			
			if (currentToken.trim().equals("")) {
				// It's simply empty, so ignore it (ROO-23)
				continue;
			}
			
			if (currentToken.startsWith("-")) {
				// We're about to start a new option marker
				// First strip of the - or -- or however many there are
				int lastIndex = currentToken.lastIndexOf("-");
				String tokenLessDelimiters = currentToken.substring(lastIndex+1);
				currentOption.append(tokenLessDelimiters);
				
				// Store this token if it's the last one, or the next token starts with a "-"
				if (i+1 == split.length) {
					// We're at the end of the tokens, so store this one and stop processing
{'fraction_non_alphanumeric': 0.10343759677918861, 'fraction_numerical': 0.0034066274388355527, 'mean_word_length': 3.2298624754420433, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5629026', 'n_tokens_mistral': 2176, 'n_tokens_neox': 1764, 'n_words': 664}