<commit_before>/*
 * JaamSim Discrete Event Simulation
 * Copyright (C) 2013 Ausenco Engineering Canada Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */
package com.jaamsim.input;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import java.util.ArrayList;

import org.junit.Test;

public class TestParser {

@Test
public void testTokenize() {
	ArrayList<String> tok = new ArrayList<String>();

	// Test the basic delimiter handling, runs of [ ,\t] are one delimiter
	tok.clear();
	Parser.tokenize(tok, "A A,A\tA  A,,A\t\tA ,\tA");
	tokenMatch(tok, "A", "A,A", "A", "A,,A", "A", ",", "A");

	tok.clear();
	Parser.tokenize(tok, " A A,A\tA  A,,A\t\tA ,\tA\t,  ");
	tokenMatch(tok, "A", "A,A", "A", "A,,A", "A", ",", "A", ",");

	tok.clear();
	Parser.tokenize(tok, "OBJECT KEYWORD{ ARG}KEYWORD\t{ARG ARG,}");
	tokenMatch(tok, "OBJECT", "KEYWORD", "{", "ARG", "}", "KEYWORD", "{", "ARG", "ARG,", "}");

	tok.clear();
	Parser.tokenize(tok, "OBJECT KEYWORD{ 'ARG  '}KEYWORD\t{ARG' ARG',}");
	tokenMatch(tok, "OBJECT", "KEYWORD", "{", "ARG  ", "}", "KEYWORD", "{", "ARG", " ARG", ",", "}");

	tok.clear();
	Parser.tokenize(tok, "OBJECT KEYWORD{ ARG }\"FOO ,\t     ");
	tokenMatch(tok, "OBJECT", "KEYWORD", "{", "ARG", "}", "\"FOO ,\t     ");
}

private static void validateTokens(ArrayList<String> toks) {
	for (String each : toks) {
		// An empty String is not a valid token
		assertTrue(!each.isEmpty());

		// no further validation of line-quotes
		if (each.startsWith("\""))
			continue;

		// valid tokens connot contain one of the delimiters
		assertTrue(!each.contains("'"));

		// If a token contains { or } it must be one of those exactly
		if (each.equals("{"))
			assertTrue(each == "{");

		if (each.equals("}"))
			assertTrue(each == "}");
	}
}

private static void tokenMatch(ArrayList<String> toks, String... expected) {
	assertTrue(toks.size() == expected.length);
	validateTokens(toks);
	for (int i = 0; i < toks.size(); i++) {
		assertTrue(expected[i].equals(toks.get(i)));
	}
}
<commit_msg>TEST: add parser test for unbalanced single-quote<commit_after>/*
 * JaamSim Discrete Event Simulation
 * Copyright (C) 2013 Ausenco Engineering Canada Inc.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 */
package com.jaamsim.input;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import java.util.ArrayList;

import org.junit.Test;

public class TestParser {

@Test
public void testTokenize() {
	ArrayList<String> tok = new ArrayList<String>();

	// Test the basic delimiter handling, runs of [ ,\t] are one delimiter
	tok.clear();
	Parser.tokenize(tok, "A A,A\tA  A,,A\t\tA ,\tA");
	tokenMatch(tok, "A", "A,A", "A", "A,,A", "A", ",", "A");

	tok.clear();
	Parser.tokenize(tok, " A A,A\tA  A,,A\t\tA ,\tA\t,  ");
	tokenMatch(tok, "A", "A,A", "A", "A,,A", "A", ",", "A", ",");

	tok.clear();
	Parser.tokenize(tok, "OBJECT KEYWORD{ ARG}KEYWORD\t{ARG ARG,}");
	tokenMatch(tok, "OBJECT", "KEYWORD", "{", "ARG", "}", "KEYWORD", "{", "ARG", "ARG,", "}");

	tok.clear();
	Parser.tokenize(tok, "OBJECT KEYWORD{ 'ARG  '}KEYWORD\t{ARG' ARG',}");
	tokenMatch(tok, "OBJECT", "KEYWORD", "{", "ARG  ", "}", "KEYWORD", "{", "ARG", " ARG", ",", "}");

	tok.clear();
	Parser.tokenize(tok, "OBJECT KEYWORD{ ARG }\"FOO ,\t     ");
	tokenMatch(tok, "OBJECT", "KEYWORD", "{", "ARG", "}", "\"FOO ,\t     ");

	tok.clear();
	Parser.tokenize(tok, "'OBJECT''KEYWORD''   ");
	tokenMatch(tok, "OBJECT", "KEYWORD", "   ");
}

private static void validateTokens(ArrayList<String> toks) {
	for (String each : toks) {
		// An empty String is not a valid token
		assertTrue(!each.isEmpty());

		// no further validation of line-quotes
		if (each.startsWith("\""))
			continue;

		// valid tokens connot contain one of the delimiters
		assertTrue(!each.contains("'"));

		// If a token contains { or } it must be one of those exactly
		if (each.equals("{"))
			assertTrue(each == "{");

		if (each.equals("}"))
			assertTrue(each == "}");
	}
}

private static void tokenMatch(ArrayList<String> toks, String... expected) {
	assertTrue(toks.size() == expected.length);
	validateTokens(toks);
	for (int i = 0; i < toks.size(); i++) {
		assertTrue(expected[i].equals(toks.get(i)));
	}
}

{'fraction_non_alphanumeric': 0.18382780686445607, 'fraction_numerical': 0.002326934264107039, 'mean_word_length': 4.142572283150549, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5443560', 'n_tokens_mistral': 1803, 'n_tokens_neox': 1744, 'n_words': 536}