<commit_before>package natlab;

import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.List;
import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import junit.framework.TestCase;
import beaver.Scanner;
import beaver.Symbol;

class ScannerTestBase extends TestCase {
	private static final Pattern SYMBOL_PATTERN = Pattern.compile("^\\s*([_A-Z]+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s*(?:\\s(.+))?$");
	
	static void checkScan(Scanner scanner, List<Symbol> symbols, Scanner.Exception exception) 
			throws IOException, Scanner.Exception {
		int i = 0;
		for(Symbol expected : symbols) {
			try {
				Symbol actual = scanner.nextToken();
				assertEquals("Token #" + i, actual, expected);
				i++;
			} catch(Scanner.Exception e) {
				assertEquals(e.getMessage(), e, exception);
				return;
			}
		}
		try {
			assertEquals(scanner.nextToken().getId(), NatlabParser.Terminals.EOF);
		} catch(Scanner.Exception e) {
			assertEquals(e.getMessage(), e, exception);
		}
	}

	static Scanner getScanner(String filename) throws FileNotFoundException {
		return new NatlabScanner(new BufferedReader(new FileReader(filename)));
	}

	static Scanner.Exception parseSymbols(String filename, /*OUT*/ List<Symbol> symbols) 
			throws IOException, IllegalArgumentException, SecurityException, IllegalAccessException, NoSuchFieldException {
		BufferedReader in = new BufferedReader(new FileReader(filename));
		while(in.ready()) {<commit_msg>Better error message for EOF case.<commit_after>package natlab;

import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.List;
import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import junit.framework.TestCase;
import beaver.Scanner;
import beaver.Symbol;

class ScannerTestBase extends TestCase {
	private static final Pattern SYMBOL_PATTERN = Pattern.compile("^\\s*([_A-Z]+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)\\s*(?:\\s(.+))?$");
	
	static void checkScan(Scanner scanner, List<Symbol> symbols, Scanner.Exception exception) 
			throws IOException, Scanner.Exception {
		int i = 0;
		for(Symbol expected : symbols) {
			try {
				Symbol actual = scanner.nextToken();
				assertEquals("Token #" + i, actual, expected);
				i++;
			} catch(Scanner.Exception e) {
				assertEquals(e.getMessage(), e, exception);
				return;
			}
		}
		try {
			short actualId = scanner.nextToken().getId();
			short expectedId = NatlabParser.Terminals.EOF;
			if(actualId != expectedId) {
				fail("Token #" + i + ": incorrect token type - " +
						"expected: " + expectedId + " (" + NatlabParser.Terminals.NAMES[expectedId] + ") " +
						"but was: " + actualId + " (" + NatlabParser.Terminals.NAMES[actualId] + ")");
			}
		} catch(Scanner.Exception e) {
			assertEquals(e.getMessage(), e, exception);
		}
	}

	static Scanner getScanner(String filename) throws FileNotFoundException {
		return new NatlabScanner(new BufferedReader(new FileReader(filename)));
	}

	static Scanner.Exception parseSymbols(String filename, /*OUT*/ List<Symbol> symbols) 
			throws IOException, IllegalArgumentException, SecurityException, IllegalAccessException, NoSuchFieldException {
		BufferedReader in = new BufferedReader(new FileReader(filename));
		while(in.ready()) {
{'fraction_non_alphanumeric': 0.13905152224824355, 'fraction_numerical': 0.000585480093676815, 'mean_word_length': 4.666666666666667, 'pattern_counts': {'":': 1, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2760853', 'n_tokens_mistral': 1152, 'n_tokens_neox': 1035, 'n_words': 231}