<commit_before>package com.electronwill.toml;

import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * Class for reading TOML v0.4.0.
 * 
 * @author TheElectronWill
 * 		
 */
public final class TomlReader {
	
	private final String data;
	private int pos = 0;// current position
	
	public TomlReader(String data) {
		this.data = data;
	}
	
	private boolean hasNext() {
		return pos < data.length();
	}
	
	private char next() {
		return data.charAt(pos++);
	}
	
	private char nextUseful(boolean skipComments) {
		char c = ' ';
		while (hasNext() && (c == ' ' || c == '\t' || c == '\r' || c == '\n' || (c == '#' && skipComments))) {
			c = next();
			if (c == '#')
				pos = data.indexOf('\n', pos) + 1;
		}
		return c;
	}
	
	private char nextUsefulOrLinebreak() {
		char c = ' ';
		while (c == ' ' || c == '\t')
			c = next();
		return c;<commit_msg>Fixed the nextUseful() method to support a comment on the last line<commit_after>package com.electronwill.toml;

import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * Class for reading TOML v0.4.0.
 * 
 * @author TheElectronWill
 * 		
 */
public final class TomlReader {
	
	private final String data;
	private int pos = 0;// current position
	
	public TomlReader(String data) {
		this.data = data;
	}
	
	private boolean hasNext() {
		return pos < data.length();
	}
	
	private char next() {
		return data.charAt(pos++);
	}
	
	private char nextUseful(boolean skipComments) {
		char c = ' ';
		while (hasNext() && (c == ' ' || c == '\t' || c == '\r' || c == '\n' || (c == '#' && skipComments))) {
			c = next();
			if (c == '#') {
				int nextLinebreak = data.indexOf('\n', pos);
				pos = (nextLinebreak == -1) ? data.length() : nextLinebreak + 1;
			}
		}
		return c;
	}
	
	private char nextUsefulOrLinebreak() {
		char c = ' ';
		while (c == ' ' || c == '\t')
			c = next();
		return c;
{'fraction_non_alphanumeric': 0.1688073394495413, 'fraction_numerical': 0.005045871559633028, 'mean_word_length': 3.138519924098672, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '400596', 'n_tokens_mistral': 838, 'n_tokens_neox': 759, 'n_words': 206}