<commit_before>package innovimax.mixthem.io;

import innovimax.mixthem.arguments.TokenType;
import innovimax.mixthem.utils.StreamUtils;

import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

/**
* This is the implementation of ITokenRange interface.
* @author Innovimax
* @version 1.0
*/
public class TokenRangeReader implements ITokenRange {

	private final TokenType tokenType;
	private final List<IToken> readers;
	
	/**
	* Constructor
	* @param inputs The list of inputs as InputResource
	* @param selection The input index selection (maybe empty)
	* @param tokenType The input tokenization type
	* @see innovimax.mixthem.io.InputResource
	*/
	public TokenRangeReader(final List<InputResource> inputs, final Set<Integer> selection, final TokenType tokenType) {
		this.tokenType = tokenType;
		this.readers = IntStream.rangeClosed(1, inputs.size())
			.filter(index -> selection.isEmpty() || selection.contains(Integer.valueOf(index)))
			.mapToObj(index -> inputs.get(index-1))
			.map(StreamUtils.apply(input -> {
				switch(this.tokenType) {
					case BYTE: return new DefaultByteReader(input);
					case CHAR: return new DefaultCharReader(input);
					case LINE: return new DefaultLineReader(input);
					case FILEBYTE: /*return new DefaultFileByteReader(input)*/ throw new RuntimeException("TODO");
					case FILECHAR: /*return new DefaultFileCharReader(input)*/ throw new RuntimeException("TODO");
					default: throw new UnsupportedOperationException("Token not expected: " + tokenType.getName());
				}
			}))
			.collect(Collectors.toList());
	}

	@Override
	public boolean hasMoreTokens() {
		//TODO: remove switch !!!
		//	see below...
		return this.readers.stream()
			.anyMatch(StreamUtils.test(reader -> {
				switch(this.tokenType) {
					case BYTE: return ((IByteInput) reader).hasByte();
					case CHAR: return ((ICharInput) reader).hasCharacter();
					case LINE: return ((ILineInput) reader).hasLine();
					case FILEBYTE: /*((IFileInput) reader).hasFile()*/ throw new RuntimeException("TODO");
					case FILECHAR: /*((IFileInput) reader).hasFile()*/ throw new RuntimeException("TODO");
					default: throw new UnsupportedOperationException("Token not expected: " + tokenType.getName());
				}
			}));
	}

	@Override
	public List<Token> nextTokenRange(List<Boolean> readingRange) {
		//TODO: remove switch !!!
		//	replace nextByte, nextChar, nextLine by nextToken
		//	add hasMoreTokens, nextToken in ITokenInput
		//	remove IByteInput, ICharInput, ILineInput
		//	DefaultByteReader, DefaultCharReader, DefaultLineReader directly implements ITokenInput
		switch(this.tokenType) {
			case BYTE: return readers.stream()
					.map(StreamUtils.apply(reader -> ((IByteInput) reader).nextByte()))
					.map(b -> Token.createByteToken(b))
					.collect(Collectors.toList());
			case CHAR: return readers.stream()
					.map(StreamUtils.apply(reader -> ((ICharInput) reader).nextCharacter()))
					.map(c -> Token.createCharToken(c))
					.collect(Collectors.toList());
			case LINE: return IntStream.range(0, readers.size())
					.mapToObj(StreamUtils.applyToInt(index -> 
						readingRange.get(index).booleanValue() ? 
						((ILineInput) readers.get(index)).nextLine() : null))
					.map(line -> Token.createLineToken(line))
					.collect(Collectors.toList());
			case FILEBYTE: /*((IFileInput) reader).hasFile()*/ throw new RuntimeException("TODO");
			case FILECHAR: /*((IFileInput) reader).hasFile()*/ throw new RuntimeException("TODO");
			default: throw new UnsupportedOperationException("Token not expected: " + tokenType.getName());
		}
	}
	
	@Override
	public void close() {
		this.readers
			.forEach(StreamUtils.consume(reader -> reader.close()));
	}
<commit_msg>Use IToken interface in token range reader<commit_after>package innovimax.mixthem.io;

import innovimax.mixthem.arguments.TokenType;
import innovimax.mixthem.utils.StreamUtils;

import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

/**
* This is the implementation of ITokenRange interface.
* @author Innovimax
* @version 1.0
*/
public class TokenRangeReader implements ITokenRange {

	private final List<IToken> readers;
	
	/**
	* Constructor
	* @param inputs The list of inputs as InputResource
	* @param selection The input index selection (maybe empty)
	* @param tokenType The input tokenization type
	* @see innovimax.mixthem.io.InputResource
	*/
	public TokenRangeReader(final List<InputResource> inputs, final Set<Integer> selection, final TokenType tokenType) {
		this.readers = IntStream.rangeClosed(1, inputs.size())
			.filter(index -> selection.isEmpty() || selection.contains(Integer.valueOf(index)))
			.mapToObj(index -> inputs.get(index-1))
			.map(StreamUtils.apply(input -> {
				switch(tokenType) {
					case BYTE: return new DefaultByteReader(input);
					case CHAR: return new DefaultCharReader(input);
					case LINE: return new DefaultLineReader(input);
					case FILEBYTE: /*return new DefaultFileByteReader(input)*/ throw new RuntimeException("TODO");
					case FILECHAR: /*return new DefaultFileCharReader(input)*/ throw new RuntimeException("TODO");
					default: throw new UnsupportedOperationException("Token not expected: " + tokenType.getName());
				}
			}))
			.collect(Collectors.toList());
	}

	@Override
	public boolean hasMoreTokens() {		
		return this.readers.stream()
			.anyMatch(StreamUtils.test(reader -> reader.hasMoreTokens()));
	}

	@Override
	public List<Token> nextTokenRange(List<Boolean> readingRange) {
		return IntStream.range(0, readers.size())
			.mapToObj(StreamUtils.applyToInt(index -> 
				readingRange.get(index).booleanValue() ? readers.get(index).nextToken() : null))
			.collect(Collectors.toList());
	}
	
	@Override
	public void close() {
		this.readers
			.forEach(StreamUtils.consume(reader -> reader.close()));
	}

{'fraction_non_alphanumeric': 0.12924016282225237, 'fraction_numerical': 0.0016960651289009499, 'mean_word_length': 4.573724007561436, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1459205', 'n_tokens_mistral': 1970, 'n_tokens_neox': 1742, 'n_words': 416}