<commit_before>/*
 *  Copyright <NAME> <<EMAIL>>
 *
 *  Licensed under the Apache License, Version 2.0; 
 *  you may obtain a copy of the License at:
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 */
package hu.webhejj.commons.io.filter;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.ReadableByteChannel;
import java.util.Arrays;
import java.util.List;

/**
 * Takes a list of ByteBufferFilters and applies them
 * to a File or ReadableByteChannel.
 *
 */
public class ChannelFilterer {
	
	private List<ByteBufferFilter> filters;
	
	public ChannelFilterer(ByteBufferFilter... filters) {
		this.filters = Arrays.asList(filters);
	}
	

	public void filter(File file, int bufSize) throws FileNotFoundException {
		filter(new FileInputStream(file).getChannel(), bufSize);
	}

	public void filter(ReadableByteChannel channel, int bufSize) {
		
		ByteBuffer buffer = ByteBuffer.allocate(bufSize);
		
		for (ByteBufferFilter filter : filters) {
			filter.reset();
		}
		
		while(true) {
			
			int read;
			try {
				read = channel.read(buffer);
			} catch (IOException e) {
				throw new RuntimeException("Error while reading from channel: ", e);
			}

			if(read < 0) {
				break;
			}
			
			reinitBuffer(buffer, read);
			
			for(ByteBufferFilter filter: filters) {
				int limit = buffer.limit();
				filter.process(buffer);
				ChannelFilterer.reinitBuffer(buffer, limit);
			}
		}
	}
	
	protected static void reinitBuffer(ByteBuffer buffer, int limit) {
		buffer.rewind();
		buffer.limit(limit);
	}
}<commit_msg>Fix: close channel in ChannelFilterer<commit_after>/*
 *  Copyright <NAME> <<EMAIL>>
 *
 *  Licensed under the Apache License, Version 2.0; 
 *  you may obtain a copy of the License at:
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 */
package hu.webhejj.commons.io.filter;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.ReadableByteChannel;
import java.util.Arrays;
import java.util.List;

/**
 * Takes a list of ByteBufferFilters and applies them
 * to a File or ReadableByteChannel.
 *
 */
public class ChannelFilterer {
	
	private List<ByteBufferFilter> filters;
	
	public ChannelFilterer(ByteBufferFilter... filters) {
		this.filters = Arrays.asList(filters);
	}
	

	public void filter(File file, int bufSize) throws FileNotFoundException {
		filter(new FileInputStream(file).getChannel(), bufSize);
	}

	public void filter(ReadableByteChannel channel, int bufSize) {
		
		ByteBuffer buffer = ByteBuffer.allocate(bufSize);
		
		for (ByteBufferFilter filter : filters) {
			filter.reset();
		}
		
		while(true) {
			
			int read;
			try {
				read = channel.read(buffer);
			} catch (IOException e) {
				throw new RuntimeException("Error while reading from channel: ", e);
			}

			if(read < 0) {
				break;
			}
			
			reinitBuffer(buffer, read);
			
			for(ByteBufferFilter filter: filters) {
				int limit = buffer.limit();
				filter.process(buffer);
				ChannelFilterer.reinitBuffer(buffer, limit);
			}
		}
		
		try {
			channel.close();
		} catch (IOException e) {
			throw new RuntimeException("Error while closing channel: ", e);
		}
	}
	
	protected static void reinitBuffer(ByteBuffer buffer, int limit) {
		buffer.rewind();
		buffer.limit(limit);
	}
}
{'fraction_non_alphanumeric': 0.11395348837209303, 'fraction_numerical': 0.0029069767441860465, 'mean_word_length': 3.688010899182561, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7593488', 'n_tokens_mistral': 1217, 'n_tokens_neox': 1058, 'n_words': 272}