<commit_before>package study.hard.javalib.encodingstring;

import static org.junit.Assert.*;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.nio.charset.StandardCharsets;

import org.apache.commons.lang.StringUtils;
import org.junit.Test;

public class InputStreamTest {

	@Test
	public void testBytesRead() throws IOException {
		URL url = new URL("http://www.google.com");
		StringBuilder stringBuilder = new StringBuilder();
		BufferedReader bufferedReader = null;

		try {
			InputStream inputStream = url.openStream();
			if (inputStream != null) {
				bufferedReader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
				char[] charBuffer = new char[1024];
				int bytesRead = -1;
				while ((bytesRead = bufferedReader.read(charBuffer)) > 0) {
					stringBuilder.append(charBuffer, 0, bytesRead);
				}
			} else {
				stringBuilder.append("");
			}
		} catch (IOException ex) {
			throw ex;
		} finally {
			if (bufferedReader != null) {
				try {
					bufferedReader.close();
				} catch (IOException ex) {
					throw ex;
				}
			}
		}

		assertTrue(StringUtils.isNotBlank(stringBuilder.toString()));
		System.out.println("> " + stringBuilder);
		System.out.println("length: " + stringBuilder.length());
	}

}
<commit_msg>Add some code to test ReadLine.<commit_after>package study.hard.javalib.encodingstring;

import static org.junit.Assert.*;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.LinkedList;

import org.apache.commons.lang.StringUtils;
import org.junit.Before;
import org.junit.Test;

public class InputStreamTest {

	private URL url;

	@Before
	public void setUp() throws MalformedURLException {
		url = new URL("http://www.google.com");
	}

	@Test
	public void testBytesRead() throws IOException {
		StringBuilder stringBuilder = new StringBuilder();
		BufferedReader bufferedReader = null;

		try {
			InputStream inputStream = url.openStream();
			if (inputStream != null) {
				bufferedReader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
				char[] charBuffer = new char[1024];
				int bytesRead = -1;
				while ((bytesRead = bufferedReader.read(charBuffer)) > 0) {
					stringBuilder.append(charBuffer, 0, bytesRead);
				}
			} else {
				stringBuilder.append("");
			}
		} catch (IOException ex) {
			throw ex;
		} finally {
			if (bufferedReader != null) {
				try {
					bufferedReader.close();
				} catch (IOException ex) {
					throw ex;
				}
			}
		}

		assertTrue(StringUtils.isNotBlank(stringBuilder.toString()));
		System.out.println("> " + stringBuilder);
		System.out.println("length: " + stringBuilder.length());
	}

	@Test
	public void testReadLine() throws IOException {
		LinkedList<String> lines = new LinkedList<String>();
		BufferedReader bufferedReader = null;

		try {
			InputStream inputStream = url.openStream();
			if (inputStream != null) {
				bufferedReader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
				String readLine;
				while ((readLine = bufferedReader.readLine()) != null) {
					lines.add(readLine);
				}
			}
		} catch (IOException ex) {
			throw ex;
		} finally {
			if (bufferedReader != null) {
				try {
					bufferedReader.close();
				} catch (IOException ex) {
					throw ex;
				}
			}
		}

		assertTrue(0 < lines.size());
		for (String line : lines) {
			System.out.println("> " + line);
		}
	}
}

{'fraction_non_alphanumeric': 0.1242876526458616, 'fraction_numerical': 0.004884667571234735, 'mean_word_length': 3.824607329842932, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3145267', 'n_tokens_mistral': 1328, 'n_tokens_neox': 1112, 'n_words': 255}