<commit_before>/*
 *   Copyright 2014 <NAME>
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 * 
 */
package com.tomgibara.radix4.test;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.StringWriter;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.Iterator;
import java.util.Random;
import java.util.regex.Pattern;

import com.tomgibara.radix4.Radix4;
import com.tomgibara.radix4.Radix4Coding;
import com.tomgibara.radix4.Radix4Config;


import junit.framework.TestCase;

public class Radix4CorrectnessTest extends TestCase {

	private static final boolean inspect = System.getProperty("inspect", "false").equals("true");
	private static final Charset ASCII = Charset.forName("ASCII");
	private static final byte[] buffer = new byte[1024];
	private static final Pattern pattern = Pattern.compile("^([-_A-Za-z0-9.]+(\\s*[-_A-Za-z0-9.]+)*)?$");
	private static final int TEST_COUNT = 10000;
<commit_msg>Removed unused import.<commit_after>/*
 *   Copyright 2014 <NAME>
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 * 
 */
package com.tomgibara.radix4.test;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.StringWriter;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.Iterator;
import java.util.Random;
import java.util.regex.Pattern;

import com.tomgibara.radix4.Radix4;
import com.tomgibara.radix4.Radix4Coding;


import junit.framework.TestCase;

public class Radix4CorrectnessTest extends TestCase {

	private static final boolean inspect = System.getProperty("inspect", "false").equals("true");
	private static final Charset ASCII = Charset.forName("ASCII");
	private static final byte[] buffer = new byte[1024];
	private static final Pattern pattern = Pattern.compile("^([-_A-Za-z0-9.]+(\\s*[-_A-Za-z0-9.]+)*)?$");
	private static final int TEST_COUNT = 10000;

{'fraction_non_alphanumeric': 0.10507824976045992, 'fraction_numerical': 0.017885659533695305, 'mean_word_length': 4.3907056798623065, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6889500', 'n_tokens_mistral': 964, 'n_tokens_neox': 931, 'n_words': 302}