<commit_before>package org.archive.url;

import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;

import org.apache.commons.httpclient.URIException;

import com.google.common.net.InetAddresses;

import junit.framework.TestCase;

public class URLParserTest extends TestCase {
	public void testGuava() throws URIException, UnsupportedEncodingException {
		Long l = Long.parseLong("3279880203");
		int i2 = l.intValue();
//		int i = Integer.decode("3279880203");
		System.err.format("FromNum(%s)\n", InetAddresses.fromInteger(i2).getHostAddress());
		
		
	}
	
	public void testAddDefaultSchemeIfNeeded() {
		assertEquals(null,URLParser.addDefaultSchemeIfNeeded(null));
		assertEquals("http://",URLParser.addDefaultSchemeIfNeeded(""));
		assertEquals("http://www.fool.com",URLParser.addDefaultSchemeIfNeeded("http://www.fool.com"));
		assertEquals("http://www.fool.com/",URLParser.addDefaultSchemeIfNeeded("http://www.fool.com/"));
		assertEquals("http://www.fool.com",URLParser.addDefaultSchemeIfNeeded("www.fool.com"));
		assertEquals("http://www.fool.com/",URLParser.addDefaultSchemeIfNeeded("www.fool.com/"));
	}

	
	public void testParse() throws URIException, UnsupportedEncodingException {
		System.out.format("O(%s) E(%s)\n","%66",URLDecoder.decode("%66","UTF-8"));
		dumpParse("http://www.archive.org/index.html#foo");
		dumpParse("http://www.archive.org/");
		dumpParse("http://www.archive.org");
		dumpParse("http://www.archive.org?");
		dumpParse("http://www.archive.org:8080/index.html?query#foo");
		dumpParse("http://www.archive.org:8080/index.html?#foo");
		dumpParse("http://www.archive.org:8080?#foo");
		dumpParse("http://bcher.ch:8080?#foo");
		
		dumpParse("dns:bcher.ch");

	}
	
	private void dumpParse(String s) throws URIException {
		HandyURL h = URLParser.parse(s);
		System.out.format("Input:(%s)\nHandyURL\t%s\n",s,h.toDebugString());
	}

}<commit_msg>fix badly encoded test string<commit_after>package org.archive.url;

import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;

import org.apache.commons.httpclient.URIException;

import com.google.common.net.InetAddresses;

import junit.framework.TestCase;

public class URLParserTest extends TestCase {
	public void testGuava() throws URIException, UnsupportedEncodingException {
		Long l = Long.parseLong("3279880203");
		int i2 = l.intValue();
//		int i = Integer.decode("3279880203");
		System.err.format("FromNum(%s)\n", InetAddresses.fromInteger(i2).getHostAddress());
		
		
	}
	
	public void testAddDefaultSchemeIfNeeded() {
		assertEquals(null,URLParser.addDefaultSchemeIfNeeded(null));
		assertEquals("http://",URLParser.addDefaultSchemeIfNeeded(""));
		assertEquals("http://www.fool.com",URLParser.addDefaultSchemeIfNeeded("http://www.fool.com"));
		assertEquals("http://www.fool.com/",URLParser.addDefaultSchemeIfNeeded("http://www.fool.com/"));
		assertEquals("http://www.fool.com",URLParser.addDefaultSchemeIfNeeded("www.fool.com"));
		assertEquals("http://www.fool.com/",URLParser.addDefaultSchemeIfNeeded("www.fool.com/"));
	}

	
	public void testParse() throws URIException, UnsupportedEncodingException {
		System.out.format("O(%s) E(%s)\n","%66",URLDecoder.decode("%66","UTF-8"));
		dumpParse("http://www.archive.org/index.html#foo");
		dumpParse("http://www.archive.org/");
		dumpParse("http://www.archive.org");
		dumpParse("http://www.archive.org?");
		dumpParse("http://www.archive.org:8080/index.html?query#foo");
		dumpParse("http://www.archive.org:8080/index.html?#foo");
		dumpParse("http://www.archive.org:8080?#foo");
		dumpParse("http://bŸcher.ch:8080?#foo");
		
		dumpParse("dns:bŸcher.ch");

	}
	
	private void dumpParse(String s) throws URIException {
		HandyURL h = URLParser.parse(s);
		System.out.format("Input:(%s)\nHandyURL\t%s\n",s,h.toDebugString());
	}

}
{'fraction_non_alphanumeric': 0.1829652996845426, 'fraction_numerical': 0.02260778128286015, 'mean_word_length': 5.8558558558558556, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 30, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6960310', 'n_tokens_mistral': 1395, 'n_tokens_neox': 1275, 'n_words': 144}