<commit_before>from munittest import *

base_http = 'http://www.linux.duke.edu/projects/urlgrabber/test/'
base_ftp  = 'ftp://localhost/test/'

# set to a proftp server only. we're working around a couple of
# bugs in their implementation in byterange.py.
base_proftp = 'ftp://localhost/test/'

reference_data = ''.join( [str(i)+'\n' for i in range(20000) ] )
ref_http = base_http + 'reference'
ref_ftp = base_ftp + 'reference'
ref_proftp = base_proftp + 'reference'
short_reference_data = ' '.join( [str(i) for i in range(10) ] )
short_ref_http = base_http + 'short_reference'
short_ref_ftp = base_ftp + 'short_reference'

ref_200 = ref_http
ref_404 = base_http + 'nonexistent_file'
ref_403 = base_http + 'mirror/broken/'

base_mirror_url    = base_http + 'mirror/'
good_mirrors       = ['m1', 'm2', 'm3']
mirror_files       = ['test1.txt', 'test2.txt']
bad_mirrors        = ['broken']<commit_msg>point to a new url for the tests still doesn't fix the http post test.<commit_after>from munittest import *

base_http = 'http://urlgrabber.baseurl.org/test/'
base_ftp  = 'ftp://localhost/test/'

# set to a proftp server only. we're working around a couple of
# bugs in their implementation in byterange.py.
base_proftp = 'ftp://localhost/test/'

reference_data = ''.join( [str(i)+'\n' for i in range(20000) ] )
ref_http = base_http + 'reference'
ref_ftp = base_ftp + 'reference'
ref_proftp = base_proftp + 'reference'
short_reference_data = ' '.join( [str(i) for i in range(10) ] )
short_ref_http = base_http + 'short_reference'
short_ref_ftp = base_ftp + 'short_reference'

ref_200 = ref_http
ref_404 = base_http + 'nonexistent_file'
ref_403 = base_http + 'mirror/broken/'

base_mirror_url    = base_http + 'mirror/'
good_mirrors       = ['m1', 'm2', 'm3']
mirror_files       = ['test1.txt', 'test2.txt']
bad_mirrors        = ['broken']
{'fraction_non_alphanumeric': 0.14005449591280653, 'fraction_numerical': 0.022888283378746595, 'mean_word_length': 4.3841642228739, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6237648', 'n_tokens_mistral': 713, 'n_tokens_neox': 661, 'n_words': 165}