<commit_before>import urllib2
import hashlib
import os

basePath = ''

def retrieveAndSave(url):
    if len(url) == 0:
       print 'Warning Empty URL for a media path'
       return

    if url.startswith('/'):
        fullURL = basePath + url
    else:  
        fullURL = url

    try:
        response = urllib2.urlopen(fullURL)
    except urllib2.HTTPError:
        print 'error downloading: ' + url
        raise<commit_msg>a hack (hopefully) temporary to download all the tour content<commit_after>import urllib2
import hashlib
import os

basePath = ''

def retrieveAndSave(url):
    if len(url) == 0:
       print 'Warning Empty URL for a media path'
       return

    if url.startswith('/'):
        fullURL = basePath + url
    else:  
        fullURL = url

    # this is a bit of a hack, probably should remove this (if the server changes)
    fullURLParts = fullURL.split('/')
    fullURLParts[-1] = urllib2.quote(fullURLParts[-1])
    fullURL = '/'.join(fullURLParts)

    try:
        response = urllib2.urlopen(fullURL)
    except urllib2.HTTPError:
        print 'error downloading: ' + url
        raise
{'fraction_non_alphanumeric': 0.0944741532976827, 'fraction_numerical': 0.00980392156862745, 'mean_word_length': 2.498442367601246, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5097604', 'n_tokens_mistral': 359, 'n_tokens_neox': 350, 'n_words': 107}