<commit_before>import urllib
import os


def process(url, destination):
    destination = "jslibs/%s" % destination

    if os.path.exists(destination):
        return

    try:
        print url
        urllib.urlretrieve(url, destination)
    except Exception:
        print "Failed"


def get_pattern(prefix, url_pattern, versions):
    for version in versions:
        url = url_pattern % version
        process(url, "%s.%s.%s" % (prefix, version, url.split("/")[-1]))


process("https://addons.cdn.mozilla.net/en-US/firefox/files/browse/149703/"
        "file-serve/bootstrap.js?token=<PASSWORD>",
        "PersonasInteractive_bootstrap.js")


DOJO_VERSIONS = ("1.1.1", "1.2.0", "1.2.3", "1.3.0", "1.3.1", "1.3.2", "1.4.0",
                 "1.4.1", "1.4.3", "1.5", "1.5.1", "1.6", "1.6.1", "1.7.0",
                 "1.7.1", "1.7.2", "1.7.3", "1.8.0")
# Google doesn't yet host EXT4.*
EXT_VERSIONS = ("3.0.0", "3.1.0")
JQUERY_CODE_VERSIONS = ("1.0.pack", "1.0.1.pack", "1.0.1", "1.0.2.pack",
                        "1.0.2", "1.0.3.pack", "1.0.3", "1.0.4.pack", "1.0.4",
                        "1.1.pack", "1.1", "1.1.1.pack", "1.1.1", "1.1.2.pack",
                        "1.1.2", "1.1.3.pack", "1.4.min", "1.4", "1.4.1.min",
                        "1.4.1", "1.4.2.min", "1.4.2", "1.4.3.min", "1.4.4",
                        "1.5.min", "1.5", "1.5.1.min", "1.5.1", "1.5.2.min",
                        "1.5.2", "1.6.min", "1.6", "1.6.1.min", "1.6.1",
                        "1.6.2.min", "1.6.2", "1.6.3.min", "1.6.3",
                        "1.6.4.min", "1.6.4", "1.7.min", "1.7", "1.7.1.min",
                        "1.7.1", "1.7.2.min", "1.7.2", "1.8.0.min", "1.8.0" )
JQUERY_GCODE_VERSIONS = ("1.1.3", "1.1.3.1.pack", "1.1.3.1", "1.1.4.pack",
                         "1.1.4", "1.2.min", "1.2.pack", "1.2", "1.2.1.min",
                         "1.2.1.pack", "1.2.1", "1.2.2.pack", "1.2.2.min",
                         "1.2.2", "1.2.3.pack", "1.2.3.min", "1.2.3",
                         "1.2.4.min", "1.2.4.pack", "1.2.4", "1.2.5.min",
                         "1.2.5.pack", "1.2.5", "1.2.6.min", "1.2.6.pack",
                         "1.2.6", "1.3.min", "1.3", "1.3.1.min", "1.3.1",
                         "1.3.2.min", "1.3.2")<commit_msg>Added jQuery libs 1.8.1 and 1.8.2<commit_after>import urllib
import os


def process(url, destination):
    destination = "jslibs/%s" % destination

    if os.path.exists(destination):
        return

    try:
        print url
        urllib.urlretrieve(url, destination)
    except Exception:
        print "Failed"


def get_pattern(prefix, url_pattern, versions):
    for version in versions:
        url = url_pattern % version
        process(url, "%s.%s.%s" % (prefix, version, url.split("/")[-1]))


process("https://addons.cdn.mozilla.net/en-US/firefox/files/browse/149703/"
        "file-serve/bootstrap.js?token=<PASSWORD>",
        "PersonasInteractive_bootstrap.js")


DOJO_VERSIONS = ("1.1.1", "1.2.0", "1.2.3", "1.3.0", "1.3.1", "1.3.2", "1.4.0",
                 "1.4.1", "1.4.3", "1.5", "1.5.1", "1.6", "1.6.1", "1.7.0",
                 "1.7.1", "1.7.2", "1.7.3", "1.8.0")
# Google doesn't yet host EXT4.*
EXT_VERSIONS = ("3.0.0", "3.1.0")
JQUERY_CODE_VERSIONS = ("1.0.pack", "1.0.1.pack", "1.0.1", "1.0.2.pack",
                        "1.0.2", "1.0.3.pack", "1.0.3", "1.0.4.pack", "1.0.4",
                        "1.1.pack", "1.1", "1.1.1.pack", "1.1.1", "1.1.2.pack",
                        "1.1.2", "1.1.3.pack", "1.4.min", "1.4", "1.4.1.min",
                        "1.4.1", "1.4.2.min", "1.4.2", "1.4.3.min", "1.4.4",
                        "1.5.min", "1.5", "1.5.1.min", "1.5.1", "1.5.2.min",
                        "1.5.2", "1.6.min", "1.6", "1.6.1.min", "1.6.1",
                        "1.6.2.min", "1.6.2", "1.6.3.min", "1.6.3",
                        "1.6.4.min", "1.6.4", "1.7.min", "1.7", "1.7.1.min",
                        "1.7.1", "1.7.2.min", "1.7.2", "1.8.0.min", "1.8.0",
                        "1.8.1min", "1.8.1", "1.8.2min", "1.8.2" )
JQUERY_GCODE_VERSIONS = ("1.1.3", "1.1.3.1.pack", "1.1.3.1", "1.1.4.pack",
                         "1.1.4", "1.2.min", "1.2.pack", "1.2", "1.2.1.min",
                         "1.2.1.pack", "1.2.1", "1.2.2.pack", "1.2.2.min",
                         "1.2.2", "1.2.3.pack", "1.2.3.min", "1.2.3",
                         "1.2.4.min", "1.2.4.pack", "1.2.4", "1.2.5.min",
                         "1.2.5.pack", "1.2.5", "1.2.6.min", "1.2.6.pack",
                         "1.2.6", "1.3.min", "1.3", "1.3.1.min", "1.3.1",
                         "1.3.2.min", "1.3.2")
{'fraction_non_alphanumeric': 0.27741655830082357, 'fraction_numerical': 0.1304724750758561, 'mean_word_length': 1.413702928870293, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1421759', 'n_tokens_mistral': 2196, 'n_tokens_neox': 2128, 'n_words': 308}