<commit_before>#!/usr/bin/env python

import grequests
from optparse import OptionParser


def validate(results, verbose):
    for r in results:
        try:
            if r.status_code != 200:
                print('ERROR: %s %s' % (r.url, r.status_code))
                continue
            elif verbose:
                print('SUCCESS: %s %s' % (r.url, r.status_code))
            yield r
        except Exception as e:
            print('ERROR: %s' % e)


def main():
    # get argument
    parser = OptionParser(
        usage='Usage: %prog [<CDN_URL>]'
        '\n\nArguments:'
        '\n  CDN_URL    Of the format "<scheme>://<fqdn>".'
        ' Trailing "/" not allowed.'
        '\n\nExamples:'
        '\n  %prog https://tiles.cdn.mozilla.net'
    )
    parser.set_defaults(
        quiet=False,<commit_msg>disable urllib3 warnings in tile_index_crawl.py<commit_after>#!/usr/bin/env python

from optparse import OptionParser
import grequests
import requests
requests.packages.urllib3.disable_warnings()


def validate(results, verbose):
    for r in results:
        try:
            if r.status_code != 200:
                print('ERROR: %s %s' % (r.url, r.status_code))
                continue
            elif verbose:
                print('SUCCESS: %s %s' % (r.url, r.status_code))
            yield r
        except Exception as e:
            print('ERROR: %s' % e)


def main():
    # get argument
    parser = OptionParser(
        usage='Usage: %prog [<CDN_URL>]'
        '\n\nArguments:'
        '\n  CDN_URL    Of the format "<scheme>://<fqdn>".'
        ' Trailing "/" not allowed.'
        '\n\nExamples:'
        '\n  %prog https://tiles.cdn.mozilla.net'
    )
    parser.set_defaults(
        quiet=False,
{'fraction_non_alphanumeric': 0.13256484149855907, 'fraction_numerical': 0.004610951008645533, 'mean_word_length': 1.8646864686468647, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4710300', 'n_tokens_mistral': 574, 'n_tokens_neox': 536, 'n_words': 138}