<commit_before>'''
Fetches some urls using aiohttp. Also serves as a minimum example of using aiohttp.

Good examples:

https://www.enterprisecarshare.com/robots.txt -- 302 redir lacking Location: raises RuntimeError

'''

import sys
from traceback import print_exc

import asyncio
import aiohttp

async def main(urls):
    for url in urls:
        async with aiohttp.ClientSession() as session:
            print(url, '\n')
            try:
                response = await session.get(url, allow_redirects=True)
            except Exception as e:
                print_exc()
                print('')
                continue

            for h in response.history:
                print(h)
            print('')
            for h in response.raw_headers:
                line = h[0]+b': '+h[1]
                print(line.decode(errors='ignore'))
            print('')

            try:
                print(await response.text(errors='ignore'))
            except Exception as e:
                print_exc()

loop = asyncio.get_event_loop()

loop.run_until_complete(main(sys.argv[1:]))
<commit_msg>stick http:// on if scheme missing; pep8<commit_after>'''
Fetches some urls using aiohttp. Also serves as a minimum example of using aiohttp.

Good examples:

https://www.enterprisecarshare.com/robots.txt -- 302 redir lacking Location: raises RuntimeError

'''

import sys
from traceback import print_exc

import asyncio
import aiohttp


async def main(urls):
    for url in urls:
        if not url.startswith('http'):
            url = 'http://' + url
        async with aiohttp.ClientSession() as session:
            print(url, '\n')
            try:
                response = await session.get(url, allow_redirects=True)
            except Exception as e:
                print_exc()
                print('')
                continue

            for h in response.history:
                print(h)
            print('')
            for h in response.raw_headers:
                line = h[0]+b': '+h[1]
                print(line.decode(errors='ignore'))
            print('')

            try:
                print(await response.text(errors='ignore'))
            except Exception as e:
                print_exc()

loop = asyncio.get_event_loop()

loop.run_until_complete(main(sys.argv[1:]))

{'fraction_non_alphanumeric': 0.1002169197396963, 'fraction_numerical': 0.005639913232104121, 'mean_word_length': 1.843403205918619, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1326038', 'n_tokens_mistral': 664, 'n_tokens_neox': 634, 'n_words': 184}