<commit_before># $Id: test_multi2.py,v 1.1 2002/07/04 11:36:20 kjetilja Exp $

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO
import pycurl

body = StringIO()
m = pycurl.multi_init()
c1 = pycurl.init()
c2 = pycurl.init()
c1.setopt(pycurl.URL, 'http://curl.haxx.se')
c2.setopt(pycurl.URL, 'http://python.org')
c1.setopt(pycurl.WRITEFUNCTION, body.write)
c2.setopt(pycurl.WRITEFUNCTION, body.write)
m.add_handle(c1)
m.add_handle(c2)
while 1:
    num_handles = m.perform()
    if num_handles == 0:
        break
m.remove_handle(c2)
m.remove_handle(c1)
m.cleanup()
c1.cleanup()
c2.cleanup()
print body.getvalue()<commit_msg>Rewritten to use the new features.<commit_after># $Id: test_multi2.py,v 1.2 2002/07/04 22:09:47 mfx Exp $
# vi:ts=4:et

import os, sys, time
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO
import pycurl


urls = (
    "http://curl.haxx.se",
    "http://www.python.org",
    "http://pycurl.sourceforge.net",
    "http://pycurl.sourceforge.net/tests/403_FORBIDDEN",  # that actually exists ;-)
    "http://pycurl.sourceforge.net/tests/404_NOT_FOUND",
)

# Read list of URIs from file specified on commandline
try:
    urls = open(sys.argv[1]).readlines()
except IndexError:
    # No file was specified
    pass


# init
m = pycurl.multi_init()
m.handles = []
for url in urls:
    c = pycurl.init()
    # save info in standard Python attributes
    c.url = url
    c.body = StringIO()
    m.handles.append(c)
    # pycurl API calls
    c.setopt(pycurl.URL, c.url)
    c.setopt(pycurl.WRITEFUNCTION, c.body.write)
    m.add_handle(c)

# get data
while 1:
    num_handles = m.perform()
    if num_handles == 0:
        break
    # currently no more I/O is pending, could do something in the meantime
    # (display a progress bar, etc.)
    time.sleep(0.1)

# close handles
for c in m.handles:
    # save info in standard Python attributes
    c.http_code = c.getinfo(pycurl.HTTP_CODE)
    # pycurl API calls
    m.remove_handle(c)
    c.cleanup()
m.cleanup()

# print result
for c in m.handles:
    data = c.body.getvalue()
    if 0:
        print "**********", c.url, "**********"
        print data
    else:
        print "%-53s http_code %3d, %6d bytes" % (c.url, c.http_code, len(data))

{'fraction_non_alphanumeric': 0.1395045632333768, 'fraction_numerical': 0.02824858757062147, 'mean_word_length': 3.193078324225865, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5769041', 'n_tokens_mistral': 924, 'n_tokens_neox': 862, 'n_words': 213}