<commit_before># $Id: test_getinfo.py,v 1.10 2002/06/18 12:09:54 kjetilja Exp $

## PycURL module
import pycurl

## Callback function invoked when progress information is updated
def progress(download_t, download_d, upload_t, upload_d):
    print 'Total to download %d bytes, have %d bytes so far' % \
          (download_t, download_d)

url = 'http://www.cnn.com'

print 'Starting downloading', url
print
f = open('body', 'w')
h = open('header', 'w')
c = pycurl.init()
c.setopt(pycurl.URL, url)
c.setopt(pycurl.FILE, f)
c.setopt(pycurl.NOPROGRESS, 0)
c.setopt(pycurl.PROGRESSFUNCTION, progress)
c.setopt(pycurl.FOLLOWLOCATION, 1)
c.setopt(pycurl.MAXREDIRS, 5)
c.setopt(pycurl.WRITEHEADER, h)
c.setopt(pycurl.OPT_FILETIME, 1)
c.perform()

print
print 'Download speed: %f bytes/second' % c.getinfo(pycurl.SPEED_DOWNLOAD)
print 'Document size: %d bytes' % c.getinfo(pycurl.SIZE_DOWNLOAD)
print 'Effective URL:', c.getinfo(pycurl.EFFECTIVE_URL)
print 'Content-type:', c.getinfo(pycurl.CONTENT_TYPE)
print 'Redirect-time:', c.getinfo(pycurl.REDIRECT_TIME)
print 'Redirect-count:', c.getinfo(pycurl.REDIRECT_COUNT)
print 'Filetime:', c.getinfo(pycurl.INFO_FILETIME)
print
print "Header is in file 'header', body is in file 'body'"

c.cleanup()
f.close()
h.close()<commit_msg>filetime test now included here<commit_after># $Id: test_getinfo.py,v 1.11 2002/06/18 12:55:01 kjetilja Exp $

import time

## PycURL module
import pycurl

## Callback function invoked when progress information is updated
def progress(download_t, download_d, upload_t, upload_d):
    print 'Total to download %d bytes, have %d bytes so far' % \
          (download_t, download_d)

url = 'http://www.cnn.com'

print 'Starting downloading', url
print
f = open('body', 'w')
h = open('header', 'w')
c = pycurl.init()
c.setopt(pycurl.URL, url)
c.setopt(pycurl.FILE, f)
c.setopt(pycurl.NOPROGRESS, 0)
c.setopt(pycurl.PROGRESSFUNCTION, progress)
c.setopt(pycurl.FOLLOWLOCATION, 1)
c.setopt(pycurl.MAXREDIRS, 5)
c.setopt(pycurl.WRITEHEADER, h)
c.setopt(pycurl.OPT_FILETIME, 1)
c.perform()

print
print 'Download speed: %f bytes/second' % c.getinfo(pycurl.SPEED_DOWNLOAD)
print 'Document size: %d bytes' % c.getinfo(pycurl.SIZE_DOWNLOAD)
print 'Effective URL:', c.getinfo(pycurl.EFFECTIVE_URL)
print 'Content-type:', c.getinfo(pycurl.CONTENT_TYPE)
print 'Redirect-time:', c.getinfo(pycurl.REDIRECT_TIME)
print 'Redirect-count:', c.getinfo(pycurl.REDIRECT_COUNT)
epoch = c.getinfo(pycurl.INFO_FILETIME)
print 'Filetime: %d (%s)' % (epoch, time.ctime(epoch))
print
print "Header is in file 'header', body is in file 'body'"

c.cleanup()
f.close()
h.close()
{'fraction_non_alphanumeric': 0.1464627151051625, 'fraction_numerical': 0.01606118546845124, 'mean_word_length': 5.54, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '198242', 'n_tokens_mistral': 1068, 'n_tokens_neox': 1004, 'n_words': 238}