<commit_before>#!/usr/bin/env python
import sys
import io
import gzip

import requests
from internetarchive import get_item


def get_gzip_file_from_url(url):
    r = requests.get(url)
    bi = io.BytesIO(r.content)
    return gzip.GzipFile(fileobj=bi, mode='rb')


if __name__ == '__main__':
    identifier = sys.argv[-1]
    item = get_item(identifier)
    count = 0
    for f in item.files():
        if f.format == 'Comma-Separated Values GZ':
            gf = get_gzip_file_from_url(f.url)
            count += len([x for x in gf if x]) - 1
    print count
    md = dict(imagecount=count)
    print item.metadata['metadata'].get('title')
    print '{0}\t{1}'.format(item.identifier, item.modify_metadata(md).get('status_code'))<commit_msg>use file objects not dicts<commit_after>#!/usr/bin/env python
import sys
import io
import gzip

import requests
from internetarchive import get_item


def get_gzip_file_from_url(url):
    r = requests.get(url)
    bi = io.BytesIO(r.content)
    return gzip.GzipFile(fileobj=bi, mode='rb')


if __name__ == '__main__':
    identifier = sys.argv[-1]
    item = get_item(identifier)
    count = 0
    for f in item.iter_files():
        if f.format == 'Comma-Separated Values GZ':
            gf = get_gzip_file_from_url(f.url)
            count += len([x for x in gf if x]) - 1
    md = dict(imagecount=count)
    print item.metadata['metadata'].get('title')
    print '{0}\t{1}'.format(item.identifier, item.modify_metadata(md).get('status_code'))
{'fraction_non_alphanumeric': 0.13154362416107382, 'fraction_numerical': 0.006711409395973154, 'mean_word_length': 3.3217391304347825, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5505049', 'n_tokens_mistral': 530, 'n_tokens_neox': 513, 'n_words': 123}