<commit_before>#!/usr/bin/env python

"""
Unfortunately the "expanded_url" as supplied by Twitter aren't fully 
expanded past t.co.

unshorten.py will attempt to unshorten URLs and add them as the 
"unshortened_url" key to each url, and emit the tweet as JSON again on stdout. 
"""

import json
import logging
import requests
import fileinput
import urlparse
import multiprocessing

# number of urls to look up in parallel
POOL_SIZE = 10

logging.basicConfig(filename="unshorten.log", level=logging.INFO)

cache = {}

def unshorten(url):
    if url in cache:
        return cache[url]
    try:
        # let requests handle redirects
        resp = requests.get(url, timeout=5)
        cache[url] = resp.url
        return unshorten(resp.url)
    except Exception as e:
        logging.error("lookup failed for %s: %s", url, e)
        return url

def rewrite_line(line):
    tweet = json.loads(line)
    for url_dict in tweet["entities"]["urls"]:
        if "expanded_url" in url_dict:
            url = url_dict["expanded_url"]
        else:
            url = url_dict['url']

        unshortened_url = unshorten(url)
        logging.info("unshortened %s to %s", url, unshortened_url)

        # add new key to the json
        url_dict['unshortened_url'] = unshortened_url
    return tweet

def main():
    pool = multiprocessing.Pool(POOL_SIZE)
    for tweet in pool.imap(rewrite_line, fileinput.input()):
        print json.dumps(tweet).encode('utf8')

if __name__ == "__main__":
    main()<commit_msg>imap_unordered uses way less memory on long iterations<commit_after>#!/usr/bin/env python

"""
Unfortunately the "expanded_url" as supplied by Twitter aren't fully 
expanded past t.co.

unshorten.py will attempt to unshorten URLs and add them as the 
"unshortened_url" key to each url, and emit the tweet as JSON again on stdout. 
"""

import json
import logging
import requests
import fileinput
import urlparse
import multiprocessing

# number of urls to look up in parallel
POOL_SIZE = 10

logging.basicConfig(filename="unshorten.log", level=logging.INFO)

cache = {}
def unshorten(url):
    if url in cache:
        return cache[url]
    try:
        # let requests handle redirects
        resp = requests.get(url, timeout=5)
        cache[url] = resp.url
        return unshorten(resp.url)
    except Exception as e:
        logging.error("lookup failed for %s: %s", url, e)
        return url

def rewrite_line(line):
    tweet = json.loads(line)
    for url_dict in tweet["entities"]["urls"]:
        if "expanded_url" in url_dict:
            url = url_dict["expanded_url"]
        else:
            url = url_dict['url']

        unshortened_url = unshorten(url)
        logging.info("unshortened %s to %s", url, unshortened_url)

        # add new key to the json
        url_dict['unshortened_url'] = unshortened_url
    return tweet

def main():
    pool = multiprocessing.Pool(POOL_SIZE)
    for tweet in pool.imap_unordered(rewrite_line, fileinput.input()):
        print json.dumps(tweet).encode('utf8')

if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.09725848563968668, 'fraction_numerical': 0.0026109660574412533, 'mean_word_length': 3.0921228304405877, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6843583', 'n_tokens_mistral': 993, 'n_tokens_neox': 977, 'n_words': 300}