<commit_before>#!/usr/bin/env python

"""
Unfortunately the "expanded_url" as supplied by Twitter aren't fully
expanded one hop past t.co.

unshorten.py will attempt to completely unshorten URLs and add them as the
"unshortened_url" key to each url, and emit the tweet as JSON again on stdout.

This script starts 10 seaprate processes which talk to an instance of unshrtn
that is running:

    http://github.com/edsu/unshrtn

"""
from __future__ import print_function

import json
import urllib
import logging
import fileinput
import multiprocessing
import argparse
import time

# number of urls to look up in parallel
POOL_SIZE = 10<commit_msg>fixed doc<commit_after>#!/usr/bin/env python

"""
Unfortunately the "expanded_url" as supplied by Twitter aren't fully
expanded one hop past t.co.

unshrtn.py will attempt to completely unshorten URLs and add them as the
"unshortened_url" key to each url, and emit the tweet as JSON again on stdout.

This script starts 10 seaprate processes which talk to an instance of unshrtn
that is running:

    http://github.com/edsu/unshrtn

"""
from __future__ import print_function

import json
import urllib
import logging
import fileinput
import multiprocessing
import argparse
import time

# number of urls to look up in parallel
POOL_SIZE = 10
{'fraction_non_alphanumeric': 0.05132192846034215, 'fraction_numerical': 0.006220839813374806, 'mean_word_length': 4.644736842105263, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '460454', 'n_tokens_mistral': 401, 'n_tokens_neox': 380, 'n_words': 174}