<commit_before>#!/usr/bin/env python            

import json
import fileinput
import sys
import optparse
from profiler import Profiler # local module
import dateutil.parser # $ pip install python-dateutil

opt_parser = optparse.OptionParser()
opt_parser.add_option("-o", "--output", dest="output", type="str", 
    help="text | json (default: json)", default="json")
opts, args = opt_parser.parse_args()

profiler = Profiler({"extended": True})

for line in fileinput.input(args):
    try:
        tweet = json.loads(line)
        profiler.process(tweet)
    except ValueError as e:
        sys.stderr.write("uhoh: %s\n" % e)

data = profiler.report()

if (opts.output == "json"):
    print data
else:
    print "Count:    " + '{:>9}'.format(str(data["count"]))
    print "Users:    " + '{:>9}'.format(str(data["usercount"]))
    print "Has URL:  " + '{:>9}'.format(str(data["urlcount"])) + " (" + str("%.2f" % (float(data["urlcount"]) / float(data["count"]) * 100.0)) + "%)"
    print "URLs:     " + '{:>9}'.format(str(data["urls"]))
    print "Retweets: " + '{:>9}'.format(str(data["retweetcount"])) + " (" + str("%.2f" % (float(data["retweetcount"]) / float(data["count"]) * 100.0)) + "%)"
    print "Geo:      " + '{:>9}'.format(str(data["geocount"])) + " (" + str("%.2f" % (float(data["geocount"]) / float(data["count"]) * 100.0)) + "%)"
    print "Earliest: " + str(data["earliest"])
    print "Latest:   " + str(data["latest"])
    print "Duration: " + str(dateutil.parser.parse(data["latest"]) - dateutil.parser.parse(data["earliest"]))

    print "Top users:"
    for user in data["topusers"]:
        print "{:>7}".format(str(user["value"])) + " " + user["name"]
    print "Top URLs:"
    for url in data["topurls"]:
        print "{:>7}".format(str(url["value"])) + " " + url["name"]<commit_msg>Add sparklines for top users and top urls<commit_after>#!/usr/bin/env python            

import json
import fileinput
import sys
import optparse
from profiler import Profiler # local module
import dateutil.parser # $ pip install python-dateutil
import sparkline

opt_parser = optparse.OptionParser()
opt_parser.add_option("-o", "--output", dest="output", type="str", 
    help="text | json (default: json)", default="json")
opts, args = opt_parser.parse_args()

profiler = Profiler({"extended": True})

for line in fileinput.input(args):
    try:
        tweet = json.loads(line)
        profiler.process(tweet)
    except ValueError as e:
        sys.stderr.write("uhoh: %s\n" % e)

data = profiler.report()

if (opts.output == "json"):
    print data
else:
    print "Count:    " + '{:>9}'.format(str(data["count"]))
    print "Users:    " + '{:>9}'.format(str(data["usercount"]))
    print "Has URL:  " + '{:>9}'.format(str(data["urlcount"])) + " (" + str("%.2f" % (float(data["urlcount"]) / float(data["count"]) * 100.0)) + "%)"
    print "URLs:     " + '{:>9}'.format(str(data["urls"]))
    print "Retweets: " + '{:>9}'.format(str(data["retweetcount"])) + " (" + str("%.2f" % (float(data["retweetcount"]) / float(data["count"]) * 100.0)) + "%)"
    print "Geo:      " + '{:>9}'.format(str(data["geocount"])) + " (" + str("%.2f" % (float(data["geocount"]) / float(data["count"]) * 100.0)) + "%)"
    print "Earliest: " + str(data["earliest"])
    print "Latest:   " + str(data["latest"])
    print "Duration: " + str(dateutil.parser.parse(data["latest"]) - dateutil.parser.parse(data["earliest"]))

    print "Top users: " + sparkline.sparkify([u["value"] for u in data["topusers"]]).encode("utf-8")
    for user in data["topusers"]:
        print "{:>7}".format(str(user["value"])) + " " + user["name"]
    print "Top URLs: " + sparkline.sparkify([u["value"] for u in data["topurls"]]).encode("utf-8")
    for url in data["topurls"]:
        print "{:>7}".format(str(url["value"])) + " " + url["name"]
{'fraction_non_alphanumeric': 0.22968012585212375, 'fraction_numerical': 0.012585212375458836, 'mean_word_length': 3.9162371134020617, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6766927', 'n_tokens_mistral': 1370, 'n_tokens_neox': 1368, 'n_words': 262}