<commit_before>from BeautifulSoup import BeautifulSoup
import urllib2
from datetime import date, timedelta
import re

def retrieve_hn_user_data(hn_username):
    hn_profile_url = "http://news.ycombinator.com/user?id=%s" % hn_username
    page = urllib2.urlopen(hn_profile_url).read()
    soup = BeautifulSoup(page)
    fields = soup.findAll("td", attrs={'valign': 'top'})
    user_data = dict()
    for f in fields:
        field_key = f.string.replace(":", "") 
        contents = f.findNextSibling().contents
        user_data[field_key] = " ".join(str(c) for c in contents)

    # convert the "created X days ago" to a python datetime
    if not "day" in user_data["created"]:
        days = 0
    else:
        days = int(re.match("\d+", user_data["created"]).group())
    user_data["created"] = date.today() - timedelta(days=days)
    return user_data

if __name__ == "__main__":
    loser = "ezl"
    user_data = retrieve_hn_user_data(loser)

<commit_msg>raise error if requested HN username isn't valid<commit_after>from BeautifulSoup import BeautifulSoup
import urllib2
from datetime import date, timedelta
import re

def retrieve_hn_user_data(hn_username):
    hn_profile_url = "http://news.ycombinator.com/user?id=%s" % hn_username
    page = urllib2.urlopen(hn_profile_url).read()
    soup = BeautifulSoup(page)
    fields = soup.findAll("td", attrs={'valign': 'top'})
    user_data = dict()
    for f in fields:
        field_key = f.string.replace(":", "") 
        contents = f.findNextSibling().contents
        user_data[field_key] = " ".join(str(c) for c in contents)

    expected_keys = ["user", "created", "karma", "about"]
    e = "Expected fields not in HN profile. Probably not valid username."
    assert all(map(lambda x: x in user_data.keys(), expected_keys)), e

    # convert the "created X days ago" to a python datetime
    if not "day" in user_data["created"]:
        days = 0
    else:
        days = int(re.match("\d+", user_data["created"]).group())
    user_data["created"] = date.today() - timedelta(days=days)
    return user_data

if __name__ == "__main__":
    loser = "ezl"
    user_data = retrieve_hn_user_data(loser)


{'fraction_non_alphanumeric': 0.13234613604812587, 'fraction_numerical': 0.002776492364645997, 'mean_word_length': 3.6196581196581197, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7357642', 'n_tokens_mistral': 728, 'n_tokens_neox': 718, 'n_words': 181}